diff --git a/.cargo/config.toml b/.cargo/config.toml index 810054d3158..f6795e3c6ec 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,7 +1,3 @@ -[alias] -# Warnings create a lot of noise, we only print errors. -check-clippy = "clippy --no-deps -- --allow warnings" - # Can be safely removed once Cargo's sparse protocol (see # https://blog.rust-lang.org/2023/03/09/Rust-1.68.0.html#cargos-sparse-protocol) # becomes the default. diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..3fa8a93aced --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,15 @@ +[[profile.default.overrides]] +filter = 'package(graphman-server)' +priority = -1 +threads-required = 'num-test-threads' # Global mutex + +[[profile.default.overrides]] +filter = 'package(test-store)' +priority = -2 +threads-required = 'num-test-threads' # Global mutex + +[[profile.default.overrides]] +filter = 'package(graph-tests)' +priority = -3 +threads-required = 'num-test-threads' # Global mutex +slow-timeout = { period = "300s", terminate-after = 4 } diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 102a8b53e78..96fa5ba1cb8 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v2 - - uses: actions-rs/audit-check@v1 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 #v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b400f1fbd7..59bbd6ce598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,21 +1,30 @@ name: Continuous Integration - on: push: branches: [master] pull_request: - types: [opened, synchronize, reopened] + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full - THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://graph:graph@localhost:5432/graph-test" + GRAPH_STORE_CONNECTION_TIMEOUT: "30000" jobs: unit-tests: name: Run unit tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -24,91 +33,104 @@ jobs: postgres: image: postgres env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_USER: graph + POSTGRES_PASSWORD: graph + POSTGRES_DB: graph-test + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- - --health-cmd pg_isready + --health-cmd "pg_isready -U graph" --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres ports: - 5432:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - - name: Tune GitHub hosted runner to reduce flakiness - # https://github.com/smorimoto/tune-github-hosted-runner-network/blob/main/action.yml - run: sudo ethtool -K eth0 tx off rx off - - name: Checkout sources - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Install lld - run: sudo apt-get install -y lld protobuf-compiler + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - - name: Run unit tests - uses: actions-rs/cargo@v1 + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - command: test - args: --verbose --workspace --exclude graph-tests -- --nocapture + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + + - name: Run unit tests + run: just test-unit runner-tests: name: Subgraph Runner integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 ports: - 5001:5001 postgres: - image: bitnami/postgresql + image: postgres env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1000" + POSTGRES_USER: graph + POSTGRES_PASSWORD: graph + POSTGRES_DB: graph-test + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- - --health-cmd "pg_isready -U postgres" + --health-cmd "pg_isready -U graph" --health-interval 10s --health-timeout 5s - --health-retries 10 + --health-retries 5 + --name postgres ports: - 5432:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" - RUNNER_TESTS_WAIT_FOR_SYNC_SECS: "600" steps: - - name: Tune GitHub hosted runner to reduce flakiness - # https://github.com/smorimoto/tune-github-hosted-runner-network/blob/main/action.yml - run: sudo ethtool -K eth0 tx off rx off - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Install Node 20 - uses: actions/setup-node@v3 - with: - node-version: "20" - cache: yarn - cache-dependency-path: "tests/runner-tests/yarn.lock" + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - - uses: Swatinem/rust-cache@v2 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Install lld - run: sudo apt-get install -y lld protobuf-compiler + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - - name: Run runner tests - id: runner-tests-1 - uses: actions-rs/cargo@v1 + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - command: test - args: --package graph-tests --test runner_tests + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + + - name: Run runner tests + run: just test-runner integration-tests: name: Run integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -120,60 +142,61 @@ jobs: POSTGRES_USER: graph-node POSTGRES_PASSWORD: let-me-in POSTGRES_DB: graph-node - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- - --health-cmd pg_isready + --health-cmd "pg_isready -U graph-node" --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres ports: - 3011:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - - name: Tune GitHub hosted runner to reduce flakiness - # https://github.com/smorimoto/tune-github-hosted-runner-network/blob/main/action.yml - run: sudo ethtool -K eth0 tx off rx off - - name: Checkout sources - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 - - - name: Install Node 20 - uses: actions/setup-node@v3 - with: - node-version: "20" - cache: yarn - cache-dependency-path: "tests/integration-tests/yarn.lock" + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Install lld and jq - run: sudo apt-get install -y lld jq protobuf-compiler + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 - - name: Start anvil - run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --port 3021 & + uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1 + with: + version: v1.4.0 - - name: Install graph CLI - run: curl -sSL http://cli.thegraph.com/install.sh | sudo bash + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 - - name: Build graph-node - uses: actions-rs/cargo@v1 + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - command: build - args: --bin graph-node --test integration_tests + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + + - name: Start anvil + run: anvil --gas-limit 100000000000 --base-fee 1 --timestamp 1743944919 --port 3021 & + + - name: Build graph-node + run: just build --test integration_tests - name: Run integration tests - id: integration-tests-1 - uses: actions-rs/cargo@v1 - env: - # Basically, unlimted concurrency - N_CONCURRENT_TESTS: "1000" - with: - command: test - args: --test integration_tests -- --nocapture + run: | + export PATH="${{ github.workspace }}/node_modules/.bin:$PATH" + just test-integration + - name: Cat graph-node.log if: always() run: cat tests/integration-tests/graph-node.log || echo "No graph-node.log" + rustfmt: name: Check rustfmt style runs-on: ubuntu-latest @@ -181,45 +204,55 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Check formatting - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + run: just format --check clippy: name: Clippy linting runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 10 + env: + RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v2 - # Unlike rustfmt, Clippy actually compiles stuff so it benefits from - # caching. - - uses: Swatinem/rust-cache@v2 - - name: Install deps - run: sudo apt-get install -y protobuf-compiler - - name: Run Clippy - uses: actions-rs/cargo@v1 - with: - command: check-clippy + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Run linting + run: just lint release-check: name: Build in release mode runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 10 env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 - - name: Install dependencies + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - name: Setup dependencies run: | sudo apt-get update - sudo apt-get -y install libpq-dev protobuf-compiler + sudo apt-get install -y protobuf-compiler + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + - name: Cargo check (release) - uses: actions-rs/cargo@v1 - with: - command: check - args: --release \ No newline at end of file + run: just check --release diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml deleted file mode 100644 index 5ca14794035..00000000000 --- a/.github/workflows/code-coverage.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Code coverage - -on: - workflow_dispatch: - schedule: - # Run it every 3 days. - - cron: "0 3 * * *" - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: full - THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" - N_CONCURRENT_TESTS: "4" - TESTS_GANACHE_HARD_WAIT_SECONDS: "30" - -jobs: - # Heavily inspired from . - coverage: - name: Code coverage of integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 - services: - ipfs: - image: ipfs/go-ipfs:v0.10.0 - ports: - - 5001:5001 - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Install Node 14 - uses: actions/setup-node@v3 - with: - node-version: "14" - cache: yarn - cache-dependency-path: "tests/integration-tests/yarn.lock" - - name: Install lld - run: sudo apt-get install -y lld jq protobuf-compiler - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-llvm-cov - - - name: Build graph-node - uses: actions-rs/cargo@v1 - with: - command: build - args: --bin graph-node - - - name: Generate code coverage - run: cargo llvm-cov --package graph-tests --lcov --output-path lcov.info -- --nocapture - - uses: actions/upload-artifact@v3 - with: - name: code-coverage-info - path: lcov.info - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - # No token needed, because the repo is public. - files: lcov.info - fail_ci_if_error: true diff --git a/.github/workflows/gnd-binary-build.yml b/.github/workflows/gnd-binary-build.yml new file mode 100644 index 00000000000..237de3342b5 --- /dev/null +++ b/.github/workflows/gnd-binary-build.yml @@ -0,0 +1,355 @@ +name: Build gnd Binaries + +on: + release: + types: [published] + workflow_dispatch: + inputs: + release_tag: + description: 'Existing release tag to upload binaries to and publish as on npm (e.g. v0.44.0). Leave empty to only build artifacts.' + type: string + default: '' + dry_run: + description: 'Dry-run npm publish (no actual publish)' + type: boolean + default: false + +jobs: + build: + name: Build gnd for ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + runner: ubuntu-22.04 + asset_name: gnd-linux-x86_64 + - target: aarch64-unknown-linux-gnu + runner: ubuntu-22.04 + asset_name: gnd-linux-aarch64 + - target: x86_64-apple-darwin + runner: macos-14 + asset_name: gnd-macos-x86_64 + - target: aarch64-apple-darwin + runner: macos-latest + asset_name: gnd-macos-aarch64 + - target: x86_64-pc-windows-msvc + runner: windows-latest + asset_name: gnd-windows-x86_64.exe + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Cache built binary + id: bin-cache + uses: actions/cache@v5 + with: + path: | + ${{ matrix.asset_name }}.gz + ${{ matrix.asset_name }}.zip + key: gnd-${{ matrix.target }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', '**/*.rs') }} + + - name: Install Rust toolchain + if: steps.bin-cache.outputs.cache-hit != 'true' + run: | + rustup toolchain install stable + rustup target add ${{ matrix.target }} + rustup default stable + + - name: Rust Cache + if: steps.bin-cache.outputs.cache-hit != 'true' + uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.target }} + + - name: Install dependencies (Ubuntu) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler musl-tools + if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then + sudo apt-get install -y gcc-aarch64-linux-gnu + fi + + - name: Install dependencies (macOS) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') + run: | + brew install protobuf + + - name: Install protobuf (Windows) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') + run: choco install protoc + + - name: Build gnd binary (Unix/Mac) + if: steps.bin-cache.outputs.cache-hit != 'true' && !startsWith(matrix.runner, 'windows') + run: cargo build --bin gnd --release --target ${{ matrix.target }} + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + + - name: Build gnd binary (Windows) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') + run: cargo build --bin gnd --release --target ${{ matrix.target }} + + - name: Sign macOS binary + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') + uses: lando/code-sign-action@v3 + with: + file: target/${{ matrix.target }}/release/gnd + certificate-data: ${{ secrets.APPLE_CERT_DATA }} + certificate-password: ${{ secrets.APPLE_CERT_PASSWORD }} + certificate-id: ${{ secrets.APPLE_TEAM_ID }} + options: --options runtime --entitlements entitlements.plist + + - name: Notarize macOS binary + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') + uses: lando/notarize-action@v2 + with: + product-path: target/${{ matrix.target }}/release/gnd + appstore-connect-username: ${{ secrets.NOTARIZATION_USERNAME }} + appstore-connect-password: ${{ secrets.NOTARIZATION_PASSWORD }} + appstore-connect-team-id: ${{ secrets.APPLE_TEAM_ID }} + + - name: Prepare binary (Unix) + if: steps.bin-cache.outputs.cache-hit != 'true' && !startsWith(matrix.runner, 'windows') + run: | + cp target/${{ matrix.target }}/release/gnd ${{ matrix.asset_name }} + chmod +x ${{ matrix.asset_name }} + gzip ${{ matrix.asset_name }} + + - name: Prepare binary (Windows) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') + run: | + copy target\${{ matrix.target }}\release\gnd.exe ${{ matrix.asset_name }} + 7z a -tzip ${{ matrix.asset_name }}.zip ${{ matrix.asset_name }} + + - name: Upload artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.asset_name }} + path: | + ${{ matrix.asset_name }}.gz + ${{ matrix.asset_name }}.zip + if-no-files-found: error + + release: + name: Create Release + needs: build + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup GitHub CLI + run: | + # GitHub CLI is pre-installed on GitHub-hosted runners + gh --version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Download all artifacts + uses: actions/download-artifact@v8 + with: + path: artifacts + + - name: Display structure of downloaded artifacts + run: ls -R artifacts + + - name: Upload Assets to Release + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + + gh release upload $VERSION --clobber --repo $GITHUB_REPOSITORY \ + artifacts/gnd-linux-x86_64/gnd-linux-x86_64.gz \ + artifacts/gnd-linux-aarch64/gnd-linux-aarch64.gz \ + artifacts/gnd-macos-x86_64/gnd-macos-x86_64.gz \ + artifacts/gnd-macos-aarch64/gnd-macos-aarch64.gz \ + artifacts/gnd-windows-x86_64.exe/gnd-windows-x86_64.exe.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-npm: + name: Publish npm package for ${{ matrix.platform }} + needs: release + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + asset: gnd-linux-x86_64.gz + os_field: linux + cpu_field: x64 + extract: gunzip + - platform: linux-arm64 + asset: gnd-linux-aarch64.gz + os_field: linux + cpu_field: arm64 + extract: gunzip + - platform: darwin-x64 + asset: gnd-macos-x86_64.gz + os_field: darwin + cpu_field: x64 + extract: gunzip + - platform: darwin-arm64 + asset: gnd-macos-aarch64.gz + os_field: darwin + cpu_field: arm64 + extract: gunzip + - platform: win32-x64 + asset: gnd-windows-x86_64.exe.zip + os_field: win32 + cpu_field: x64 + extract: unzip + steps: + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Download gnd binary + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" \ + --repo "${{ github.repository }}" \ + --pattern "${{ matrix.asset }}" \ + --output ./binary-archive + + - name: Extract binary + run: | + mkdir -p pkg/bin + if [ "${{ matrix.extract }}" = "gunzip" ]; then + gunzip -c ./binary-archive > pkg/bin/gnd + chmod +x pkg/bin/gnd + else + unzip ./binary-archive -d pkg/bin + mv pkg/bin/*.exe pkg/bin/gnd.exe + fi + + - name: Determine version and npm tag + id: version + shell: bash + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + VERSION="${VERSION#v}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + # Prerelease versions (e.g. 0.42.2-dev.1) need an explicit --tag + if [[ "$VERSION" == *-* ]]; then + # Extract prerelease identifier (e.g. "dev" from "0.42.2-dev.1") + PRE="${VERSION#*-}" + TAG="${PRE%%.*}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi + + - name: Create package.json + shell: bash + run: | + if [ "${{ matrix.os_field }}" = "win32" ]; then + BIN_PATH="./bin/gnd.exe" + else + BIN_PATH="./bin/gnd" + fi + + cat > pkg/package.json << EOF + { + "name": "@graphprotocol/gnd-${{ matrix.platform }}", + "version": "${{ steps.version.outputs.version }}", + "description": "gnd binary for ${{ matrix.platform }}", + "os": ["${{ matrix.os_field }}"], + "cpu": ["${{ matrix.cpu_field }}"], + "bin": { + "gnd": "${BIN_PATH}" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "license": "(Apache-2.0 OR MIT)", + "repository": { + "type": "git", + "url": "https://github.com/graphprotocol/graph-node.git" + } + } + EOF + + - name: Publish + run: npm publish --provenance --access public --tag ${{ steps.version.outputs.tag }} ${{ inputs.dry_run && '--dry-run' || '' }} + working-directory: pkg + + publish-npm-wrapper: + name: Publish @graphprotocol/gnd wrapper + needs: publish-npm + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Determine version and npm tag + id: version + shell: bash + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + VERSION="${VERSION#v}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + if [[ "$VERSION" == *-* ]]; then + PRE="${VERSION#*-}" + TAG="${PRE%%.*}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi + + - name: Create wrapper package + shell: bash + run: | + VERSION="${{ steps.version.outputs.version }}" + + mkdir -p pkg/bin + cp gnd/npm/bin/gnd.js pkg/bin/gnd.js + cp gnd/npm/README.md pkg/README.md + + cat > pkg/package.json << EOF + { + "name": "@graphprotocol/gnd", + "version": "${VERSION}", + "description": "Graph Node Development CLI", + "bin": { + "gnd": "./bin/gnd.js" + }, + "optionalDependencies": { + "@graphprotocol/gnd-darwin-arm64": "${VERSION}", + "@graphprotocol/gnd-darwin-x64": "${VERSION}", + "@graphprotocol/gnd-linux-arm64": "${VERSION}", + "@graphprotocol/gnd-linux-x64": "${VERSION}", + "@graphprotocol/gnd-win32-x64": "${VERSION}" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "license": "(Apache-2.0 OR MIT)", + "repository": { + "type": "git", + "url": "https://github.com/graphprotocol/graph-node.git" + } + } + EOF + + - name: Publish + run: npm publish --provenance --access public --tag ${{ steps.version.outputs.tag }} ${{ inputs.dry_run && '--dry-run' || '' }} + working-directory: pkg \ No newline at end of file diff --git a/.gitignore b/.gitignore index 15ad2465251..0480ebcf19b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,14 +22,20 @@ lcov.info /tests/**/build /tests/**/generated -/tests/**/node_modules -/tests/**/yarn-error.log -/tests/**/pnpm-lock.yaml -# Built solidity contracts. -/tests/**/bin -/tests/**/truffle_output +# Node dependencies +node_modules/ +.pnpm-store/ # Docker volumes and debug logs .postgres -logfile \ No newline at end of file +logfile + +# Nix related files +.direnv +.envrc +.data + +# Local claude settings +.claude/settings.local.json +.claude/ralph-loop.local.md diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dcaabb18e56..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,87 +0,0 @@ -dist: bionic -language: rust -# This line would cache cargo-audit once installed, -# but instead will fail from the 10 minute timeout after -# printing the line "creating directory /home/travis/.cache/sccache" -#cache: cargo -rust: - - stable - - beta - -# Select pre-installed services -addons: - postgresql: "10" - apt: - packages: - - postgresql-10 - - postgresql-client-10 -services: - - postgresql - - docker - -before_install: - # Install Node.js 11.x - - nvm install 11 && nvm use 11 - # Install IPFS - - wget "https://dist.ipfs.io/go-ipfs/v0.10.0/go-ipfs_v0.10.0_linux-amd64.tar.gz" -O /tmp/ipfs.tar.gz - - pushd . && cd $HOME/bin && tar -xzvf /tmp/ipfs.tar.gz && popd - - export PATH="$HOME/bin/go-ipfs:$PATH" - - ipfs init - -matrix: - fast_finish: true - include: - # Some env var is always necessary to differentiate included builds - # Check coding style - - env: CHECK_FORMATTING=true - rust: stable - script: - - rustup component add rustfmt - - cargo fmt --all -- --check - - # Make sure release builds compile - - env: CHECK_RELEASE=true - rust: stable - script: - - cargo check --release - - # Check for warnings - - env: RUSTFLAGS="-D warnings" - rust: stable - script: - - cargo check --tests - - # Build tagged commits in release mode - - env: RELEASE=true - if: tag IS present - script: - - cargo build -p graph-node --release - - mv target/release/graph-node target/release/graph-node-$TRAVIS_OS_NAME - -env: - global: - - PGPORT=5432 - - THEGRAPH_STORE_POSTGRES_DIESEL_URL=postgresql://travis:travis@localhost:5432/graph_node_test - # Added because https://nodejs.org/dist/ had issues - - NVM_NODEJS_ORG_MIRROR=https://cnpmjs.org/mirrors/node/ - -# Test pipeline -before_script: - - psql -c "ALTER USER travis WITH PASSWORD 'travis';" - - psql -c 'create database graph_node_test;' -U travis - -script: - # Run tests - - ipfs daemon &> /dev/null & - - RUST_BACKTRACE=1 cargo test --verbose --all -- --nocapture - - killall ipfs - -deploy: - provider: releases - api_key: - secure: ygpZedRG+/Qg/lPhifyNQ+4rExjZ4nGyJjB4DYT1fuePMyKXfiCPGicaWRGR3ZnZGNRjdKaIkF97vBsZ0aHwW+AykwOxlXrkAFvCKA0Tb82vaYqCLrBs/Y5AEhuCWLFDz5cXDPMkptf+uLX/s3JCF0Mxo5EBN2JfBQ8vS6ScKEwqn2TiLLBQKTQ4658TFM4H5KiXktpyVVdlRvpoS3pRIPMqNU/QpGPQigaiKyYD5+azCrAXeaKT9bBS1njVbxI69Go4nraWZn7wIhZCrwJ+MxGNTOxwasypsWm/u1umhRVLM1rL2i7RRqkIvzwn22YMaU7FZKCx8huXcj0cB8NtHZSw7GhJDDDv3e7puZxl3m/c/7ks76UF95syLzoM/9FWEFew8Ti+5MApzKQj5YWHOCIEzBWPeqAcA8Y+Az7w2h1ZgNbjDgSvjGAFSpE8m+SM0A2TOOZ1g/t/yfbEl8CWO6Y8v2x1EONkp7X0CqJgASMp+h8kzKCbuYyRnghlToY+5wYuh4M9Qg9UeJCt9dOblRBVJwW5CFr62kgE/gso8F9tXXHkRTv3hfk5madZR1Vn5A7KadEO8epfV4IQNsd+VHfoxoJSprx5f77Q2bLMBD1GT/qMqECgSznoTkU5ajkKJRqUw4AwLTohrYir76j61eQfxOhXExY/EM8xvlxpd1w= - file: target/release/graph-node-$TRAVIS_OS_NAME - repo: graphprotocol/graph-node - on: - tags: true - skip_cleanup: true diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..b8ec983428b --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,288 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Graph Node is a Rust-based decentralized blockchain indexing protocol that enables efficient querying of blockchain data through GraphQL. It's the core component of The Graph protocol, written as a Cargo workspace with multiple crates organized by functionality. + +## Essential Development Commands + +### Testing Workflow + +⚠️ **Only run integration tests when explicitly requested or when changes require full system testing** + +Use unit tests for regular development and only run integration tests when: + +- Explicitly asked to do so +- Making changes to integration/end-to-end functionality +- Debugging issues that require full system testing +- Preparing releases or major changes + +### Unit Tests + +Unit tests are inlined with source code. + +**Prerequisites:** + +1. PostgreSQL running on localhost:5432 (with initialised `graph-test` database) +2. IPFS running on localhost:5001 +3. PNPM +4. Foundry (for smart contract compilation) +5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set to `postgresql://graph:graph@127.0.0.1:5432/graph-test` + +The environment dependencies and environment setup are operated by the human. + +**Running Unit Tests:** + +```bash +# Run unit tests +just test-unit + +# Run specific tests (e.g. `data_source::common::tests`) +just test-unit data_source::common::tests +``` + +**⚠️ Test Verification Requirements:** +When filtering for specific tests, ensure the intended test name(s) appear in the output. + +### Runner Tests (Integration Tests) + +**Prerequisites:** + +1. PostgreSQL running on localhost:5432 (with initialised `graph-test` database) +2. IPFS running on localhost:5001 +3. PNPM +4. Foundry (for smart contract compilation) +5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set to `postgresql://graph:graph@127.0.0.1:5432/graph-test` + +**Running Runner Tests:** + +```bash +# Run runner tests. +just test-runner + +# Run specific tests (e.g. `block_handlers`) +just test-runner block_handlers +``` + +**⚠️ Test Verification Requirements:** +When filtering for specific tests, ensure the intended test name(s) appear in the output. + +**Important Notes:** + +- Runner tests take moderate time (10-20 seconds) +- Tests automatically reset the database between runs +- Some tests can pass without IPFS, but tests involving file data sources require it + +### Integration Tests + +**Prerequisites:** + +1. PostgreSQL running on localhost:3011 (with initialised `graph-node` database) +2. IPFS running on localhost:3001 +3. Anvil running on localhost:3021 +4. PNPM +5. Foundry (for smart contract compilation) + +The environment dependencies and environment setup are operated by the human. + +**Running Integration Tests:** + +```bash +# Run all integration tests (automatically builds graph-node and gnd) +just test-integration + +# Run a specific integration test case (e.g., "grafted" test case) +TEST_CASE=grafted just test-integration + +# (Optional) Use graph-cli instead of gnd for compatibility testing +GRAPH_CLI=node_modules/.bin/graph just test-integration + +# Override ports if using different service ports (e.g., for local development) +POSTGRES_TEST_PORT=5432 ETHEREUM_TEST_PORT=8545 IPFS_TEST_PORT=5001 just test-integration +``` + +**⚠️ Test Verification Requirements:** + +- **ALWAYS verify tests actually ran** - Check the output for "test result: ok. X passed" where X > 0 +- **If output shows "0 passed" or "0 tests run"**, the TEST_CASE variable or filter was wrong - fix and re-run +- **Never trust exit code 0 alone** - Cargo can exit successfully even when no tests matched your filter + +**Important Notes:** + +- Integration tests take significant time (several minutes) +- Tests automatically reset the database between runs +- Logs are written to `tests/integration-tests/graph-node.log` +- **If a test hangs for >10 minutes**, it's likely stuck - kill with `pkill -9 integration_tests` and check logs +- CI uses the default ports (3011, 3021, 3001) - local development can override with environment variables + +### Code Quality + +```bash +# 🚨 MANDATORY: Format all code IMMEDIATELY after any .rs file edit +just format + +# 🚨 MANDATORY: Check code for warnings and errors - MUST have zero warnings +just lint + +# 🚨 MANDATORY: Check in release mode to catch linking/optimization issues that cargo check misses +just check --release +``` + +🚨 **CRITICAL REQUIREMENTS for ANY implementation**: + +- **🚨 MANDATORY**: `cargo fmt --all` MUST be run before any commit +- **🚨 MANDATORY**: `just lint` MUST show zero warnings before any commit +- **🚨 MANDATORY**: `cargo check --release` MUST complete successfully before any commit +- **🚨 MANDATORY**: The unit test suite MUST pass before any commit + +Forgetting any of these means you failed to follow instructions. Before any commit or PR, ALL of the above MUST be satisfied! No exceptions! + +## High-Level Architecture + +### Core Components + +- **`graph/`**: Core abstractions, traits, and shared types +- **`node/`**: Main executable and CLI (graphman) +- **`chain/`**: Blockchain-specific adapters (ethereum, near, substreams) +- **`runtime/`**: WebAssembly runtime for subgraph execution +- **`store/`**: PostgreSQL-based storage layer +- **`graphql/`**: GraphQL query execution engine +- **`server/`**: HTTP/WebSocket APIs + +### Data Flow + +``` +Blockchain → Chain Adapter → Block Stream → Trigger Processing → Runtime → Store → GraphQL API +``` + +1. **Chain Adapters** connect to blockchain nodes and convert data to standardized formats +2. **Block Streams** provide event-driven streaming of blockchain blocks +3. **Trigger Processing** matches blockchain events to subgraph handlers +4. **Runtime** executes subgraph code in WebAssembly sandbox +5. **Store** persists entities with block-level granularity +6. **GraphQL** processes queries and returns results + +### Key Abstractions + +- **`Blockchain`** trait: Core blockchain interface +- **`Store`** trait: Storage abstraction with read/write variants +- **`RuntimeHost`**: WASM execution environment +- **`TriggerData`**: Standardized blockchain events +- **`EventConsumer`/`EventProducer`**: Component communication + +### Architecture Patterns + +- **Event-driven**: Components communicate through async streams and channels +- **Trait-based**: Extensive use of traits for abstraction and modularity +- **Async/await**: Tokio-based async runtime throughout +- **Multi-shard**: Database sharding for scalability +- **Sandboxed execution**: WASM runtime with gas metering + +## Development Guidelines + +### Commit Convention + +Use format: `{crate-name}: {description}` + +- Single crate: `store: Support 'Or' filters` +- Multiple crates: `core, graphql: Add event source to store` +- All crates: `all: {description}` + +### Git Workflow + +- Rebase on master (don't merge master into feature branch) +- Keep commits logical and atomic +- Squash commits to clean up history before merging + +## Crate Structure + +### Core Crates + +- **`graph`**: Shared types, traits, and utilities +- **`node`**: Main binary and component wiring +- **`core`**: Business logic and subgraph management + +### Blockchain Integration + +- **`chain/ethereum`**: Ethereum chain support +- **`chain/near`**: NEAR protocol support +- **`chain/substreams`**: Substreams data source support + +### Infrastructure + +- **`store/postgres`**: PostgreSQL storage implementation +- **`runtime/wasm`**: WebAssembly runtime and host functions +- **`graphql`**: Query processing and execution +- **`server/`**: HTTP/WebSocket servers + +### Key Dependencies + +- **`diesel`**: PostgreSQL ORM +- **`tokio`**: Async runtime +- **`tonic`**: gRPC framework +- **`wasmtime`**: WebAssembly runtime +- **`web3`**: Ethereum interaction + +## Test Environment Requirements + +### Process Compose Setup (Recommended) + +The repository includes a process-compose-flake setup that provides native, declarative service management. + +Currently, the human is required to operate the service dependencies as illustrated below. + +**Unit Tests:** + +```bash +# Human: Start PostgreSQL + IPFS for unit tests in a separate terminal +# PostgreSQL: localhost:5432, IPFS: localhost:5001 +nix run .#unit + +# Claude: Run unit tests +just test-unit +``` + +**Runner Tests:** + +```bash +# Human: Start PostgreSQL + IPFS for runner tests in a separate terminal +# PostgreSQL: localhost:5432, IPFS: localhost:5001 +nix run .#unit # NOTE: Runner tests are using the same nix services stack as the unit test + +# Claude: Run runner tests +just test-runner +``` + +**Integration Tests:** + +```bash +# Human: Start all services for integration tests in a separate terminal +# PostgreSQL: localhost:3011, IPFS: localhost:3001, Anvil: localhost:3021 +nix run .#integration + +# Claude: Run integration tests (automatically builds graph-node and gnd) +just test-integration +``` + +**Services Configuration:** +The services are configured to use the test suite default ports for unit- and integration tests respectively. + +| Service | Unit Tests Port | Integration Tests Port | Database/Config | +| ---------------- | --------------- | ---------------------- | ----------------------------------------------- | +| PostgreSQL | 5432 | 3011 | `graph-test` / `graph-node` | +| IPFS | 5001 | 3001 | Data in `./.data/unit` or `./.data/integration` | +| Anvil (Ethereum) | - | 3021 | Deterministic test chain | + +**Service Configuration:** +The setup combines built-in services-flake services with custom multiService modules: + +**Built-in Services:** + +- **PostgreSQL**: Uses services-flake's postgres service with a helper function (`mkPostgresConfig`) that provides graph-specific defaults including required extensions. + +**Custom Services** (located in `./nix`): + +- `ipfs.nix`: IPFS (kubo) with automatic initialization and configurable ports +- `anvil.nix`: Ethereum test chain with deterministic configuration diff --git a/Cargo.lock b/Cargo.lock index cf3392a9fa7..5e204e6ad4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -14,5755 +14,10318 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.15.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" dependencies = [ - "gimli 0.24.0", + "gimli", ] [[package]] -name = "addr2line" -version = "0.16.0" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" -dependencies = [ - "gimli 0.25.0", -] +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] -name = "adler" -version = "1.0.2" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.1", + "once_cell", + "version_check", + "zerocopy", +] [[package]] name = "aho-corasick" -version = "0.7.18" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "alloc-no-stdlib" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] -name = "android_system_properties" -version = "0.1.5" +name = "alloc-stdlib" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" dependencies = [ - "libc", + "alloc-no-stdlib", ] [[package]] -name = "anyhow" -version = "1.0.75" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] -name = "arc-swap" -version = "1.3.0" +name = "alloy" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" +checksum = "5e1e915a830ea2ee123c9d3886bfec3302a7ddcd73a973ab165ed45aca2e42c3" +dependencies = [ + "alloy-consensus", + "alloy-contract", + "alloy-core", + "alloy-eips", + "alloy-genesis", + "alloy-json-rpc", + "alloy-network", + "alloy-provider", + "alloy-pubsub", + "alloy-rpc-client", + "alloy-rpc-types", + "alloy-serde", + "alloy-signer", + "alloy-signer-local", + "alloy-transport", + "alloy-transport-http", + "alloy-transport-ipc", + "alloy-transport-ws", + "alloy-trie", +] [[package]] -name = "arrayref" -version = "0.3.6" +name = "alloy-chains" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "ef3a72a2247c34a8545ee99e562b1b9b69168e5000567257ae51e91b4e6b1193" +dependencies = [ + "alloy-primitives", + "num_enum", + "strum 0.27.2", +] [[package]] -name = "arrayvec" -version = "0.5.2" +name = "alloy-consensus" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83447eeb17816e172f1dfc0db1f9dc0b7c5d069bd1f7cecbecceb382bf931015" +dependencies = [ + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "alloy-trie", + "alloy-tx-macros", + "arbitrary", + "auto_impl", + "borsh", + "c-kzg", + "derive_more", + "either", + "k256", + "once_cell", + "rand 0.8.6", + "secp256k1 0.30.0", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", +] + +[[package]] +name = "alloy-consensus-any" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +checksum = "5406343e306856dc2be762700e98a16904de45dee14a07f233e742ce68daff2f" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "arbitrary", + "serde", +] [[package]] -name = "arrayvec" -version = "0.7.4" +name = "alloy-contract" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "3ae8b60d71b92824e095b4003ff01fd2bc923017b7568997c5f459240e83499c" +dependencies = [ + "alloy-consensus", + "alloy-dyn-abi", + "alloy-json-abi", + "alloy-network", + "alloy-network-primitives", + "alloy-primitives", + "alloy-provider", + "alloy-pubsub", + "alloy-rpc-types-eth", + "alloy-sol-types", + "alloy-transport", + "futures 0.3.31", + "futures-util", + "serde_json", + "thiserror 2.0.18", + "tracing", +] [[package]] -name = "ascii" -version = "0.9.3" +name = "alloy-core" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" +checksum = "62ddde5968de6044d67af107ad835bc0069a7ca245870b94c5958a7d8712b184" +dependencies = [ + "alloy-dyn-abi", + "alloy-json-abi", + "alloy-primitives", + "alloy-rlp", + "alloy-sol-types", +] [[package]] -name = "assert-json-diff" -version = "2.0.2" +name = "alloy-dyn-abi" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ + "alloy-json-abi", + "alloy-primitives", + "alloy-sol-type-parser", + "alloy-sol-types", + "arbitrary", + "itoa", + "proptest", "serde", "serde_json", + "winnow 1.0.0", ] [[package]] -name = "async-recursion" -version = "1.0.5" +name = "alloy-eip2124" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "741bdd7499908b3aa0b159bba11e71c8cddd009a2c2eb7a06e825f1ec87900a5" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "crc", + "rand 0.8.6", + "serde", + "thiserror 2.0.18", ] [[package]] -name = "async-stream" -version = "0.3.5" +name = "alloy-eip2930" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "9441120fa82df73e8959ae0e4ab8ade03de2aaae61be313fbf5746277847ce25" dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "rand 0.8.6", + "serde", ] [[package]] -name = "async-stream-impl" -version = "0.3.5" +name = "alloy-eip7702" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "2919c5a56a1007492da313e7a3b6d45ef5edc5d33416fdec63c0d7a2702a0d20" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "k256", + "rand 0.8.6", + "serde", + "thiserror 2.0.18", ] [[package]] -name = "async-trait" -version = "0.1.51" +name = "alloy-eip7928" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +checksum = "d3231de68d5d6e75332b7489cfcc7f4dfabeba94d990a10e4b923af0e6623540" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "serde", ] [[package]] -name = "atomic_refcell" -version = "0.1.13" +name = "alloy-eips" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" +checksum = "0dca4c89ace90684b4b77366d00631ed498c9af962079af2a5dbc593a0618a77" +dependencies = [ + "alloy-eip2124", + "alloy-eip2930", + "alloy-eip7702", + "alloy-eip7928", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "arbitrary", + "auto_impl", + "borsh", + "c-kzg", + "derive_more", + "either", + "serde", + "serde_with", + "sha2 0.10.9", +] [[package]] -name = "atty" -version = "0.2.14" +name = "alloy-genesis" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "ab0e0fe9e6d1120ad7bb9254c3fc2b9bc80a8df42a033fb626be6559c13d5153" dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "alloy-eips", + "alloy-primitives", + "alloy-serde", + "alloy-trie", + "borsh", + "serde", + "serde_with", ] [[package]] -name = "autocfg" -version = "1.1.0" +name = "alloy-json-abi" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" +dependencies = [ + "alloy-primitives", + "alloy-sol-type-parser", + "serde", + "serde_json", +] [[package]] -name = "axum" -version = "0.6.1" +name = "alloy-json-rpc" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" +checksum = "ec0a82e56b1843bce483942d54fcadea92e676f1bde162e93c7d3b621fabc4e1" dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.1", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", + "alloy-primitives", + "alloy-sol-types", + "http 1.4.2", "serde", - "sync_wrapper", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-http", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.1", + "serde_json", + "thiserror 2.0.18", + "tracing", ] [[package]] -name = "axum-core" -version = "0.3.0" +name = "alloy-network" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" +checksum = "a7db7b095b0b1db1d18ce7e91dcd2e82007f2d52bfb8125e6b64633a74a06bc3" dependencies = [ + "alloy-consensus", + "alloy-consensus-any", + "alloy-eips", + "alloy-json-rpc", + "alloy-network-primitives", + "alloy-primitives", + "alloy-rpc-types-any", + "alloy-rpc-types-eth", + "alloy-serde", + "alloy-signer", + "alloy-sol-types", "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.1", + "auto_impl", + "derive_more", + "futures-utils-wasm", + "serde", + "serde_json", + "thiserror 2.0.18", ] [[package]] -name = "backtrace" -version = "0.3.61" +name = "alloy-network-primitives" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +checksum = "cd28d9bfd11729037d194f2b1d43db8642eb3f342032691f4ca96bb745479c3c" dependencies = [ - "addr2line 0.16.0", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide 0.4.4", - "object 0.26.0", - "rustc-demangle", + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-serde", + "serde", ] [[package]] -name = "base-x" -version = "0.2.11" +name = "alloy-primitives" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" +dependencies = [ + "alloy-rlp", + "arbitrary", + "bytes", + "cfg-if", + "const-hex", + "derive_more", + "foldhash 0.2.0", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "itoa", + "k256", + "keccak-asm", + "paste", + "proptest", + "proptest-derive 0.8.0", + "rand 0.9.3", + "rapidhash", + "ruint", + "rustc-hash", + "secp256k1 0.31.1", + "serde", + "sha3", +] [[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +name = "alloy-provider" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8955ab30418343de57b356de2ea60200f9fb8016a7ea3bc7f5c6176f01a8b1cf" +dependencies = [ + "alloy-chains", + "alloy-consensus", + "alloy-eips", + "alloy-json-rpc", + "alloy-network", + "alloy-network-primitives", + "alloy-primitives", + "alloy-pubsub", + "alloy-rpc-client", + "alloy-rpc-types-anvil", + "alloy-rpc-types-debug", + "alloy-rpc-types-eth", + "alloy-rpc-types-trace", + "alloy-rpc-types-txpool", + "alloy-signer", + "alloy-sol-types", + "alloy-transport", + "alloy-transport-http", + "alloy-transport-ipc", + "alloy-transport-ws", + "async-stream", + "async-trait", + "auto_impl", + "dashmap", + "either", + "futures 0.3.31", + "futures-utils-wasm", + "lru", + "parking_lot", + "pin-project", + "reqwest 0.13.2", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "wasmtimer", +] [[package]] -name = "base64" -version = "0.20.0" +name = "alloy-pubsub" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" +checksum = "7cd85cfea1fa8ebd20d3475e961fe3a3624c0eb4659ea137715c0c83c8aeaff0" +dependencies = [ + "alloy-json-rpc", + "alloy-primitives", + "alloy-transport", + "auto_impl", + "bimap", + "futures 0.3.31", + "parking_lot", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tracing", + "wasmtimer", +] [[package]] -name = "base64" -version = "0.21.0" +name = "alloy-rlp" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "5f70d83b765fdc080dbcd4f4db70d8d23fe4761f2f02ebfa9146b833900634b4" +dependencies = [ + "alloy-rlp-derive", + "arrayvec", + "bytes", +] [[package]] -name = "base64-url" -version = "2.0.2" +name = "alloy-rlp-derive" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb9fb9fb058cc3063b5fc88d9a21eefa2735871498a04e1650da76ed511c8569" +checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ - "base64 0.21.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "beef" -version = "0.5.2" +name = "alloy-rpc-client" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +checksum = "24f461f091dc8f657e73b5dea18fd63d5c7049720cd252f1eade4a7ebed6a7e1" dependencies = [ + "alloy-json-rpc", + "alloy-primitives", + "alloy-pubsub", + "alloy-transport", + "alloy-transport-http", + "alloy-transport-ipc", + "alloy-transport-ws", + "futures 0.3.31", + "pin-project", + "reqwest 0.13.2", "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tracing", + "url", + "wasmtimer", ] [[package]] -name = "bigdecimal" -version = "0.1.2" +name = "alloy-rpc-types" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1374191e2dd25f9ae02e3aa95041ed5d747fc77b3c102b49fe2dd9a8117a6244" +checksum = "052c031d1f7c5611997056bbcb8814e5cbf20f7efeee8c3de690555172038cf2" dependencies = [ - "num-bigint 0.2.6", - "num-integer", - "num-traits", + "alloy-primitives", + "alloy-rpc-types-anvil", + "alloy-rpc-types-debug", + "alloy-rpc-types-engine", + "alloy-rpc-types-eth", + "alloy-rpc-types-trace", + "alloy-rpc-types-txpool", + "alloy-serde", "serde", ] [[package]] -name = "bigdecimal" -version = "0.3.1" +name = "alloy-rpc-types-anvil" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +checksum = "2ff111a54268dc0bbd3b17f98571a7e27cc661dc081ad2999d91888647eb2e11" dependencies = [ - "num-bigint 0.4.4", - "num-integer", - "num-traits", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", ] [[package]] -name = "bincode" -version = "1.3.3" +name = "alloy-rpc-types-any" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "0a6561ed4759c974d9c144500a59e3fb8c1d87327a12900d5ce455c0cae6dcb6" dependencies = [ + "alloy-consensus-any", + "alloy-network-primitives", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", "serde", + "serde_json", ] [[package]] -name = "bitflags" -version = "1.3.1" +name = "alloy-rpc-types-debug" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1" +checksum = "48b9ad6eee93dd35a9ec0a6c1c6b180892a900ee17a6ed6500921552dd71e846" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "derive_more", + "serde", + "serde_with", +] [[package]] -name = "bitflags" -version = "2.4.0" +name = "alloy-rpc-types-engine" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "7eba59e1c069f168a01982f42a57797736923b76aa854194df4930be17867e1c" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "arbitrary", + "derive_more", + "rand 0.8.6", + "serde", + "strum 0.27.2", +] [[package]] -name = "bitvec" -version = "1.0.0" +name = "alloy-rpc-types-eth" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +checksum = "175a2a5b6017d7f61b5e4b800d21215fe8e94fe729d00828e13bb6d93dcf3492" dependencies = [ - "funty", - "radium", - "tap", - "wyz", + "alloy-consensus", + "alloy-consensus-any", + "alloy-eips", + "alloy-network-primitives", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "alloy-sol-types", + "arbitrary", + "itertools 0.13.0", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", ] [[package]] -name = "blake3" -version = "0.3.8" +name = "alloy-rpc-types-trace" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" +checksum = "514b4b1ce3354f65067b4fc7eb75358e0f2ec8be3340c96dea65d6894f9ca435" dependencies = [ - "arrayref", - "arrayvec 0.5.2", - "cc", - "cfg-if 0.1.10", - "constant_time_eq 0.1.5", - "crypto-mac 0.8.0", - "digest 0.9.0", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", + "serde_json", + "thiserror 2.0.18", ] [[package]] -name = "blake3" -version = "1.5.0" +name = "alloy-rpc-types-txpool" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "76e34a42ebb4a71ab0bfdebc6d2f3c7bf809f01edf154d08fed159d10d1ef1d4" dependencies = [ - "arrayref", - "arrayvec 0.7.4", - "cc", - "cfg-if 1.0.0", - "constant_time_eq 0.3.0", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", ] [[package]] -name = "block-buffer" -version = "0.9.0" +name = "alloy-serde" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "cc21a8772af7d78bba286726aa245bd2ff81cd9abe230afea2e91578996831c9" dependencies = [ - "generic-array", + "alloy-primitives", + "arbitrary", + "serde", + "serde_json", ] [[package]] -name = "block-buffer" -version = "0.10.2" +name = "alloy-signer" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "8ffbce94c50dd9d4d1f83e044c5595bbd3ada981bd3057ce28b3a5470e77385d" dependencies = [ - "generic-array", + "alloy-primitives", + "async-trait", + "auto_impl", + "either", + "elliptic-curve", + "k256", + "thiserror 2.0.18", ] [[package]] -name = "bs58" -version = "0.4.0" +name = "alloy-signer-local" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +checksum = "e48366d2c42b8d95ef951101bafa28486590f21b7a1e68b6b2d069746557bbe3" +dependencies = [ + "alloy-consensus", + "alloy-network", + "alloy-primitives", + "alloy-signer", + "async-trait", + "k256", + "rand 0.8.6", + "thiserror 2.0.18", +] [[package]] -name = "bstr" -version = "0.2.16" +name = "alloy-sol-macro" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" dependencies = [ - "memchr", + "alloy-sol-macro-expander", + "alloy-sol-macro-input", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "bumpalo" -version = "3.12.0" +name = "alloy-sol-macro-expander" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" +dependencies = [ + "alloy-json-abi", + "alloy-sol-macro-input", + "const-hex", + "heck 0.5.0", + "indexmap 2.14.0", + "proc-macro-error2", + "proc-macro2", + "quote", + "sha3", + "syn 2.0.118", + "syn-solidity", +] [[package]] -name = "byte-slice-cast" -version = "1.2.0" +name = "alloy-sol-macro-input" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c751592b77c499e7bce34d99d67c2c11bdc0574e9a488ddade14150a4698" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" +dependencies = [ + "alloy-json-abi", + "const-hex", + "dunce", + "heck 0.5.0", + "macro-string", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.118", + "syn-solidity", +] [[package]] -name = "byteorder" -version = "1.4.3" +name = "alloy-sol-type-parser" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" +dependencies = [ + "serde", + "winnow 1.0.0", +] [[package]] -name = "bytes" -version = "1.2.1" +name = "alloy-sol-types" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" +dependencies = [ + "alloy-json-abi", + "alloy-primitives", + "alloy-sol-macro", + "serde", +] [[package]] -name = "cc" -version = "1.0.69" +name = "alloy-transport" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" +checksum = "86052fdcec72d37ca4aa4b66254601e7453c45a6e1c70aa4561033d002fb80cc" dependencies = [ - "jobserver", + "alloy-json-rpc", + "auto_impl", + "base64", + "derive_more", + "futures 0.3.31", + "futures-utils-wasm", + "parking_lot", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tower 0.5.2", + "tracing", + "url", + "wasmtimer", ] [[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "cfg-if" -version = "1.0.0" +name = "alloy-transport-http" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "b273587487921274f4f5d0ef2c7ef36944dcbb75a4e2318e69eae822bd263f91" +dependencies = [ + "alloy-json-rpc", + "alloy-transport", + "itertools 0.13.0", + "reqwest 0.13.2", + "serde_json", + "tower 0.5.2", + "tracing", + "url", +] [[package]] -name = "chrono" -version = "0.4.31" +name = "alloy-transport-ipc" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "bfb89df168b24773ef603af14f2449c05a7d3f27d05d3eceaea6bf96cccae168" dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-targets 0.48.0", + "alloy-json-rpc", + "alloy-pubsub", + "alloy-transport", + "bytes", + "futures 0.3.31", + "interprocess", + "pin-project", + "serde", + "serde_json", + "tokio", + "tokio-util", + "tracing", ] [[package]] -name = "cid" -version = "0.11.0" +name = "alloy-transport-ws" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "472ac98592f38dfd48f188d5713a328422ed22fa39eb52b8bca495370134762a" +checksum = "33e32e0b47d3b3bf5770b7c132090c614b008d307c5e1544f1925f5b7e9e9af6" dependencies = [ - "core2", - "multibase", - "multihash 0.19.1", - "unsigned-varint 0.8.0", + "alloy-pubsub", + "alloy-transport", + "futures 0.3.31", + "http 1.4.2", + "rustls", + "serde_json", + "tokio", + "tokio-tungstenite 0.28.0", + "tracing", + "url", + "ws_stream_wasm", ] [[package]] -name = "clap" -version = "3.2.25" +name = "alloy-trie" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +checksum = "428aa0f0e0658ff091f8f667c406e034b431cb10abd39de4f507520968acc499" dependencies = [ - "atty", - "bitflags 1.3.1", - "clap_derive", - "clap_lex", - "indexmap 1.9.3", - "once_cell", - "strsim", - "termcolor", - "textwrap", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "arrayvec", + "derive_arbitrary", + "derive_more", + "nybbles", + "proptest", + "proptest-derive 0.5.1", + "serde", + "smallvec", + "tracing", ] [[package]] -name = "clap_derive" -version = "3.2.25" +name = "alloy-tx-macros" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +checksum = "01a0035943b75fe1e249f52e688492d7a1b1826bc2d19b8e1d5d3c24a2ad8f50" dependencies = [ - "heck", - "proc-macro-error", + "darling 0.23.0", "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.118", ] [[package]] -name = "clap_lex" -version = "0.2.2" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ - "os_str_bytes", + "libc", ] [[package]] -name = "combine" -version = "3.8.1" +name = "anstream" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ - "ascii", - "byteorder", - "either", - "memchr", - "unreachable", + "anstyle", + "anstyle-parse 0.2.4", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", ] [[package]] -name = "common-multipart-rfc7578" -version = "0.6.0" +name = "anstream" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baee326bc603965b0f26583e1ecd7c111c41b49bd92a344897476a352798869" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ - "bytes", - "futures-core", - "futures-util", - "http", - "mime", - "mime_guess", - "rand", - "thiserror", + "anstyle", + "anstyle-parse 1.0.0", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", ] [[package]] -name = "console" -version = "0.13.0" +name = "anstyle" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50aab2529019abfabfa93f1e6c41ef392f91fbf179b347a7e96abb524884a08" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "regex", - "terminal_size", - "unicode-width", - "winapi", - "winapi-util", -] +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "anstyle-parse" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] [[package]] -name = "constant_time_eq" -version = "0.3.0" +name = "anstyle-parse" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] [[package]] -name = "convert_case" -version = "0.4.0" +name = "anstyle-query" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys 0.52.0", +] [[package]] -name = "core-foundation" -version = "0.9.1" +name = "anstyle-wincon" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ - "core-foundation-sys", - "libc", + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", ] [[package]] -name = "core-foundation-sys" -version = "0.8.3" +name = "anyhow" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] -name = "core2" -version = "0.4.0" +name = "arbitrary" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" dependencies = [ - "memchr", + "derive_arbitrary", ] [[package]] -name = "cpp_demangle" -version = "0.3.3" +name = "arc-swap" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea47428dc9d2237f3c6bc134472edfd63ebba0af932e783506dcfd66f10d18a" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ - "cfg-if 1.0.0", + "rustversion", ] [[package]] -name = "cpufeatures" -version = "0.1.5" +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + +[[package]] +name = "ark-ff" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +checksum = "6b3235cc41ee7a12aaaf2c575a2ad7b46713a8a50bda2fc3b003a04845c05dd6" dependencies = [ - "libc", + "ark-ff-asm 0.3.0", + "ark-ff-macros 0.3.0", + "ark-serialize 0.3.0", + "ark-std 0.3.0", + "derivative", + "num-bigint 0.4.6", + "num-traits", + "paste", + "rustc_version 0.3.3", + "zeroize", ] [[package]] -name = "cpufeatures" -version = "0.2.2" +name = "ark-ff" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" dependencies = [ - "libc", + "ark-ff-asm 0.4.2", + "ark-ff-macros 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "digest 0.10.7", + "itertools 0.10.5", + "num-bigint 0.4.6", + "num-traits", + "paste", + "rustc_version 0.4.0", + "zeroize", ] [[package]] -name = "cranelift-bforest" -version = "0.74.0" +name = "ark-ff" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ca3560686e7c9c7ed7e0fe77469f2410ba5d7781b1acaa9adc8d8deea28e3e" +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" dependencies = [ - "cranelift-entity", + "ark-ff-asm 0.5.0", + "ark-ff-macros 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "educe", + "itertools 0.13.0", + "num-bigint 0.4.6", + "num-traits", + "paste", + "zeroize", ] [[package]] -name = "cranelift-codegen" -version = "0.74.0" +name = "ark-ff-asm" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf9bf1ffffb6ce3d2e5ebc83549bd2436426c99b31cc550d521364cbe35d276" +checksum = "db02d390bf6643fb404d3d22d31aee1c4bc4459600aef9113833d17e786c6e44" dependencies = [ - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-entity", - "gimli 0.24.0", - "log", - "regalloc", - "serde", - "smallvec", - "target-lexicon", + "quote", + "syn 1.0.109", ] [[package]] -name = "cranelift-codegen-meta" -version = "0.74.0" +name = "ark-ff-asm" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc21936a5a6d07e23849ffe83e5c1f6f50305c074f4b2970ca50c13bf55b821" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" dependencies = [ - "cranelift-codegen-shared", - "cranelift-entity", + "quote", + "syn 1.0.109", ] [[package]] -name = "cranelift-codegen-shared" -version = "0.74.0" +name = "ark-ff-asm" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5b6ffaa87560bebe69a5446449da18090b126037920b0c1c6d5945f72faf6b" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ - "serde", + "quote", + "syn 2.0.118", ] [[package]] -name = "cranelift-entity" -version = "0.74.0" +name = "ark-ff-macros" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6b4a8bef04f82e4296782646f733c641d09497df2fabf791323fefaa44c64c" +checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" dependencies = [ - "serde", + "num-bigint 0.4.6", + "num-traits", + "quote", + "syn 1.0.109", ] [[package]] -name = "cranelift-frontend" -version = "0.74.0" +name = "ark-ff-macros" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b783b351f966fce33e3c03498cb116d16d97a8f9978164a60920bd0d3a99c" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ - "cranelift-codegen", - "log", - "smallvec", - "target-lexicon", + "num-bigint 0.4.6", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] -name = "cranelift-native" -version = "0.74.0" +name = "ark-ff-macros" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c88d3dd48021ff1e37e978a00098524abd3513444ae252c08d37b310b3d2a" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" dependencies = [ - "cranelift-codegen", - "target-lexicon", + "num-bigint 0.4.6", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "cranelift-wasm" -version = "0.74.0" +name = "ark-serialize" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb6d408e2da77cdbbd65466298d44c86ae71c1785d2ab0d8657753cdb4d9d89" +checksum = "1d6c2b318ee6e10f8c2853e73a83adc0ccb88995aa978d8a3408d492ab2ee671" dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "itertools 0.10.5", - "log", - "serde", - "smallvec", - "thiserror", - "wasmparser", + "ark-std 0.3.0", + "digest 0.9.0", ] [[package]] -name = "crc32fast" -version = "1.2.1" +name = "ark-serialize" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" dependencies = [ - "cfg-if 1.0.0", + "ark-std 0.4.0", + "digest 0.10.7", + "num-bigint 0.4.6", ] [[package]] -name = "crossbeam" -version = "0.8.2" +name = "ark-serialize" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "num-bigint 0.4.6", ] [[package]] -name = "crossbeam-channel" -version = "0.5.7" +name = "ark-std" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "num-traits", + "rand 0.8.6", ] [[package]] -name = "crossbeam-deque" -version = "0.8.1" +name = "ark-std" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils", + "num-traits", + "rand 0.8.6", ] [[package]] -name = "crossbeam-epoch" -version = "0.9.5" +name = "ark-std" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", - "lazy_static", - "memoffset", - "scopeguard", + "num-traits", + "rand 0.8.6", ] [[package]] -name = "crossbeam-queue" -version = "0.3.2" +name = "arrayvec" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "serde", ] [[package]] -name = "crossbeam-utils" -version = "0.8.8" +name = "arrow" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7" dependencies = [ - "cfg-if 1.0.0", - "lazy_static", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] -name = "crunchy" -version = "0.2.2" +name = "arrow-arith" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] [[package]] -name = "crypto-common" -version = "0.1.3" +name = "arrow-array" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137" dependencies = [ - "generic-array", - "typenum", + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.0", + "num-complex", + "num-integer", + "num-traits", ] [[package]] -name = "crypto-mac" -version = "0.8.0" +name = "arrow-buffer" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a" dependencies = [ - "generic-array", - "subtle", + "bytes", + "half", + "num-bigint 0.4.6", + "num-traits", ] [[package]] -name = "crypto-mac" -version = "0.10.1" +name = "arrow-cast" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9" dependencies = [ - "generic-array", - "subtle", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", ] [[package]] -name = "data-encoding" -version = "2.3.2" +name = "arrow-csv" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] [[package]] -name = "data-encoding-macro" -version = "0.1.12" +name = "arrow-data" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" +checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab" dependencies = [ - "data-encoding", - "data-encoding-macro-internal", + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", ] [[package]] -name = "data-encoding-macro-internal" -version = "0.1.10" +name = "arrow-flight" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" +checksum = "68365401e834743d708094927e2ca727a32d639fe900df04b936e07a36701b74" dependencies = [ - "data-encoding", - "syn 1.0.107", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64", + "bytes", + "futures 0.3.31", + "once_cell", + "paste", + "prost", + "prost-types", + "tonic", + "tonic-prost", ] [[package]] -name = "defer" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "647605a6345d5e89c3950a36a638c56478af9b414c55c6f2477c73b115f9acde" - -[[package]] -name = "derive_more" -version = "0.99.17" +name = "arrow-ipc" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5" dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.107", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", ] [[package]] -name = "diesel" -version = "1.4.8" +name = "arrow-json" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" +checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed" dependencies = [ - "bigdecimal 0.1.2", - "bitflags 1.3.1", - "byteorder", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", "chrono", - "diesel_derives", - "num-bigint 0.2.6", - "num-integer", + "half", + "indexmap 2.14.0", + "itoa", + "lexical-core", + "memchr", "num-traits", - "pq-sys", - "r2d2", + "ryu", + "serde_core", "serde_json", + "simdutf8", ] [[package]] -name = "diesel-derive-enum" -version = "1.1.2" +name = "arrow-ord" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8910921b014e2af16298f006de12aa08af894b71f0f49a486ab6d74b17bbed" +checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983" dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 1.0.107", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] -name = "diesel-dynamic-schema" -version = "1.0.0" -source = "git+https://github.com/diesel-rs/diesel-dynamic-schema?rev=a8ec4fb1#a8ec4fb11de6242488ba3698d74406f4b5073dc4" +name = "arrow-row" +version = "59.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879" dependencies = [ - "diesel", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", ] [[package]] -name = "diesel_derives" -version = "1.4.1" +name = "arrow-schema" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", -] +checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1" [[package]] -name = "diesel_migrations" -version = "1.4.0" +name = "arrow-select" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624" dependencies = [ - "migrations_internals", - "migrations_macros", + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", ] [[package]] -name = "diff" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" - -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - -[[package]] -name = "digest" -version = "0.9.0" +name = "arrow-string" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d" dependencies = [ - "generic-array", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", ] [[package]] -name = "digest" -version = "0.10.7" +name = "ascii_utils" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.2", - "crypto-common", -] +checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" [[package]] -name = "directories-next" -version = "2.0.0" +name = "assert-json-diff" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", + "serde", + "serde_json", ] [[package]] -name = "dirs" -version = "4.0.0" +name = "async-compression" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" dependencies = [ - "dirs-sys", + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", ] [[package]] -name = "dirs-next" -version = "2.0.0" +name = "async-graphql" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +checksum = "1057a9f7ccf2404d94571dec3451ade1cb524790df6f1ada0d19c2a49f6b0f40" dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", + "async-graphql-derive", + "async-graphql-parser", + "async-graphql-value", + "async-io", + "async-trait", + "asynk-strim", + "base64", + "bytes", + "chrono", + "fast_chemail", + "fnv", + "futures-util", + "handlebars", + "http 1.4.2", + "indexmap 2.14.0", + "mime", + "multer", + "num-traits", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "static_assertions_next", + "tempfile", + "thiserror 2.0.18", ] [[package]] -name = "dirs-sys" -version = "0.3.7" +name = "async-graphql-axum" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "a1e37c5532e4b686acf45e7162bc93da91fc2c702fb0d465efc2c20c8f973795" dependencies = [ - "libc", - "redox_users", - "winapi", + "async-graphql", + "axum", + "bytes", + "futures-util", + "serde_json", + "tokio", + "tokio-stream", + "tokio-util", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "dirs-sys-next" -version = "0.1.2" +name = "async-graphql-derive" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +checksum = "2e6cbeadc8515e66450fba0985ce722192e28443697799988265d86304d7cc68" dependencies = [ - "libc", - "redox_users", - "winapi", + "Inflector", + "async-graphql-parser", + "darling 0.23.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "strum 0.27.2", + "syn 2.0.118", + "thiserror 2.0.18", ] [[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.28" +name = "async-graphql-parser" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "e64ef70f77a1c689111e52076da1cd18f91834bcb847de0a9171f83624b07fbf" dependencies = [ - "cfg-if 1.0.0", + "async-graphql-value", + "pest", + "serde", + "serde_json", ] [[package]] -name = "env_logger" -version = "0.7.1" +name = "async-graphql-value" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +checksum = "3e3ef112905abea9dea592fc868a6873b10ebd3f983e83308f995d6284e9ba41" dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", + "bytes", + "indexmap 2.14.0", + "serde", + "serde_json", ] [[package]] -name = "env_logger" -version = "0.10.1" +name = "async-io" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" dependencies = [ - "humantime 2.1.0", - "is-terminal", - "log", - "regex", - "termcolor", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.4", + "slab", + "windows-sys 0.61.2", ] [[package]] -name = "envconfig" -version = "0.10.0" +name = "async-lock" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "envconfig_derive", + "event-listener", + "event-listener-strategy", + "pin-project-lite", ] [[package]] -name = "envconfig_derive" -version = "0.10.0" +name = "async-recursion" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.118", ] [[package]] -name = "equivalent" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" - -[[package]] -name = "errno" -version = "0.2.7" +name = "async-stream" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ - "errno-dragonfly", - "libc", - "winapi", + "async-stream-impl", + "futures-core", + "pin-project-lite", ] [[package]] -name = "errno" -version = "0.3.0" +name = "async-stream-impl" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.45.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "errno-dragonfly" -version = "0.1.1" +name = "async-trait" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ - "gcc", - "libc", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "ethabi" -version = "17.2.0" +name = "async_io_stream" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ - "ethereum-types", - "hex", - "once_cell", - "regex", - "serde", - "serde_json", - "sha3", - "thiserror", - "uint 0.9.1", + "futures 0.3.31", + "pharos", + "rustc_version 0.4.0", ] [[package]] -name = "ethbloom" -version = "0.12.1" +name = "asynk-strim" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +checksum = "52697735bdaac441a29391a9e97102c74c6ef0f9b60a40cf109b1b404e29d2f6" dependencies = [ - "crunchy", - "fixed-hash", - "impl-rlp", - "impl-serde", - "tiny-keccak 2.0.2", + "futures-core", + "pin-project-lite", ] [[package]] -name = "ethereum-types" -version = "0.13.1" +name = "atoi" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" dependencies = [ - "ethbloom", - "fixed-hash", - "impl-rlp", - "impl-serde", - "primitive-types", - "uint 0.9.1", + "num-traits", ] [[package]] -name = "fallible-iterator" -version = "0.2.0" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "fallible-iterator" -version = "0.3.0" +name = "atomic_refcell" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" [[package]] -name = "file-per-thread-logger" -version = "0.1.4" +name = "auto_impl" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ - "env_logger 0.7.1", - "log", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "firestorm" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499" - -[[package]] -name = "firestorm" -version = "0.5.0" +name = "autocfg" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3d6188b8804df28032815ea256b6955c9625c24da7525f387a7af02fbb8f01" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] -name = "fixed-hash" -version = "0.7.0" +name = "aws-lc-rs" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ - "byteorder", - "rand", - "rustc-hex", - "static_assertions", + "aws-lc-sys", + "zeroize", ] [[package]] -name = "fixedbitset" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" - -[[package]] -name = "flate2" -version = "1.0.25" +name = "aws-lc-sys" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ - "crc32fast", - "miniz_oxide 0.6.2", + "cc", + "cmake", + "dunce", + "fs_extra", ] [[package]] -name = "fnv" -version = "1.0.7" +name = "axum" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "foreign-types-shared", + "axum-core", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1 0.10.6", + "sync_wrapper", + "tokio", + "tokio-tungstenite 0.29.0", + "tower 0.5.2", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", ] [[package]] -name = "foreign-types-shared" -version = "0.1.1" +name = "axum-core" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", +] [[package]] -name = "form_urlencoded" -version = "1.2.1" +name = "backon" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "percent-encoding", + "fastrand", ] [[package]] -name = "funty" -version = "2.0.0" +name = "base-x" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" [[package]] -name = "futures" -version = "0.1.31" +name = "base16ct" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] -name = "futures" -version = "0.3.16" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adc00f486adfc9ce99f77d717836f0c5aa84965eb0b4f051f4e83f7cab53f8b" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "futures-channel" -version = "0.3.26" +name = "base64ct" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d809780667f4410e7c41b07f52439b94d2bdf8528eeedc287fa38d3b7f95d82" + +[[package]] +name = "bigdecimal" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "1374191e2dd25f9ae02e3aa95041ed5d747fc77b3c102b49fe2dd9a8117a6244" dependencies = [ - "futures-core", - "futures-sink", + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "serde", ] [[package]] -name = "futures-core" -version = "0.3.26" +name = "bimap" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" [[package]] -name = "futures-executor" -version = "0.3.16" +name = "bit-set" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d0d535a57b87e1ae31437b892713aee90cd2d7b0ee48727cd11fc72ef54761c" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "futures-core", - "futures-task", - "futures-util", + "bit-vec", ] [[package]] -name = "futures-io" -version = "0.3.26" +name = "bit-vec" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] -name = "futures-macro" -version = "0.3.26" +name = "bitcoin-io" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" + +[[package]] +name = "bitcoin_hashes" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "bitcoin-io", + "hex-conservative", ] [[package]] -name = "futures-sink" -version = "0.3.26" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "futures-task" -version = "0.3.26" +name = "bitflags" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] -name = "futures-timer" -version = "3.0.2" +name = "bitvec" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] [[package]] -name = "futures-util" -version = "0.3.26" +name = "blake3" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "futures 0.1.31", - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] -name = "gcc" -version = "0.3.55" +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] [[package]] -name = "generic-array" -version = "0.14.4" +name = "block-buffer" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ - "typenum", - "version_check", + "hybrid-array", ] [[package]] -name = "getrandom" -version = "0.2.10" +name = "blst" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "dcdb4c7013139a150f9fc55d123186dbfaba0d912817466282c73ac49e71fb45" dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi", + "cc", + "glob", + "threadpool", + "zeroize", ] [[package]] -name = "gimli" -version = "0.24.0" +name = "borsh" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" dependencies = [ - "fallible-iterator 0.2.0", - "indexmap 1.9.3", - "stable_deref_trait", + "borsh-derive", + "cfg_aliases", ] [[package]] -name = "gimli" -version = "0.25.0" +name = "borsh-derive" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" +checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "git-testament" -version = "0.2.5" +name = "brotli" +version = "8.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710c78d2b68e46e62f5ba63ba0a7a2986640f37f9ecc07903b9ad4e7b2dbfc8e" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" dependencies = [ - "git-testament-derive", + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", ] [[package]] -name = "git-testament-derive" -version = "0.2.0" +name = "brotli-decompressor" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b31494efbbe1a6730f6943759c21b92c8dc431cb4df177e6f2a6429c3c96842" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" dependencies = [ - "log", - "proc-macro2", - "quote", - "syn 2.0.48", - "time", + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] -name = "globset" -version = "0.4.8" +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bs58" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ - "aho-corasick", - "bstr", - "fnv", - "log", - "regex", + "tinyvec", ] [[package]] -name = "graph" -version = "0.34.0" +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ - "Inflector", - "anyhow", - "async-stream", - "async-trait", - "atomic_refcell", - "bigdecimal 0.1.2", - "bytes", - "chrono", - "cid", - "clap", - "defer", - "diesel", - "diesel_derives", - "envconfig", - "ethabi", - "futures 0.1.31", - "futures 0.3.16", - "graphql-parser", - "hex", - "hex-literal 0.4.1", - "http", - "humantime 2.1.0", - "isatty", - "itertools 0.12.0", - "lazy_static", - "maplit", - "num-bigint 0.2.6", - "num-traits", - "num_cpus", - "parking_lot 0.12.1", - "petgraph", - "priority-queue", - "prometheus", - "prost", - "prost-types", - "rand", - "regex", - "reqwest", - "semver", + "memchr", "serde", - "serde_derive", - "serde_json", - "serde_plain", - "serde_regex", - "serde_yaml", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "stable-hash 0.3.4", - "stable-hash 0.4.4", - "strum", - "strum_macros", - "thiserror", - "tiny-keccak 1.5.0", - "tokio", - "tokio-retry", - "tokio-stream", - "toml 0.8.8", - "tonic", - "tonic-build", - "url", - "wasmparser", - "web3", ] [[package]] -name = "graph-chain-arweave" -version = "0.34.0" +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" dependencies = [ - "base64-url", - "diesel", - "graph", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost", - "prost-types", - "serde", - "sha2 0.10.8", - "tonic-build", + "allocator-api2", ] [[package]] -name = "graph-chain-common" -version = "0.34.0" +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ - "anyhow", - "heck", - "protobuf 3.3.0", - "protobuf-parse", + "serde", ] [[package]] -name = "graph-chain-cosmos" -version = "0.34.0" +name = "c-kzg" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e00bf4b112b07b505472dbefd19e37e53307e2bfed5a79e0cc161d58ccd0e687" dependencies = [ - "anyhow", - "graph", - "graph-chain-common", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost", - "prost-types", - "semver", + "arbitrary", + "blst", + "cc", + "glob", + "hex", + "libc", + "once_cell", "serde", - "tonic-build", ] [[package]] -name = "graph-chain-ethereum" -version = "0.34.0" +name = "cc" +version = "1.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ - "anyhow", - "base64 0.20.0", - "dirs-next", + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link 0.2.1", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "unsigned-varint", +] + +[[package]] +name = "clap" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +dependencies = [ + "anstream 0.6.21", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_complete" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b4be9c4c4b1f30b78d8a750e0822b6a6102d97e62061c583a6c1dea2dfb33ae" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + +[[package]] +name = "const-hex" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.12", + "proptest", + "serde_core", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpp_demangle" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96e58d342ad113c2b878f16d5d034c03be492ae460cdbc02b7f0f2284d310c7d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "cranelift-assembler-x64" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06aeba2c965fc446d13c56a6ccb2631b78445d7544543dd9a25289977630914" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2d2dde4ec1352715595b5cfa6fe2e5b8ebb9da3457b3ee8db0aa2808c069aa" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b4982ef9fa54ec9eee841e891e7ddc5434be1250e88de31572e000c888f30b" +dependencies = [ + "cranelift-entity", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-bitset" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529143118c4eeb58c39ecb02319557d512be6c61348486422974ab8e3906b8a8" +dependencies = [ + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7780677247ad3577e3a6a3ebf43f39b325a11d6393db72b2c9968a910d4d13d" +dependencies = [ + "bumpalo", + "cranelift-assembler-x64", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.17.0", + "libm", + "log", + "postcard", + "pulley-interpreter", + "regalloc2", + "rustc-hash", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9645250416cbf92454fe61160e17e026e0ce405906a54500b114f923ddffc9" +dependencies = [ + "cranelift-assembler-x64-meta", + "cranelift-codegen-shared", + "cranelift-srcgen", + "heck 0.5.0", + "pulley-interpreter", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ee8d222ff0fd3681791979afbf88586ac9f49010d3db96b3cbe4c96759aee3" + +[[package]] +name = "cranelift-control" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "591abe6f5312bd2c4220f1b3bead56c2ad00257c52668015ba013b85dcf2a17a" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5300c49cf940526fe771517b3b3eabd5d0ff164ee61698579cf403fe8d3af3c" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-frontend" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4adbf760207fdbbe130f1191cce01cdef66831a9f648b1f39ff2800d126d45" +dependencies = [ + "cranelift-codegen", + "hashbrown 0.17.0", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8315b21ff018226a42a60a4702c2dd75f6447cac26e9bca622e14c22088c2ff5" + +[[package]] +name = "cranelift-native" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d506ef23a60715bde451b06620b14402166ded3b648454fccbf04f3e46a4aa70" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-srcgen" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ed47e602652e3410f9387fc0db70fefadcee4d78a78881421aabcab4e26b89" + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.11.1", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix 1.1.4", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core 0.20.10", + "darling_macro 0.20.10", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "serde", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core 0.20.10", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + +[[package]] +name = "data-encoding-macro" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" +dependencies = [ + "data-encoding", + "syn 1.0.109", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime 0.1.4", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime 0.3.1", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + +[[package]] +name = "defer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930c7171c8df9fb1782bdf9b918ed9ed2d33d1d22300abb754f9085bc48bf8e8" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.118", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn 2.0.118", + "unicode-xid", +] + +[[package]] +name = "diesel" +version = "2.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9940fb8467a0a06312218ed384185cb8536aa10d8ec017d0ce7fad2c1bd882d5" +dependencies = [ + "bigdecimal", + "bitflags 2.11.1", + "byteorder", + "chrono", + "diesel_derives", + "downcast-rs", + "itoa", + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "pq-sys", + "r2d2", + "serde_json", +] + +[[package]] +name = "diesel-async" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd39af30158d444884f166fe4c58f35dc40ad71ad017bb59408a3448526ff4bd" +dependencies = [ + "deadpool 0.13.0", + "diesel", + "futures-core", + "futures-util", + "pin-project-lite", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "diesel-derive-enum" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "diesel-dynamic-schema" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a2287b125235908614c5f32f9b3bdc43c4d639846853d66e8a68c75a02756" +dependencies = [ + "diesel", +] + +[[package]] +name = "diesel_derives" +version = "2.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "diesel_migrations" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d0f4a98124ba6d4ca75da535f65984badec16a003b6e2f94a01e31a79490b8" +dependencies = [ + "diesel", + "migrations_internals", + "migrations_macros", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe2444076b48641147115697648dc743c2c00b61adade0f01ce67133c7babe8c" +dependencies = [ + "syn 2.0.118", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", +] + +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "doctest-file" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "dsl_auto_type" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd122633e4bef06db27737f21d3738fb89c8f6d5360d6d9d7635dda142a7757e" +dependencies = [ + "darling 0.21.3", + "either", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "serdect", + "signature", + "spki", +] + +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "serdect", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-ordinalize" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream 1.0.0", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "envconfig" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2934d78aeb06e54961db5b96cfca2b01f5cb01cf11c34f7b088932fd48233ac6" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452c458dfb890a2fea64e4c894f83daad61689fb9bbe84c382e1ce6d7536710b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "erased-serde" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +dependencies = [ + "serde", +] + +[[package]] +name = "errno" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fast_chemail" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495a39d30d624c2caabe6312bfead73e7717692b44e0b32df168c275a2e8e9e4" +dependencies = [ + "ascii_utils", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fastrlp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + +[[package]] +name = "fastrlp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8dba4714ef14b8274c371879b175aa55b16b30f269663f19d576f380018dc4" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "firestorm" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499" + +[[package]] +name = "firestorm" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" + +[[package]] +name = "fixed-hash" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" +dependencies = [ + "byteorder", + "rand 0.8.6", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.11.1", + "rustc_version 0.4.0", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures 0.1.31", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "futures-utils-wasm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" + +[[package]] +name = "fuzzy-matcher" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94" +dependencies = [ + "thread_local", +] + +[[package]] +name = "fxprof-processed-profile" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" +dependencies = [ + "bitflags 2.11.1", + "debugid", + "rustc-hash", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", + "windows-targets 0.52.6", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gimli" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap 2.14.0", + "stable_deref_trait", +] + +[[package]] +name = "git-testament" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a74999c921479f919c87a9d2e6922a79a18683f18105344df8e067149232e51" +dependencies = [ + "git-testament-derive", +] + +[[package]] +name = "git-testament-derive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbeac967e71eb3dc1656742fc7521ec7cd3b6b88738face65bf1fddf702bc4c0" +dependencies = [ + "log", + "proc-macro2", + "quote", + "syn 2.0.118", + "time", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "gnd" +version = "0.45.0" +dependencies = [ + "Inflector", + "anyhow", + "async-trait", + "clap", + "clap_complete", + "console 0.16.4", + "env_logger", + "git-testament", + "globset", + "graph", + "graph-chain-ethereum", + "graph-core", + "graph-graphql", + "graph-node", + "graph-store-postgres", + "graphql-tools", + "hex", + "indicatif", + "inquire", + "lazy_static", + "notify", + "open", + "pgtemp", + "pq-sys", + "regex", + "reqwest 0.12.23", + "semver 1.0.28", + "serde", + "serde_json", + "serde_yaml", + "sha1 0.11.0", + "similar", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tower 0.5.2", + "url", + "walkdir", + "wasmparser 0.118.2", +] + +[[package]] +name = "graph" +version = "0.45.0" +dependencies = [ + "Inflector", + "ahash", + "alloy", + "anyhow", + "arrow", + "arrow-flight", + "async-stream", + "async-trait", + "atomic_refcell", + "base64", + "bigdecimal", + "bs58 0.5.1", + "bytes", + "chrono", + "cid", + "clap", + "csv", + "defer", + "derive_more", + "diesel", + "diesel_derives", + "envconfig", + "futures 0.1.31", + "futures 0.3.31", + "graph_derive", + "graphql-tools", + "half", + "hex", + "hex-literal", + "http 0.2.12", + "http 1.4.2", + "http-body-util", + "hyper", + "hyper-util", + "indoc", + "itertools 0.15.0", + "lazy-regex", + "lazy_static", + "lru_time_cache", + "maplit", + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "object_store", + "parking_lot", + "petgraph 0.8.3", + "portable-atomic", + "priority-queue", + "prometheus", + "prost", + "prost-types", + "rand 0.9.3", + "redis", + "regex", + "reqwest 0.12.23", + "rustls", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "serde_plain", + "serde_regex", + "serde_yaml", + "sha2 0.11.0", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "sqlparser 0.57.0", + "sqlparser 0.62.0", + "stable-hash 0.3.4", + "stable-hash 0.4.4", + "strum_macros 0.28.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tokio-stream", + "tokio-util", + "toml 1.1.2+spec-1.1.0", + "tonic", + "tonic-prost", + "tonic-prost-build", + "url", + "wasmparser 0.118.2", + "wiremock", +] + +[[package]] +name = "graph-chain-common" +version = "0.45.0" +dependencies = [ + "anyhow", + "heck 0.5.0", + "protobuf", + "protobuf-parse", +] + +[[package]] +name = "graph-chain-ethereum" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "base64", "envconfig", - "futures 0.1.31", + "futures 0.3.31", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "hex", + "itertools 0.15.0", + "jsonrpc-core", + "prost", + "prost-types", + "semver 1.0.28", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic-prost-build", + "tower 0.5.2", +] + +[[package]] +name = "graph-chain-near" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "diesel", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "prost", + "prost-types", + "serde", + "tokio", + "tonic-prost-build", +] + +[[package]] +name = "graph-core" +version = "0.45.0" +dependencies = [ + "alloy", + "anyhow", + "arrow", + "async-trait", + "bytes", + "chrono", + "futures 0.3.31", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-runtime-wasm", + "indoc", + "itertools 0.15.0", + "parking_lot", + "prometheus", + "serde_yaml", + "slog", + "strum 0.28.0", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tokio-util", + "tower 0.5.3", + "tower-test", + "wiremock", +] + +[[package]] +name = "graph-graphql" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-recursion", + "async-trait", + "crossbeam", + "graph", + "graphql-tools", + "lazy_static", + "parking_lot", + "stable-hash 0.3.4", + "stable-hash 0.4.4", +] + +[[package]] +name = "graph-node" +version = "0.45.0" +dependencies = [ + "anyhow", + "clap", + "console 0.16.4", + "diesel", + "diesel-async", + "env_logger", + "git-testament", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-core", + "graph-graphql", + "graph-server-http", + "graph-server-index-node", + "graph-server-json-rpc", + "graph-server-metrics", + "graph-store-postgres", + "graphman", + "graphman-server", + "indicatif", + "itertools 0.15.0", + "json-structural-diff", + "lazy_static", + "prometheus", + "serde", + "shellexpand", + "termcolor", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "graph-runtime-derive" +version = "0.45.0" +dependencies = [ + "quote", + "syn 2.0.118", +] + +[[package]] +name = "graph-runtime-test" +version = "0.45.0" +dependencies = [ + "async-trait", + "graph", + "graph-chain-ethereum", + "graph-runtime-wasm", + "rand 0.9.3", + "semver 1.0.28", + "test-store", + "wasmtime", +] + +[[package]] +name = "graph-runtime-wasm" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "bs58 0.4.0", + "graph", + "graph-runtime-derive", + "hex", + "never", + "parity-wasm", + "semver 1.0.28", + "serde_yaml", + "wasm-instrument", + "wasmtime", +] + +[[package]] +name = "graph-server-http" +version = "0.45.0" +dependencies = [ + "async-trait", + "graph", + "graph-core", + "graph-graphql", + "serde", +] + +[[package]] +name = "graph-server-index-node" +version = "0.45.0" +dependencies = [ + "async-trait", + "blake3", + "git-testament", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-graphql", +] + +[[package]] +name = "graph-server-json-rpc" +version = "0.45.0" +dependencies = [ + "axum", + "graph", + "serde", + "serde_json", + "slog", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "graph-server-metrics" +version = "0.45.0" +dependencies = [ + "graph", +] + +[[package]] +name = "graph-store-postgres" +version = "0.45.0" +dependencies = [ + "Inflector", + "anyhow", + "arrow", + "async-trait", + "blake3", + "chrono", + "clap", + "deadpool 0.13.0", + "derive_more", + "diesel", + "diesel-async", + "diesel-derive-enum", + "diesel-dynamic-schema", + "diesel_derives", + "diesel_migrations", + "fallible-iterator 0.3.0", + "git-testament", + "graph", + "graphman-store", + "hex", + "itertools 0.15.0", + "lazy_static", + "lru_time_cache", + "openssl", + "parquet", + "postgres", + "postgres-openssl", + "pretty_assertions", + "rand 0.9.3", + "serde", + "serde_json", + "serde_yaml", + "sqlparser 0.62.0", + "stable-hash 0.3.4", + "thiserror 2.0.18", + "tokio", + "tokio-stream", +] + +[[package]] +name = "graph-tests" +version = "0.45.0" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-stream", + "async-trait", + "graph", + "graph-chain-ethereum", + "graph-core", + "graph-graphql", + "graph-node", + "graph-runtime-wasm", + "graph-server-index-node", + "graph-store-postgres", + "serde", + "serde_yaml", + "slog", + "slog-async", + "slog-term", + "tokio", + "tokio-stream", + "tokio-util", +] + +[[package]] +name = "graph_derive" +version = "0.45.0" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "graphman" +version = "0.45.0" +dependencies = [ + "anyhow", + "diesel", + "diesel-async", "graph", - "graph-runtime-derive", - "graph-runtime-wasm", - "hex", - "http", - "itertools 0.12.0", - "jsonrpc-core", + "graph-store-postgres", + "graphman-store", + "itertools 0.15.0", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "graphman-server" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-graphql", + "async-graphql-axum", + "axum", + "chrono", + "diesel", + "diesel-async", + "graph", + "graph-store-postgres", + "graphman", + "graphman-store", "lazy_static", - "prost", - "prost-types", - "semver", + "reqwest 0.12.23", + "serde", + "serde_json", + "slog", + "test-store", + "thiserror 2.0.18", + "tokio", + "tower-http 0.7.0", +] + +[[package]] +name = "graphman-store" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "diesel", + "strum 0.28.0", +] + +[[package]] +name = "graphql-tools" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1ae57fa544e67a661c97805be3e36b97e7d15d65d67bb34beac24d940f987e" +dependencies = [ + "combine", + "itoa", + "lazy_static", + "ryu", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "handlebars" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3f9296c208515b87bd915a2f5d1163d4b3f863ba83337d7713cf478055948e" +dependencies = [ + "derive_builder", + "log", + "num-order", + "pest", + "pest_derive", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "byteorder", + "num-traits", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-conservative" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "hex-literal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.4.2", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hybrid-array" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a79f2aff40c18ab8615ddc5caa9eb5b96314aef18fe5823090f204ad988e813" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http 1.4.2", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.4.2", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.4.2", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ibig" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcc7f316b2c079dde77564a1360639c1a956a23fa96122732e416cb10717bb" +dependencies = [ + "cfg-if", + "num-traits", + "rand 0.8.6", + "static_assertions", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "arbitrary", + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "indicatif" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f007684f2e9727160da8b960ec161264703bfd1af084fd2e34d040c9a0dd4" +dependencies = [ + "console 0.16.4", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.11.1", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + +[[package]] +name = "inquire" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6654738b8024300cf062d04a1c13c10c8e2cea598ec1c47dc9b6641159429756" +dependencies = [ + "bitflags 2.11.1", + "crossterm", + "dyn-clone", + "fuzzy-matcher", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "interprocess" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d" +dependencies = [ + "doctest-file", + "futures-core", + "libc", + "recvmsg", + "tokio", + "widestring", + "windows-sys 0.52.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "ittapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b996fe614c41395cdaedf3cf408a9534851090959d90d54a535f675550b64b1" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5385394064fa2c886205dba02598013ce83d3e92d33dbdc0c52fe0e7bf4fc" +dependencies = [ + "cc", +] + +[[package]] +name = "jiff" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.61", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "json-structural-diff" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e878e36a8a44c158505c2c818abdc1350413ad83dcb774a0459f6a7ef2b65cbf" +dependencies = [ + "console 0.15.11", + "difflib", + "regex", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures 0.3.31", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "serdect", + "sha2 0.10.9", +] + +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + +[[package]] +name = "keccak-asm" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1766b89733097006f3a1388a02849865d6bc98c89273cb622e29fdd209922183" +dependencies = [ + "digest 0.10.7", + "sha3-asm", +] + +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "lazy-regex" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496" +dependencies = [ + "lazy-regex-proc_macros", + "once_cell", + "regex", +] + +[[package]] +name = "lazy-regex-proc_macros" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.118", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.11.1", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lru_time_cache" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" + +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + +[[package]] +name = "macro-string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memfd" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix 1.1.4", +] + +[[package]] +name = "migrations_internals" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c791ecdf977c99f45f23280405d7723727470f6689a5e6dbf513ac547ae10d" +dependencies = [ + "serde", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "migrations_macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fc5ac76be324cfd2d3f2cf0fdf5d5d3c4f14ed8aaebadb09e304ba42282703" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.4.2", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "unsigned-varint", +] + +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.0", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.11.1", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-modular" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f" + +[[package]] +name = "num-order" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537b596b97c40fcf8056d153049eb22f481c17ebce72a513ec9286e4986d1bb6" +dependencies = [ + "num-modular", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "nybbles" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5676b5c379cf5b03da1df2b3061c4a4e2aa691086a56ac923e08c143f53f59" +dependencies = [ + "alloy-rlp", + "arbitrary", + "cfg-if", + "proptest", + "ruint", "serde", - "tiny-keccak 1.5.0", - "tonic-build", - "uuid", + "smallvec", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "crc32fast", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "memchr", +] + +[[package]] +name = "object_store" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body-util", + "humantime", + "hyper", + "itertools 0.15.0", + "nix", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest 0.13.2", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "open" +version = "5.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d3b65c44123a56e0133d2cd06ce4361bd3ca99d41198b2f25e3c3db9b8b4a" +dependencies = [ + "is-wsl", + "libc", +] + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.5.0+3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parity-scale-codec" +version = "3.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "parity-wasm" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.2", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "parquet" +version = "59.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970dff83e97d953c827ae8176f6bf4e9f77bf62daacc01ec5df348ec5eacd913" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.17.0", + "lz4_flex", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "paste", + "seq-macro", + "simdutf8", + "snap", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +dependencies = [ + "memchr", + "thiserror 1.0.61", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pest_meta" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.9", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.2", + "indexmap 2.14.0", + "serde", +] + +[[package]] +name = "pgtemp" +version = "0.6.0" +source = "git+https://github.com/graphprotocol/pgtemp?branch=initdb-args#08a95d441d74ce0a50b6e0a55dbf96d8362d8fb7" +dependencies = [ + "libc", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures 0.3.31", + "rustc_version 0.4.0", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", ] [[package]] -name = "graph-chain-near" -version = "0.34.0" +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ - "anyhow", - "base64 0.20.0", - "diesel", - "graph", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost", - "prost-types", - "serde", - "tonic-build", - "trigger-filters", + "pin-project-internal", ] [[package]] -name = "graph-chain-starknet" -version = "0.34.0" +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ - "graph", - "graph-runtime-derive", - "graph-runtime-wasm", - "hex", - "prost", - "prost-types", - "serde", - "sha3", - "tonic-build", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "graph-chain-substreams" -version = "0.34.0" +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "anyhow", - "async-stream", - "base64 0.20.0", - "dirs-next", - "envconfig", - "futures 0.1.31", - "graph", - "graph-runtime-wasm", - "hex", - "http", - "itertools 0.12.0", - "jsonrpc-core", - "lazy_static", - "prost", - "prost-types", - "semver", + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.5.2", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", "serde", - "tiny-keccak 1.5.0", +] + +[[package]] +name = "postgres" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c48ece1c6cda0db61b058c1721378da76855140e9214339fa1317decacb176" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "futures-util", + "log", "tokio", - "tonic-build", + "tokio-postgres", ] [[package]] -name = "graph-core" -version = "0.34.0" +name = "postgres-openssl" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4" dependencies = [ - "anyhow", - "async-stream", - "async-trait", - "atomic_refcell", + "openssl", + "tokio", + "tokio-openssl", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" +dependencies = [ + "base64", + "byteorder", "bytes", - "cid", - "futures 0.1.31", - "futures 0.3.16", - "graph", - "graph-chain-arweave", - "graph-chain-cosmos", - "graph-chain-ethereum", - "graph-chain-near", - "graph-chain-starknet", - "graph-chain-substreams", - "graph-runtime-wasm", - "ipfs-api", - "ipfs-api-backend-hyper", - "lazy_static", - "lru_time_cache", - "semver", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand 0.9.3", + "sha2 0.10.9", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4605b7c057056dd35baeb6ac0c0338e4975b1f2bef0f65da953285eb007095" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "pq-src" +version = "0.3.10+libpq-18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ef39ce621f4993d6084fdcd4cbf1e01c84bdba53109cfad095d2cf441b85b9" +dependencies = [ + "cc", + "openssl-sys", +] + +[[package]] +name = "pq-sys" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "574ddd6a267294433f140b02a726b0640c43cf7c6f717084684aaa3b285aba61" +dependencies = [ + "libc", + "pkg-config", + "pq-src", + "vcpkg", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.118", +] + +[[package]] +name = "primitive-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" +dependencies = [ + "fixed-hash", + "impl-codec", + "uint 0.9.5", +] + +[[package]] +name = "priority-queue" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93980406f12d9f8140ed5abe7155acb10bb1e69ea55c88960b9c2f117445ef96" +dependencies = [ + "equivalent", + "indexmap 2.14.0", "serde", - "serde_json", - "serde_yaml", - "tower 0.4.13 (git+https://github.com/tower-rs/tower.git)", - "tower-test", - "uuid", ] [[package]] -name = "graph-graphql" -version = "0.34.0" +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "Inflector", - "anyhow", - "async-recursion", - "crossbeam", - "graph", - "graphql-parser", - "graphql-tools", - "indexmap 2.1.0", - "lazy_static", - "parking_lot 0.12.1", - "stable-hash 0.3.4", - "stable-hash 0.4.4", + "toml_edit", ] [[package]] -name = "graph-node" -version = "0.34.0" +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ - "clap", - "diesel", - "env_logger 0.10.1", - "futures 0.3.16", - "git-testament", - "graph", - "graph-chain-arweave", - "graph-chain-cosmos", - "graph-chain-ethereum", - "graph-chain-near", - "graph-chain-starknet", - "graph-chain-substreams", - "graph-core", - "graph-graphql", - "graph-runtime-wasm", - "graph-server-http", - "graph-server-index-node", - "graph-server-json-rpc", - "graph-server-metrics", - "graph-server-websocket", - "graph-store-postgres", - "graphql-parser", - "http", - "json-structural-diff", + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", "lazy_static", - "prometheus", - "serde", - "shellexpand", - "termcolor", - "url", + "libc", + "memchr", + "parking_lot", + "protobuf", + "reqwest 0.12.23", + "thiserror 2.0.18", ] [[package]] -name = "graph-runtime-derive" -version = "0.34.0" +name = "proptest" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.11.1", + "num-traits", + "rand 0.9.3", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "proptest-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ - "heck", "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.118", ] [[package]] -name = "graph-runtime-test" -version = "0.34.0" +name = "proptest-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" dependencies = [ - "graph", - "graph-chain-ethereum", - "graph-core", - "graph-runtime-derive", - "graph-runtime-wasm", - "rand", - "semver", - "test-store", - "wasmtime", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "graph-runtime-wasm" -version = "0.34.0" +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ - "anyhow", - "async-trait", - "atomic_refcell", - "bs58", "bytes", - "defer", - "ethabi", - "futures 0.1.31", - "graph", - "graph-runtime-derive", - "hex", - "lazy_static", - "never", - "parity-wasm", - "semver", - "strum", - "strum_macros", - "uuid", - "wasm-instrument", - "wasmtime", + "prost-derive", ] [[package]] -name = "graph-server-http" -version = "0.34.0" +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "futures 0.1.31", - "graph", - "graph-core", - "graph-graphql", - "graphql-parser", - "http", - "hyper", - "serde", + "heck 0.5.0", + "itertools 0.13.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.118", + "tempfile", ] [[package]] -name = "graph-server-index-node" -version = "0.34.0" +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ - "blake3 1.5.0", - "either", - "futures 0.3.16", - "git-testament", - "graph", - "graph-chain-arweave", - "graph-chain-cosmos", - "graph-chain-ethereum", - "graph-chain-near", - "graph-chain-starknet", - "graph-chain-substreams", - "graph-graphql", - "graphql-parser", - "http", - "hyper", - "lazy_static", - "serde", + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "graph-server-json-rpc" -version = "0.34.0" +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "graph", - "jsonrpsee", - "serde", + "prost", ] [[package]] -name = "graph-server-metrics" -version = "0.34.0" +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" dependencies = [ - "graph", - "hyper", + "once_cell", + "protobuf-support", + "thiserror 1.0.61", ] [[package]] -name = "graph-server-websocket" -version = "0.34.0" +name = "protobuf-parse" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ "anyhow", - "futures 0.1.31", - "graph", - "graphql-parser", - "http", - "lazy_static", - "serde", - "serde_derive", - "tokio-tungstenite", - "uuid", + "indexmap 2.14.0", + "log", + "protobuf", + "protobuf-support", + "tempfile", + "thiserror 1.0.61", + "which", ] [[package]] -name = "graph-store-postgres" -version = "0.34.0" +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" dependencies = [ - "Inflector", - "anyhow", - "async-trait", - "blake3 1.5.0", - "clap", - "derive_more", - "diesel", - "diesel-derive-enum", - "diesel-dynamic-schema", - "diesel_derives", - "diesel_migrations", - "fallible-iterator 0.3.0", - "git-testament", - "graph", - "graphql-parser", - "hex", - "itertools 0.12.0", - "lazy_static", - "lru_time_cache", - "maybe-owned", - "openssl", - "pin-utils", - "postgres", - "postgres-openssl", - "pretty_assertions", - "rand", - "serde", - "stable-hash 0.3.4", - "uuid", + "thiserror 1.0.61", ] [[package]] -name = "graph-tests" -version = "0.34.0" +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" dependencies = [ - "anyhow", - "assert-json-diff", - "async-stream", - "futures 0.3.16", - "graph", - "graph-chain-ethereum", - "graph-chain-substreams", - "graph-core", - "graph-graphql", - "graph-node", - "graph-runtime-wasm", - "graph-server-index-node", - "graph-store-postgres", - "graphql-parser", - "hyper", - "secp256k1", - "serde", - "serde_yaml", - "slog", - "termcolor", - "tokio", - "tokio-stream", + "cc", ] [[package]] -name = "graphql-parser" -version = "0.4.0" +name = "pulldown-cmark" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "combine", - "thiserror", + "bitflags 2.11.1", + "memchr", + "unicase", ] [[package]] -name = "graphql-tools" -version = "0.2.1" +name = "pulldown-cmark-to-cmark" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3a979aca9d796ff03ff71f4013e203a1f69bf1f37899ae4a8e676bb236608" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" dependencies = [ - "graphql-parser", - "lazy_static", - "serde", - "serde_json", + "pulldown-cmark", ] [[package]] -name = "h2" -version = "0.3.17" +name = "pulley-interpreter" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "38b92604caae1a1899b6a5b54967289dd538177c626004c91accf9d0ec7e4a12" dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 1.9.3", - "slab", - "tokio", - "tokio-util 0.7.1", - "tracing", + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", ] [[package]] -name = "hashbrown" -version = "0.12.1" +name = "pulley-macros" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" +checksum = "5a7ac85c0bb3fb351f10d531230aaa5e366b46d7c4e5328e5f02801d6dac1165" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "hashbrown" -version = "0.14.1" +name = "quick-error" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] -name = "hdrhistogram" -version = "7.5.2" +name = "quick-xml" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ - "byteorder", - "num-traits", + "memchr", + "serde", ] [[package]] -name = "headers" -version = "0.3.5" +name = "quinn" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ - "base64 0.13.1", - "bitflags 1.3.1", "bytes", - "headers-core", - "http", - "httpdate", - "mime", - "sha-1 0.9.7", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", ] [[package]] -name = "headers-core" -version = "0.2.0" +name = "quinn-proto" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ - "http", + "aws-lc-rs", + "bytes", + "getrandom 0.3.1", + "lru-slab", + "rand 0.9.3", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", ] [[package]] -name = "heck" -version = "0.4.1" +name = "quinn-udp" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "quote" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ - "libc", + "proc-macro2", ] [[package]] -name = "hermit-abi" -version = "0.3.1" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "hex" -version = "0.4.3" +name = "r2d2" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ - "serde", + "log", + "parking_lot", + "scheduled-thread-pool", ] [[package]] -name = "hex-literal" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" - -[[package]] -name = "hex-literal" -version = "0.4.1" +name = "radium" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] -name = "hmac" -version = "0.10.1" +name = "rand" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ - "crypto-mac 0.10.1", - "digest 0.9.0", + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", + "serde", ] [[package]] -name = "http" -version = "0.2.8" +name = "rand" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ - "bytes", - "fnv", - "itoa", + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "serde", ] [[package]] -name = "http-body" -version = "0.4.5" +name = "rand" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "bytes", - "http", - "pin-project-lite", + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", ] [[package]] -name = "http-range-header" -version = "0.3.0" +name = "rand_chacha" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] [[package]] -name = "httparse" -version = "1.7.1" +name = "rand_chacha" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] [[package]] -name = "httpdate" -version = "1.0.1" +name = "rand_core" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] [[package]] -name = "humantime" -version = "1.3.0" +name = "rand_core" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "quick-error", + "getrandom 0.3.1", + "serde", ] [[package]] -name = "humantime" -version = "2.1.0" +name = "rand_core" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" [[package]] -name = "hyper" -version = "0.14.18" +name = "rand_xorshift" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.9", - "tokio", - "tower-service 0.3.1", - "tracing", - "want", + "rand_core 0.9.3", ] [[package]] -name = "hyper-multipart-rfc7578" -version = "0.8.0" +name = "rapidhash" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb2cf73e96e9925f4bed948e763aa2901c2f1a3a5f713ee41917433ced6671" +checksum = "5d8b5b858a440a0bc02625b62dd95131b9201aa9f69f411195dd4a7cfb1de3d7" dependencies = [ - "bytes", - "common-multipart-rfc7578", - "futures-core", - "http", - "hyper", + "rustversion", ] [[package]] -name = "hyper-rustls" -version = "0.23.2" +name = "rayon" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ - "http", - "hyper", - "log", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", + "either", + "rayon-core", ] [[package]] -name = "hyper-timeout" -version = "0.4.1" +name = "rayon-core" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", + "crossbeam-deque", + "crossbeam-utils", ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "recursive" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", + "recursive-proc-macro-impl", + "stacker", ] [[package]] -name = "iana-time-zone" -version = "0.1.47" +name = "recursive-proc-macro-impl" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "js-sys", - "once_cell", - "wasm-bindgen", - "winapi", + "quote", + "syn 2.0.118", ] [[package]] -name = "ibig" -version = "0.3.5" +name = "recvmsg" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7d04a53d0ca4a37b47741ff98dd517a0e1e6a0ec22a72c748e50197052d595b" -dependencies = [ - "cfg-if 1.0.0", - "num-traits", - "rand", - "static_assertions", -] +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" [[package]] -name = "idna" -version = "0.2.3" +name = "redis" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543" dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", + "arc-swap", + "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", + "combine", + "futures-channel", + "futures-util", + "itoa", + "num-bigint 0.4.6", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-util", + "url", + "xxhash-rust", ] [[package]] -name = "idna" -version = "0.5.0" +name = "redox_syscall" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "bitflags 1.3.2", ] [[package]] -name = "impl-codec" -version = "0.6.0" +name = "redox_syscall" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "parity-scale-codec", + "bitflags 2.11.1", ] [[package]] -name = "impl-rlp" -version = "0.3.0" +name = "redox_users" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "rlp", + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.61", ] [[package]] -name = "impl-serde" -version = "0.3.2" +name = "ref-cast" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ - "serde", + "ref-cast-impl", ] [[package]] -name = "impl-trait-for-tuples" -version = "0.2.1" +name = "ref-cast-impl" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.118", ] [[package]] -name = "indexmap" -version = "1.9.3" +name = "regalloc2" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186" dependencies = [ - "autocfg", - "hashbrown 0.12.1", + "allocator-api2", + "bumpalo", + "hashbrown 0.17.0", + "log", + "rustc-hash", "serde", + "smallvec", ] [[package]] -name = "indexmap" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" -dependencies = [ - "equivalent", - "hashbrown 0.14.1", -] - -[[package]] -name = "instant" -version = "0.1.10" +name = "regex" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ - "cfg-if 1.0.0", + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", ] [[package]] -name = "io-lifetimes" -version = "1.0.9" +name = "regex-automata" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.45.0", + "aho-corasick", + "memchr", + "regex-syntax", ] [[package]] -name = "ipfs-api" -version = "0.17.0" +name = "regex-syntax" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d8cc57cf12ae4af611e53dd04053e1cfb815917c51c410aa30399bf377046ab" -dependencies = [ - "ipfs-api-backend-hyper", -] +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] -name = "ipfs-api-backend-hyper" -version = "0.6.0" +name = "reqwest" +version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9d131b408b4caafe1e7c00d410a09ad3eb7e3ab68690cf668e86904b2176b4" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ - "async-trait", - "base64 0.13.1", + "async-compression", + "base64", "bytes", - "futures 0.3.16", - "http", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", "hyper", - "hyper-multipart-rfc7578", "hyper-rustls", - "ipfs-api-prelude", - "thiserror", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower 0.5.2", + "tower-http 0.6.11", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.0", + "web-sys", ] [[package]] -name = "ipfs-api-prelude" -version = "0.6.0" +name = "reqwest" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b74065805db266ba2c6edbd670b23c4714824a955628472b2e46cc9f3a869cb" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ - "async-trait", + "base64", "bytes", - "cfg-if 1.0.0", - "common-multipart-rfc7578", - "dirs", - "futures 0.3.16", - "http", - "multiaddr", - "multibase", + "futures-core", + "futures-util", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", - "serde_urlencoded", - "thiserror", + "sync_wrapper", "tokio", - "tokio-util 0.7.1", - "tracing", - "typed-builder", - "walkdir", + "tokio-rustls", + "tokio-util", + "tower 0.5.2", + "tower-http 0.6.11", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", ] [[package]] -name = "ipnet" -version = "2.3.1" +name = "rfc6979" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] [[package]] -name = "is-terminal" -version = "0.4.6" +name = "ring" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.45.0", + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "untrusted", + "windows-sys 0.52.0", ] [[package]] -name = "isatty" -version = "0.1.9" +name = "rlp" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31a8281fc93ec9693494da65fbf28c0c2aa60a2eaec25dc58e2f31952e95edc" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ - "cfg-if 0.1.10", - "libc", - "redox_syscall 0.1.57", - "winapi", + "bytes", + "rustc-hex", ] [[package]] -name = "itertools" -version = "0.10.5" +name = "ruint" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" dependencies = [ - "either", + "alloy-rlp", + "arbitrary", + "ark-ff 0.3.0", + "ark-ff 0.4.2", + "ark-ff 0.5.0", + "bytes", + "fastrlp 0.3.1", + "fastrlp 0.4.0", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "parity-scale-codec", + "primitive-types", + "proptest", + "rand 0.8.6", + "rand 0.9.3", + "rlp", + "ruint-macro", + "serde_core", + "valuable", + "zeroize", ] [[package]] -name = "itertools" -version = "0.12.0" +name = "ruint-macro" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" -dependencies = [ - "either", -] +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] -name = "itoa" -version = "1.0.1" +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] -name = "jobserver" -version = "0.1.23" +name = "rustc-hex" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5ca711fd837261e14ec9e674f092cbb931d3fa1482b017ae59328ddc6f3212b" -dependencies = [ - "libc", -] +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] -name = "js-sys" -version = "0.3.59" +name = "rustc_version" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" dependencies = [ - "wasm-bindgen", + "semver 0.11.0", ] [[package]] -name = "json-structural-diff" -version = "0.1.0" +name = "rustc_version" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25c7940d3c84d2079306c176c7b2b37622b6bc5e43fbd1541b1e4a4e1fd02045" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "console", - "difflib", - "regex", - "serde_json", + "semver 1.0.28", ] [[package]] -name = "jsonrpc-core" -version = "18.0.0" +name = "rustix" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "futures 0.3.16", - "futures-executor", - "futures-util", - "log", - "serde", - "serde_derive", - "serde_json", + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.4.14", + "windows-sys 0.52.0", ] [[package]] -name = "jsonrpsee" -version = "0.15.1" +name = "rustix" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd0d559d5e679b1ab2f869b486a11182923863b1b3ee8b421763cdd707b783a" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "jsonrpsee-core", - "jsonrpsee-http-server", - "jsonrpsee-types", + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.52.0", ] [[package]] -name = "jsonrpsee-core" -version = "0.15.1" +name = "rustls" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3dc3e9cf2ba50b7b1d7d76a667619f82846caa39e8e8daa8a4962d74acaddca" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ - "anyhow", - "arrayvec 0.7.4", - "async-trait", - "beef", - "futures-channel", - "futures-util", - "globset", - "http", - "hyper", - "jsonrpsee-types", - "lazy_static", - "parking_lot 0.12.1", - "rand", - "rustc-hash", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", - "unicase", + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", ] [[package]] -name = "jsonrpsee-http-server" -version = "0.15.1" +name = "rustls-native-certs" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03802f0373a38c2420c70b5144742d800b509e2937edc4afb116434f07120117" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ - "futures-channel", - "futures-util", - "hyper", - "jsonrpsee-core", - "jsonrpsee-types", - "serde", - "serde_json", - "tokio", - "tracing", - "tracing-futures", + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.7.0", ] [[package]] -name = "jsonrpsee-types" -version = "0.15.1" +name = "rustls-pki-types" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e290bba767401b646812f608c099b922d8142603c9e73a50fb192d3ac86f4a0d" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ - "anyhow", - "beef", - "serde", - "serde_json", - "thiserror", - "tracing", + "web-time", + "zeroize", ] [[package]] -name = "keccak" -version = "0.1.4" +name = "rustls-platform-verifier" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ - "cpufeatures 0.2.2", + "core-foundation 0.10.0", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework 3.7.0", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", ] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "rustls-platform-verifier-android" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] -name = "leb128" -version = "0.2.4" +name = "rustls-webpki" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] [[package]] -name = "libc" -version = "0.2.150" +name = "rustversion" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] -name = "linux-raw-sys" +name = "rusty-fork" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" - -[[package]] -name = "lock_api" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", + "fnv", + "quick-error", + "tempfile", + "wait-timeout", ] [[package]] -name = "lru_time_cache" -version = "0.11.11" +name = "ryu" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] -name = "mach" -version = "0.3.2" +name = "same-file" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ - "libc", + "winapi-util", ] [[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - -[[package]] -name = "matches" -version = "0.1.8" +name = "schannel" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] [[package]] -name = "matchit" -version = "0.7.0" +name = "scheduled-thread-pool" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] [[package]] -name = "maybe-owned" -version = "0.3.4" +name = "schemars" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] [[package]] -name = "md-5" -version = "0.9.1" +name = "schemars" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", + "dyn-clone", + "ref-cast", + "serde", + "serde_json", ] [[package]] -name = "memchr" -version = "2.5.0" +name = "scopeguard" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "memoffset" -version = "0.6.4" +name = "sec1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "autocfg", + "base16ct", + "der", + "generic-array", + "pkcs8", + "serdect", + "subtle", + "zeroize", ] [[package]] -name = "migrations_internals" -version = "1.4.1" +name = "secp256k1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ - "diesel", + "bitcoin_hashes", + "rand 0.8.6", + "secp256k1-sys 0.10.1", + "serde", ] [[package]] -name = "migrations_macros" -version = "1.4.2" +name = "secp256k1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", - "syn 1.0.107", + "bitcoin_hashes", + "rand 0.9.3", + "secp256k1-sys 0.11.0", ] [[package]] -name = "mime" -version = "0.3.16" +name = "secp256k1-sys" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" +dependencies = [ + "cc", +] [[package]] -name = "mime_guess" -version = "2.0.3" +name = "secp256k1-sys" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" dependencies = [ - "mime", - "unicase", + "cc", ] [[package]] -name = "miniz_oxide" -version = "0.4.4" +name = "security-framework" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "adler", - "autocfg", + "bitflags 2.11.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", ] [[package]] -name = "miniz_oxide" -version = "0.6.2" +name = "security-framework" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "adler", + "bitflags 2.11.1", + "core-foundation 0.10.0", + "core-foundation-sys", + "libc", + "security-framework-sys", ] [[package]] -name = "mio" -version = "0.8.9" +name = "security-framework-sys" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ + "core-foundation-sys", "libc", - "wasi", - "windows-sys 0.48.0", ] [[package]] -name = "more-asserts" -version = "0.2.1" +name = "semver" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] [[package]] -name = "multiaddr" -version = "0.17.1" +name = "semver" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b36f567c7099511fa8612bbbb52dda2419ce0bdbacf31714e3a5ffdb766d3bd" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ - "arrayref", - "byteorder", - "data-encoding", - "log", - "multibase", - "multihash 0.17.0", - "percent-encoding", "serde", - "static_assertions", - "unsigned-varint 0.7.1", - "url", + "serde_core", ] [[package]] -name = "multibase" -version = "0.9.1" +name = "semver-parser" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +checksum = "9900206b54a3527fdc7b8a938bffd94a568bac4f4aa8113b209df75a09c0dec2" dependencies = [ - "base-x", - "data-encoding", - "data-encoding-macro", + "pest", ] [[package]] -name = "multihash" -version = "0.17.0" +name = "send_wrapper" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" -dependencies = [ - "core2", - "multihash-derive", - "unsigned-varint 0.7.1", -] +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] -name = "multihash" -version = "0.19.1" +name = "seq-macro" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" -dependencies = [ - "core2", - "unsigned-varint 0.7.1", -] +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] -name = "multihash-derive" -version = "0.8.0" +name = "serde" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc076939022111618a5026d3be019fd8b366e76314538ff9a1b59ffbcbf98bcd" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ - "proc-macro-crate", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.107", - "synstructure", + "serde_core", + "serde_derive", ] [[package]] -name = "multimap" -version = "0.8.3" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] [[package]] -name = "native-tls" -version = "0.2.11" +name = "serde_derive" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "never" -version = "0.1.0" +name = "serde_json" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] [[package]] -name = "num-bigint" -version = "0.2.6" +name = "serde_path_to_error" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ - "autocfg", - "num-integer", - "num-traits", + "itoa", "serde", ] [[package]] -name = "num-bigint" -version = "0.4.4" +name = "serde_plain" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" dependencies = [ - "autocfg", - "num-integer", - "num-traits", + "serde", ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "serde_regex" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "bafc8d0c5330cecff10f16b459b479fd9acaa5b4acd7167301414e21b0057012" dependencies = [ - "autocfg", - "num-traits", + "regex", + "serde", ] [[package]] -name = "num-traits" -version = "0.2.17" +name = "serde_spanned" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "autocfg", + "serde_core", ] [[package]] -name = "num_cpus" -version = "1.16.0" +name = "serde_urlencoded" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ - "hermit-abi 0.3.1", - "libc", + "form_urlencoded", + "itoa", + "ryu", + "serde", ] [[package]] -name = "object" -version = "0.24.0" +name = "serde_with" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "crc32fast", + "base64", + "bs58 0.5.1", + "chrono", + "hex", "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", ] [[package]] -name = "object" -version = "0.26.0" +name = "serde_with_macros" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "memchr", + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "once_cell" -version = "1.13.1" +name = "serde_yaml" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] [[package]] -name = "opaque-debug" -version = "0.3.0" +name = "serdect" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" +dependencies = [ + "base16ct", + "serde", +] [[package]] -name = "openssl" -version = "0.10.60" +name = "sha1" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "bitflags 2.4.0", - "cfg-if 1.0.0", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "cfg-if", + "cpufeatures 0.2.12", + "digest 0.10.7", ] [[package]] -name = "openssl-macros" -version = "0.1.0" +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] -name = "openssl-probe" -version = "0.1.4" +name = "sha1_smol" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] -name = "openssl-sys" -version = "0.9.96" +name = "sha2" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", + "cfg-if", + "cpufeatures 0.2.12", + "digest 0.10.7", ] [[package]] -name = "os_str_bytes" -version = "6.0.0" +name = "sha2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] [[package]] -name = "pad" -version = "0.1.6" +name = "sha3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "unicode-width", + "digest 0.11.2", + "keccak", ] [[package]] -name = "parity-scale-codec" -version = "3.0.0" +name = "sha3-asm" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a7f3fcf5e45fc28b84dcdab6b983e77f197ec01f325a33f404ba6855afd1070" +checksum = "9f3f15d4e239ebe08413eed880e0f9b5af4b40ee0472543320efa91d488e96a7" dependencies = [ - "arrayvec 0.7.4", - "bitvec", - "byte-slice-cast", - "impl-trait-for-tuples", - "parity-scale-codec-derive", - "serde", + "cc", + "cfg-if", ] [[package]] -name = "parity-scale-codec-derive" -version = "3.0.0" +name = "shellexpand" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6e626dc84025ff56bf1476ed0e30d10c84d7f89a475ef46ebabee1095a8fba" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 1.0.107", + "dirs", ] [[package]] -name = "parity-wasm" -version = "0.45.0" +name = "shlex" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] -name = "parking_lot" -version = "0.11.2" +name = "signal-hook" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.5", + "libc", + "signal-hook-registry", ] [[package]] -name = "parking_lot" -version = "0.12.1" +name = "signal-hook-mio" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ - "lock_api", - "parking_lot_core 0.9.1", + "libc", + "mio", + "signal-hook", ] [[package]] -name = "parking_lot_core" -version = "0.8.5" +name = "signal-hook-registry" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ - "cfg-if 1.0.0", - "instant", "libc", - "redox_syscall 0.2.10", - "smallvec", - "winapi", ] [[package]] -name = "parking_lot_core" -version = "0.9.1" +name = "signature" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "cfg-if 1.0.0", - "libc", - "redox_syscall 0.2.10", - "smallvec", - "windows-sys 0.32.0", + "digest 0.10.7", + "rand_core 0.6.4", ] [[package]] -name = "paste" -version = "1.0.5" +name = "simd-adler32" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] -name = "percent-encoding" -version = "2.3.1" +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] -name = "petgraph" -version = "0.6.4" +name = "similar" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "e6505efef05804732ed8a3f2d4f279429eb485bd69d5b0cc6b19cc02005cda16" dependencies = [ - "fixedbitset", - "indexmap 2.1.0", + "bstr", ] [[package]] -name = "phf" -version = "0.8.0" +name = "siphasher" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_shared", -] +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] -name = "phf_shared" -version = "0.8.0" +name = "slab" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "siphasher", + "autocfg", ] [[package]] -name = "pin-project" -version = "1.0.12" +name = "slog" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "9b3b8565691b22d2bdfc066426ed48f837fc0c5f2c8cad8d9718f7f99d6995c1" dependencies = [ - "pin-project-internal", + "anyhow", + "erased-serde", + "rustversion", + "serde_core", ] [[package]] -name = "pin-project-internal" -version = "1.0.12" +name = "slog-async" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", ] [[package]] -name = "pin-project-lite" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" - -[[package]] -name = "postgres" -version = "0.19.1" +name = "slog-envlogger" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7871ee579860d8183f542e387b176a25f2656b9fb5211e045397f745a68d1c2" +checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" dependencies = [ - "bytes", - "fallible-iterator 0.2.0", - "futures 0.3.16", "log", - "tokio", - "tokio-postgres", + "regex", + "slog", + "slog-async", + "slog-scope", + "slog-stdlog", + "slog-term", ] [[package]] -name = "postgres-openssl" -version = "0.5.0" +name = "slog-scope" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de0ea6504e07ca78355a6fb88ad0f36cafe9e696cbc6717f16a207f3a60be72" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" dependencies = [ - "futures 0.3.16", - "openssl", - "tokio", - "tokio-openssl", - "tokio-postgres", + "arc-swap", + "lazy_static", + "slog", ] [[package]] -name = "postgres-protocol" -version = "0.6.1" +name = "slog-stdlog" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "fallible-iterator 0.2.0", - "hmac", - "md-5", - "memchr", - "rand", - "sha2 0.9.5", - "stringprep", + "log", + "slog", + "slog-scope", ] [[package]] -name = "postgres-types" -version = "0.2.1" +name = "slog-term" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" dependencies = [ - "bytes", - "fallible-iterator 0.2.0", - "postgres-protocol", + "is-terminal", + "slog", + "term", + "thread_local", + "time", ] [[package]] -name = "ppv-lite86" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" - -[[package]] -name = "pq-sys" -version = "0.4.6" +name = "smallvec" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ - "vcpkg", + "arbitrary", + "serde", ] [[package]] -name = "pretty_assertions" -version = "1.4.0" +name = "snap" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" -dependencies = [ - "diff", - "yansi", -] +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] -name = "prettyplease" -version = "0.1.10" +name = "socket2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" dependencies = [ - "proc-macro2", - "syn 1.0.107", + "libc", + "windows-sys 0.59.0", ] [[package]] -name = "primitive-types" -version = "0.11.1" +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "uint 0.9.1", + "base64ct", + "der", ] [[package]] -name = "priority-queue" -version = "0.7.0" +name = "sqlparser" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a03dfae8e64d4aa651415e2a4321f9f09f2e388a2f8bec36bed03bc22c0b687" +checksum = "07c5f081b292a3d19637f0b32a79e28ff14a9fd23ef47bd7fce08ff5de221eca" dependencies = [ - "indexmap 1.9.3", - "take_mut", + "log", + "recursive", + "sqlparser_derive 0.3.0", ] [[package]] -name = "proc-macro-crate" -version = "1.1.0" +name = "sqlparser" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ - "thiserror", - "toml 0.5.11", + "log", + "recursive", + "sqlparser_derive 0.5.0", ] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "sqlparser_derive" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ - "proc-macro-error-attr", "proc-macro2", "quote", - "syn 1.0.107", - "version_check", + "syn 2.0.118", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "sqlparser_derive" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "version_check", + "syn 2.0.118", ] [[package]] -name = "proc-macro2" -version = "1.0.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +name = "stable-hash" +version = "0.3.4" +source = "git+https://github.com/graphprotocol/stable-hash?rev=42b85d936c6b9f1b7a4b8385d2f081d5564daa69#42b85d936c6b9f1b7a4b8385d2f081d5564daa69" dependencies = [ - "unicode-ident", + "blake3", + "firestorm 0.4.6", + "ibig", + "lazy_static", + "leb128", + "num-traits", ] [[package]] -name = "prometheus" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +name = "stable-hash" +version = "0.4.4" +source = "git+https://github.com/graphprotocol/stable-hash?rev=70300f974e61529675aef3f20803f26a23f0e0ea#70300f974e61529675aef3f20803f26a23f0e0ea" dependencies = [ - "cfg-if 1.0.0", - "fnv", + "blake3", + "firestorm 0.5.1", + "ibig", "lazy_static", - "libc", - "memchr", - "parking_lot 0.12.1", - "protobuf 2.25.0", - "reqwest", - "thiserror", + "leb128", + "num-traits", + "uint 0.10.1", + "xxhash-rust", ] [[package]] -name = "prost" -version = "0.11.9" +name = "stable_deref_trait" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive", -] +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -name = "prost-build" -version = "0.11.5" +name = "stacker" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5320c680de74ba083512704acb90fe00f28f79207286a848e730c45dd73ed6" +checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" dependencies = [ - "bytes", - "heck", - "itertools 0.10.5", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 1.0.107", - "tempfile", - "which", + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", ] [[package]] -name = "prost-derive" -version = "0.11.9" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.107", -] +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "prost-types" -version = "0.11.9" +name = "static_assertions_next" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" + +[[package]] +name = "stringprep" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "prost", + "unicode-bidi", + "unicode-normalization", + "unicode-properties", ] [[package]] -name = "protobuf" -version = "2.25.0" +name = "strsim" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020f86b07722c5c4291f7c723eac4676b3892d47d9a7708dc2779696407f039b" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "protobuf" -version = "3.3.0" +name = "strum" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65f4a8ec18723a734e5dc09c173e0abf9690432da5340285d536edcb4dac190" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "once_cell", - "protobuf-support", - "thiserror", + "strum_macros 0.27.2", ] [[package]] -name = "protobuf-parse" -version = "3.3.0" +name = "strum" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77d6fbd6697c9e531873e81cec565a85e226b99a0f10e1acc079be057fe2fcba" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "anyhow", - "indexmap 1.9.3", - "log", - "protobuf 3.3.0", - "protobuf-support", - "tempfile", - "thiserror", - "which", + "strum_macros 0.28.0", ] [[package]] -name = "protobuf-support" -version = "3.3.0" +name = "strum_macros" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6872f4d4f4b98303239a2b5838f5bbbb77b01ffc892d627957f37a22d7cfe69c" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "thiserror", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "psm" -version = "0.1.14" +name = "strum_macros" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ce37fa8c0428a37307d163292add09b3aedc003472e6b3622486878404191d" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ - "cc", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "quick-error" -version = "1.2.3" +name = "subtle" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] -name = "quote" -version = "1.0.35" +name = "syn" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", + "quote", + "unicode-ident", ] [[package]] -name = "r2d2" -version = "0.8.9" +name = "syn" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ - "log", - "parking_lot 0.11.2", - "scheduled-thread-pool", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rand" -version = "0.8.5" +name = "syn-solidity" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ - "libc", - "rand_chacha", - "rand_core", + "paste", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "sync_wrapper" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ - "ppv-lite86", - "rand_core", + "futures-core", ] [[package]] -name = "rand_core" -version = "0.6.3" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ - "getrandom", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "rayon" -version = "1.5.1" +name = "system-configuration" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "autocfg", - "crossbeam-deque", - "either", - "rayon-core", + "bitflags 2.11.1", + "core-foundation 0.9.4", + "system-configuration-sys", ] [[package]] -name = "rayon-core" -version = "1.9.1" +name = "system-configuration-sys" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "lazy_static", - "num_cpus", + "core-foundation-sys", + "libc", ] [[package]] -name = "redox_syscall" -version = "0.1.57" +name = "take_mut" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] -name = "redox_syscall" -version = "0.2.10" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" -dependencies = [ - "bitflags 1.3.1", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] -name = "redox_users" -version = "0.4.0" +name = "target-lexicon" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" -dependencies = [ - "getrandom", - "redox_syscall 0.2.10", -] +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] -name = "regalloc" -version = "0.0.31" +name = "tempfile" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "log", - "rustc-hash", - "serde", - "smallvec", + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.52.0", ] [[package]] -name = "regex" -version = "1.5.5" +name = "term" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "dirs-next", + "rustversion", + "winapi", ] [[package]] -name = "regex-syntax" -version = "0.6.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" - -[[package]] -name = "region" -version = "2.2.0" +name = "termcolor" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ - "bitflags 1.3.1", - "libc", - "mach", - "winapi", + "winapi-util", ] [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "terminal_size" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "winapi", + "rustix 0.38.34", + "windows-sys 0.48.0", ] [[package]] -name = "reqwest" -version = "0.11.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +name = "test-store" +version = "0.45.0" dependencies = [ - "base64 0.21.0", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "mime_guess", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", + "async-trait", + "diesel", + "diesel-async", + "graph", + "graph-chain-ethereum", + "graph-graphql", + "graph-node", + "graph-store-postgres", + "hex", + "hex-literal", + "lazy_static", + "pretty_assertions", + "prost-types", "serde_json", - "serde_urlencoded", "tokio", - "tokio-native-tls", - "tokio-util 0.7.1", - "tower-service 0.3.1", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "winreg", ] [[package]] -name = "ring" -version = "0.16.20" +name = "thiserror" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", + "thiserror-impl 1.0.61", ] [[package]] -name = "rlp" -version = "0.5.1" +name = "thiserror" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "bytes", - "rustc-hex", + "thiserror-impl 2.0.18", ] [[package]] -name = "rustc-demangle" -version = "0.1.20" +name = "thiserror-impl" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "thiserror-impl" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "rustc-hex" -version = "2.1.0" +name = "thread_local" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] [[package]] -name = "rustc_version" -version = "0.4.0" +name = "threadpool" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" dependencies = [ - "semver", + "num_cpus", ] [[package]] -name = "rustix" -version = "0.37.7" +name = "time" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ - "bitflags 1.3.1", - "errno 0.3.0", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys 0.45.0", + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", ] [[package]] -name = "rustls" -version = "0.20.4" +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ - "log", - "ring", - "sct", - "webpki", + "num-conv", + "time-core", ] [[package]] -name = "rustls-native-certs" -version = "0.6.2" +name = "tiny-keccak" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", + "crunchy", ] [[package]] -name = "rustls-pemfile" -version = "1.0.0" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "base64 0.13.1", + "displaydoc", + "zerovec", ] [[package]] -name = "rustversion" -version = "1.0.11" +name = "tinyvec" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22" +dependencies = [ + "tinyvec_macros", +] [[package]] -name = "ryu" -version = "1.0.5" +name = "tinyvec_macros" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] -name = "same-file" -version = "1.0.6" +name = "tokio" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ - "winapi-util", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", ] [[package]] -name = "schannel" -version = "0.1.19" +name = "tokio-macros" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ - "lazy_static", - "winapi", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "scheduled-thread-pool" -version = "0.2.5" +name = "tokio-native-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ - "parking_lot 0.11.2", + "native-tls", + "tokio", ] [[package]] -name = "scopeguard" -version = "1.1.0" +name = "tokio-openssl" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "6ffab79df67727f6acf57f1ff743091873c24c579b1e2ce4d8f53e47ded4d63d" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio", +] [[package]] -name = "scroll" -version = "0.10.2" +name = "tokio-postgres" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" +checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" dependencies = [ - "scroll_derive", + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.9.3", + "socket2", + "tokio", + "tokio-util", + "whoami", ] [[package]] -name = "scroll_derive" -version = "0.10.5" +name = "tokio-retry" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaaae8f38bb311444cfb7f1979af0bc9240d95795f75f9ceddf6a59b79ceffa0" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "pin-project-lite", + "rand 0.10.1", + "tokio", ] [[package]] -name = "sct" -version = "0.7.0" +name = "tokio-rustls" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "ring", - "untrusted", + "rustls", + "tokio", ] [[package]] -name = "secp256k1" -version = "0.21.3" +name = "tokio-stream" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ - "secp256k1-sys", + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] -name = "secp256k1-sys" -version = "0.4.2" +name = "tokio-test" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" dependencies = [ - "cc", + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", ] [[package]] -name = "security-framework" -version = "2.3.1" +name = "tokio-tungstenite" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ - "bitflags 1.3.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite 0.28.0", + "webpki-roots 0.26.11", ] [[package]] -name = "security-framework-sys" -version = "2.3.0" +name = "tokio-tungstenite" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ - "core-foundation-sys", - "libc", + "futures-util", + "log", + "tokio", + "tungstenite 0.29.0", ] [[package]] -name = "semver" -version = "1.0.21" +name = "tokio-util" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ - "serde", + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", ] [[package]] -name = "serde" -version = "1.0.195" +name = "toml" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "serde_derive", + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.13", ] [[package]] -name = "serde_derive" -version = "1.0.195" +name = "toml" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.0", ] [[package]] -name = "serde_json" -version = "1.0.107" +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ - "itoa", - "ryu", - "serde", + "serde_core", ] [[package]] -name = "serde_plain" -version = "1.0.2" +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ - "serde", + "serde_core", ] [[package]] -name = "serde_regex" -version = "1.1.0" +name = "toml_edit" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "regex", - "serde", + "indexmap 2.14.0", + "toml_datetime 0.6.6", + "winnow 0.5.40", ] [[package]] -name = "serde_spanned" -version = "0.6.4" +name = "toml_parser" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "serde", + "winnow 1.0.0", ] [[package]] -name = "serde_urlencoded" -version = "0.7.1" +name = "toml_writer" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "rustls-native-certs", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower 0.5.2", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", ] [[package]] -name = "serde_yaml" -version = "0.9.21" +name = "tonic-build" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ - "indexmap 1.9.3", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "sha-1" -version = "0.9.7" +name = "tonic-prost" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures 0.1.5", - "digest 0.9.0", - "opaque-debug", + "bytes", + "prost", + "tonic", ] [[package]] -name = "sha-1" -version = "0.10.0" +name = "tonic-prost-build" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures 0.2.2", - "digest 0.10.7", + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.118", + "tempfile", + "tonic-build", ] [[package]] -name = "sha2" -version = "0.9.5" +name = "tower" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures 0.1.5", - "digest 0.9.0", - "opaque-debug", + "futures-core", + "futures-util", + "hdrhistogram", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", ] [[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +name = "tower" +version = "0.5.3" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures 0.2.2", - "digest 0.10.7", + "futures-core", + "futures-util", + "hdrhistogram", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer 0.3.3 (git+https://github.com/tower-rs/tower.git)", + "tower-service 0.3.3 (git+https://github.com/tower-rs/tower.git)", + "tracing", ] [[package]] -name = "sha3" -version = "0.10.8" +name = "tower-http" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "digest 0.10.7", - "keccak", + "bitflags 2.11.1", + "bytes", + "futures-util", + "http 1.4.2", + "http-body", + "pin-project-lite", + "tower 0.5.2", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", ] [[package]] -name = "shellexpand" -version = "3.1.0" +name = "tower-http" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ - "dirs", + "bitflags 2.11.1", + "bytes", + "http 1.4.2", + "percent-encoding", + "pin-project-lite", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "signal-hook-registry" -version = "1.4.0" +name = "tower-layer" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" -dependencies = [ - "libc", -] +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] -name = "siphasher" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729a25c17d72b06c68cb47955d44fda88ad2d3e7d77e025663fdd69b93dd71a1" +name = "tower-layer" +version = "0.3.3" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" [[package]] -name = "slab" -version = "0.4.4" +name = "tower-service" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" +name = "tower-service" +version = "0.3.3" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" [[package]] -name = "slog-async" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" +name = "tower-test" +version = "0.4.1" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", + "pin-project-lite", + "tokio", + "tokio-test", + "tower-layer 0.3.3 (git+https://github.com/tower-rs/tower.git)", + "tower-service 0.3.3 (git+https://github.com/tower-rs/tower.git)", ] [[package]] -name = "slog-envlogger" -version = "2.2.0" +name = "tracing" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", - "regex", - "slog", - "slog-async", - "slog-scope", - "slog-stdlog", - "slog-term", -] - -[[package]] -name = "slog-scope" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" -dependencies = [ - "arc-swap", - "lazy_static", - "slog", + "pin-project-lite", + "tracing-attributes", + "tracing-core", ] [[package]] -name = "slog-stdlog" -version = "4.1.1" +name = "tracing-attributes" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "log", - "slog", - "slog-scope", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "slog-term" -version = "2.8.0" +name = "tracing-core" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c1e7e5aab61ced6006149ea772770b84a0d16ce0f7885def313e4829946d76" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ - "atty", - "chrono", - "slog", - "term", - "thread_local", + "once_cell", ] [[package]] -name = "smallvec" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" - -[[package]] -name = "socket2" -version = "0.4.9" +name = "try-lock" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] -name = "socket2" -version = "0.5.5" +name = "tungstenite" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ - "libc", - "windows-sys 0.48.0", + "bytes", + "data-encoding", + "http 1.4.2", + "httparse", + "log", + "rand 0.9.3", + "rustls", + "rustls-pki-types", + "sha1 0.10.6", + "thiserror 2.0.18", + "utf-8", ] [[package]] -name = "soketto" -version = "0.7.1" +name = "tungstenite" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ - "base64 0.13.1", "bytes", - "futures 0.3.16", + "data-encoding", + "http 1.4.2", "httparse", "log", - "rand", - "sha-1 0.9.7", + "rand 0.9.3", + "sha1 0.10.6", + "thiserror 2.0.18", ] [[package]] -name = "spin" -version = "0.5.2" +name = "twox-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] -name = "stable-hash" -version = "0.3.4" -source = "git+https://github.com/graphprotocol/stable-hash?branch=old#7af76261e8098c58bfadd5b7c31810e1c0fdeccb" -dependencies = [ - "blake3 0.3.8", - "firestorm 0.4.6", - "ibig", - "lazy_static", - "leb128", - "num-traits", -] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] -name = "stable-hash" -version = "0.4.4" -source = "git+https://github.com/graphprotocol/stable-hash?branch=main#e50aabef55b8c4de581ca5c4ffa7ed8beed7e998" +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ - "blake3 0.3.8", - "firestorm 0.5.0", - "ibig", - "lazy_static", - "leb128", - "num-traits", - "uint 0.8.5", - "xxhash-rust", + "byteorder", + "crunchy", + "hex", + "static_assertions", ] [[package]] -name = "stable_deref_trait" -version = "1.2.0" +name = "uint" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6f9227a75a5a540a464c832ad4a4195dbdbecd8787610a56262721fde6f04f90" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] [[package]] -name = "static_assertions" -version = "1.1.0" +name = "unarray" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] -name = "stringprep" -version = "0.1.2" +name = "unicase" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "version_check", ] [[package]] -name = "strsim" -version = "0.10.0" +name = "unicode-bidi" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] -name = "strum" -version = "0.21.0" +name = "unicode-ident" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] -name = "strum_macros" -version = "0.25.3" +name = "unicode-normalization" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.48", + "tinyvec", ] [[package]] -name = "substreams" -version = "0.5.12" +name = "unicode-properties" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3524a4e2931ff6cd58783e62adbd7e44f461752eca0c423793cfb462351f24" -dependencies = [ - "anyhow", - "bigdecimal 0.3.1", - "hex", - "hex-literal 0.3.4", - "num-bigint 0.4.4", - "num-integer", - "num-traits", - "pad", - "prost", - "prost-build", - "prost-types", - "substreams-macro", - "thiserror", -] +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" [[package]] -name = "substreams-entity-change" -version = "1.3.1" +name = "unicode-segmentation" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07421917bd53518cb65b03e03671ecda0653995c71e5a2be815c3c755ea23c0" -dependencies = [ - "base64 0.13.1", - "prost", - "prost-types", - "substreams", -] +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] -name = "substreams-head-tracker" -version = "1.0.0" +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] -name = "substreams-macro" -version = "0.5.12" +name = "unicode-xid" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63c2b15adf5b4d7a6d1a73c73df951a6b2df6fbb4f0b41304dc28c5550ce0ed0" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", - "thiserror", -] +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] -name = "substreams-near-core" -version = "0.10.1" +name = "unit-prefix" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9922f437e6cb86b62cfd8bdede93937def710616ac2825ffff06b8770bbd06df" -dependencies = [ - "bs58", - "prost", - "prost-build", - "prost-types", -] +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] -name = "substreams-trigger-filter" -version = "0.34.0" -dependencies = [ - "anyhow", - "hex", - "prost", - "substreams", - "substreams-entity-change", - "substreams-near-core", - "tonic-build", - "trigger-filters", -] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] -name = "subtle" -version = "2.4.1" +name = "unsigned-varint" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" [[package]] -name = "syn" -version = "1.0.107" +name = "untrusted" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "syn" -version = "2.0.48" +name = "url" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", ] [[package]] -name = "sync_wrapper" -version = "0.1.1" +name = "utf-8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] -name = "synstructure" -version = "0.12.5" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474aaa926faa1603c40b7885a9eaea29b444d1cb2850cb7c0e37bb1a4182f4fa" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", - "unicode-xid", -] +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" [[package]] -name = "take_mut" -version = "0.2.2" +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] -name = "tap" -version = "1.0.1" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] -name = "target-lexicon" -version = "0.12.1" +name = "uuid" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0652da4c4121005e9ed22b79f6c5f2d9e2752906b53a33e9490489ba421a6fb" +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" [[package]] -name = "tempfile" -version = "3.2.0" +name = "valuable" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "rand", - "redox_syscall 0.2.10", - "remove_dir_all", - "winapi", -] +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] -name = "term" -version = "0.7.0" +name = "vcpkg" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] -name = "termcolor" -version = "1.4.0" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" -dependencies = [ - "winapi-util", -] +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "terminal_size" -version = "0.1.17" +name = "wait-timeout" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", - "winapi", ] [[package]] -name = "test-store" -version = "0.34.0" +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ - "diesel", - "graph", - "graph-chain-ethereum", - "graph-core", - "graph-graphql", - "graph-node", - "graph-store-postgres", - "graphql-parser", - "hex", - "hex-literal 0.4.1", - "lazy_static", - "pretty_assertions", - "prost-types", - "serde", + "same-file", + "winapi-util", ] [[package]] -name = "textwrap" -version = "0.16.0" +name = "want" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] [[package]] -name = "thiserror" -version = "1.0.31" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" -dependencies = [ - "thiserror-impl", -] +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "thiserror-impl" -version = "1.0.31" +name = "wasi" +version = "0.13.3+wasi-0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "wit-bindgen-rt", ] [[package]] -name = "thread_local" -version = "1.1.4" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "once_cell", + "wit-bindgen", ] [[package]] -name = "time" -version = "0.3.17" +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "itoa", - "serde", - "time-core", - "time-macros", + "wit-bindgen", ] [[package]] -name = "time-core" +name = "wasite" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - -[[package]] -name = "time-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" -dependencies = [ - "time-core", -] +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] -name = "tiny-keccak" -version = "1.5.0" +name = "wasm-bindgen" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a021c69bb74a44ccedb824a046447e2c84a01df9e5c20779750acb38e11b2" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ - "crunchy", + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] -name = "tiny-keccak" -version = "2.0.2" +name = "wasm-bindgen-futures" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ - "crunchy", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "tinyvec" -version = "1.3.1" +name = "wasm-bindgen-macro" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ - "tinyvec_macros", + "quote", + "wasm-bindgen-macro-support", ] [[package]] -name = "tinyvec_macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" - -[[package]] -name = "tokio" -version = "1.35.1" +name = "wasm-bindgen-macro-support" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot 0.12.1", - "pin-project-lite", - "signal-hook-registry", - "socket2 0.5.5", - "tokio-macros", - "windows-sys 0.48.0", + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.118", + "wasm-bindgen-shared", ] [[package]] -name = "tokio-io-timeout" -version = "1.1.1" +name = "wasm-bindgen-shared" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90c49f106be240de154571dd31fbe48acb10ba6c6dd6f6517ad603abffa42de9" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ - "pin-project-lite", - "tokio", + "unicode-ident", ] [[package]] -name = "tokio-macros" -version = "2.2.0" +name = "wasm-compose" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "b089037d7eb453ed57b560fe7833de0707411c8b9fdc429745ced77e2a1bacb9" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "anyhow", + "heck 0.5.0", + "indexmap 2.14.0", + "log", + "petgraph 0.6.5", + "smallvec", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wat", ] [[package]] -name = "tokio-native-tls" -version = "0.3.0" +name = "wasm-encoder" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" dependencies = [ - "native-tls", - "tokio", + "leb128fmt", + "wasmparser 0.244.0", ] [[package]] -name = "tokio-openssl" -version = "0.6.3" +name = "wasm-encoder" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +checksum = "5a879a421bd17c528b74721b2abf4c62e8f1d1889c2ba8c3c50d02deaf2ce395" dependencies = [ - "futures-util", - "openssl", - "openssl-sys", - "tokio", + "leb128fmt", + "wasmparser 0.251.0", ] [[package]] -name = "tokio-postgres" -version = "0.7.2" +name = "wasm-encoder" +version = "0.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b" +checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator 0.2.0", - "futures 0.3.16", - "log", - "parking_lot 0.11.2", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "socket2 0.4.9", - "tokio", - "tokio-util 0.6.7", + "leb128fmt", + "wasmparser 0.252.0", ] [[package]] -name = "tokio-retry" -version = "0.3.0" +name = "wasm-instrument" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +checksum = "2a47ecb37b9734d1085eaa5ae1a81e60801fd8c28d4cabdd8aedb982021918bc" dependencies = [ - "pin-project", - "rand", - "tokio", + "parity-wasm", ] [[package]] -name = "tokio-rustls" -version = "0.23.3" +name = "wasm-metadata" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ - "rustls", - "tokio", - "webpki", + "anyhow", + "indexmap 2.14.0", + "wasm-encoder 0.244.0", + "wasmparser 0.244.0", ] [[package]] -name = "tokio-stream" -version = "0.1.14" +name = "wasm-streams" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util 0.7.1", + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] -name = "tokio-test" -version = "0.4.2" +name = "wasm-streams" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ - "async-stream", - "bytes", - "futures-core", - "tokio", - "tokio-stream", + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] -name = "tokio-tungstenite" -version = "0.17.2" +name = "wasmparser" +version = "0.118.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "77f1154f1ab868e2a01d9834a805faca7bf8b50d041b4ca714d005d0dab1c50c" dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] -name = "tokio-util" -version = "0.6.7" +name = "wasmparser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bytes", - "futures-core", - "futures-io", - "futures-sink", - "log", - "pin-project-lite", - "tokio", + "bitflags 2.11.1", + "hashbrown 0.15.2", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] -name = "tokio-util" -version = "0.7.1" +name = "wasmparser" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "437970b35b1a85cfde9c74b2398352d8d653f3bd8e3a3db0c063ea8f5b4b36ff" dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", + "bitflags 2.11.1", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "semver 1.0.28", + "serde", ] [[package]] -name = "toml" -version = "0.5.11" +name = "wasmparser" +version = "0.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" dependencies = [ - "serde", + "bitflags 2.11.1", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] -name = "toml" -version = "0.8.8" +name = "wasmprinter" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "8798c1a699bd25648b6708eefe94d97c6f9891febb94b42cca1f7a4b086ea64e" dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", + "anyhow", + "termcolor", + "wasmparser 0.251.0", ] [[package]] -name = "toml_datetime" -version = "0.6.5" +name = "wasmtime" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "c4213d2f019a5e44aa8a61d8826dd33a505bff79f749b14a8bafd67321cb9351" dependencies = [ + "addr2line", + "async-trait", + "bitflags 2.11.1", + "bumpalo", + "cc", + "cfg-if", + "encoding_rs", + "futures 0.3.31", + "fxprof-processed-profile", + "gimli", + "ittapi", + "libc", + "log", + "mach2", + "memfd", + "object", + "once_cell", + "postcard", + "pulley-interpreter", + "rayon", + "rustix 1.1.4", + "semver 1.0.28", "serde", + "serde_derive", + "serde_json", + "smallvec", + "target-lexicon", + "tempfile", + "wasm-compose", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wasmtime-environ", + "wasmtime-internal-cache", + "wasmtime-internal-component-macro", + "wasmtime-internal-component-util", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "wat", + "windows-sys 0.61.2", + "wit-parser 0.251.0", ] [[package]] -name = "toml_edit" -version = "0.21.0" +name = "wasmtime-environ" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "d45863de41977ec6453e859cf843d456fa3fcb45a659b66d16e794f90ec4f5b7" dependencies = [ - "indexmap 2.1.0", + "anyhow", + "cpp_demangle", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "log", + "object", + "postcard", + "rustc-demangle", + "semver 1.0.28", "serde", - "serde_spanned", - "toml_datetime", - "winnow", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wasmprinter", + "wasmtime-internal-component-util", + "wasmtime-internal-core", ] [[package]] -name = "tonic" -version = "0.8.3" +name = "wasmtime-internal-cache" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +checksum = "438bc7dc45fb75297d75f79a9a0ce852345d13ebc6a6863f6f688f013836a9dd" dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes", - "flate2", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "rustls-native-certs", - "rustls-pemfile", - "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-util 0.7.1", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.1", - "tracing", - "tracing-futures", + "base64", + "directories-next", + "log", + "postcard", + "rustix 1.1.4", + "serde", + "serde_derive", + "sha2 0.10.9", + "toml 0.9.12+spec-1.1.0", + "wasmtime-environ", + "windows-sys 0.61.2", + "zstd", ] [[package]] -name = "tonic-build" -version = "0.8.4" +name = "wasmtime-internal-component-macro" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +checksum = "f1e48f8d4966d62a10b6d70722bc432c1e163890be2801d3b5784589ad36ffc3" dependencies = [ - "prettyplease", + "anyhow", "proc-macro2", - "prost-build", "quote", - "syn 1.0.107", + "syn 2.0.118", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser 0.251.0", ] [[package]] -name = "tower" -version = "0.4.13" +name = "wasmtime-internal-component-util" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util 0.7.1", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.1", - "tracing", -] +checksum = "819ad5abd5822a22dbf4014475cdfd1fe790707761cd732d74aaa3ba4d5ba489" [[package]] -name = "tower" -version = "0.4.13" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +name = "wasmtime-internal-core" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc28372e36eaf8cf70faa83b5779137f7e99c8d18569a125d1580e735cc9e4d" dependencies = [ - "futures-core", - "futures-util", - "hdrhistogram", - "indexmap 1.9.3", - "pin-project-lite", - "slab", - "sync_wrapper", - "tokio", - "tokio-util 0.7.1", - "tower-layer 0.3.2 (git+https://github.com/tower-rs/tower.git)", - "tower-service 0.3.2", - "tracing", + "anyhow", + "hashbrown 0.17.0", + "libm", + "serde", ] [[package]] -name = "tower-http" -version = "0.3.2" +name = "wasmtime-internal-cranelift" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" +checksum = "a433efc6e35112a5457e1dc8bc4d8d39820ac7722267e89bc04e5df641f32124" dependencies = [ - "bitflags 1.3.1", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.1", + "cfg-if", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli", + "itertools 0.14.0", + "log", + "object", + "pulley-interpreter", + "smallvec", + "target-lexicon", + "thiserror 2.0.18", + "wasmparser 0.251.0", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", ] [[package]] -name = "tower-layer" -version = "0.3.2" +name = "wasmtime-internal-fiber" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "18a1d3a39d0d210f6b8574ee96a4315e0a14c67f3a1fc3cd5372cb10d2fb4422" +dependencies = [ + "cc", + "cfg-if", + "libc", + "rustix 1.1.4", + "wasmtime-environ", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] [[package]] -name = "tower-layer" -version = "0.3.2" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +name = "wasmtime-internal-jit-debug" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f667288cb4dfa68a4639ffac4d5628535dda64ebdc2b990526efb12b30ba803" +dependencies = [ + "cc", + "object", + "rustix 1.1.4", + "wasmtime-internal-versioned-export-macros", +] [[package]] -name = "tower-service" -version = "0.3.1" +name = "wasmtime-internal-jit-icache-coherence" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "eba651d44ab0faad4c58106b3adb45068189fb65ef50f0c404b6d9e3bf81a357" +dependencies = [ + "cfg-if", + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] [[package]] -name = "tower-service" -version = "0.3.2" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +name = "wasmtime-internal-unwinder" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ecc52563b0558af2a7487eb710de07cc4532564b55528876129238e83118cb1" +dependencies = [ + "cfg-if", + "cranelift-codegen", + "log", + "object", + "wasmtime-environ", +] [[package]] -name = "tower-test" -version = "0.4.0" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +name = "wasmtime-internal-versioned-export-macros" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e747f4a074699ba1b4e4d841fb263f9b7df5bd1555181c4752bf5990d21ba676" dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-test", - "tower-layer 0.3.2 (git+https://github.com/tower-rs/tower.git)", - "tower-service 0.3.2", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "tracing" -version = "0.1.36" +name = "wasmtime-internal-wit-bindgen" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "80009f46991622814196d96fac6fc0a938f46b5cba737a8f4e21e24e5a03856f" dependencies = [ - "cfg-if 1.0.0", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", + "anyhow", + "bitflags 2.11.1", + "heck 0.5.0", + "indexmap 2.14.0", + "wit-parser 0.251.0", ] [[package]] -name = "tracing-attributes" -version = "0.1.22" +name = "wasmtimer" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "futures 0.3.31", + "js-sys", + "parking_lot", + "pin-utils", + "slab", + "wasm-bindgen", ] [[package]] -name = "tracing-core" -version = "0.1.29" +name = "wast" +version = "252.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "942a3449d6a593fccc111a6241c8df52bda168af30e40bf9580d4394d7374c65" dependencies = [ - "once_cell", + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder 0.252.0", ] [[package]] -name = "tracing-futures" -version = "0.2.5" +name = "wat" +version = "1.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +checksum = "c72a4ba7088f7bac94cf516e49882bdf97068904a563768cf249efc839ec42cb" dependencies = [ - "pin-project", - "tracing", + "wast", ] [[package]] -name = "trigger-filters" -version = "0.34.0" +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ - "anyhow", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "try-lock" -version = "0.2.3" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] -name = "tungstenite" -version = "0.17.3" +name = "webpki-root-certs" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand", - "sha-1 0.10.0", - "thiserror", - "url", - "utf-8", + "rustls-pki-types", ] [[package]] -name = "typed-builder" -version = "0.10.0" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "webpki-roots 1.0.5", ] [[package]] -name = "typenum" -version = "1.15.0" +name = "webpki-roots" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +dependencies = [ + "rustls-pki-types", +] [[package]] -name = "uint" -version = "0.8.5" +name = "which" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db035e67dfaf7edd9aebfe8676afcd63eed53c8a4044fed514c8cccf1835177" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "byteorder", - "crunchy", - "rustc-hex", - "static_assertions", + "either", + "home", + "once_cell", + "rustix 0.38.34", ] [[package]] -name = "uint" -version = "0.9.1" +name = "whoami" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "byteorder", - "crunchy", - "hex", - "static_assertions", + "redox_syscall 0.4.1", + "wasite", + "web-sys", ] [[package]] -name = "unicase" -version = "2.6.0" +name = "widestring" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "version_check", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] -name = "unicode-bidi" -version = "0.3.13" +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] [[package]] -name = "unicode-ident" -version = "1.0.1" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "unicode-normalization" -version = "0.1.22" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "tinyvec", + "windows-targets 0.52.6", ] [[package]] -name = "unicode-width" -version = "0.1.8" +name = "windows-link" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] -name = "unicode-xid" -version = "0.2.2" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "unreachable" -version = "1.0.0" +name = "windows-registry" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" dependencies = [ - "void", + "windows-link 0.1.3", + "windows-result", + "windows-strings", ] [[package]] -name = "unsafe-libyaml" -version = "0.2.8" +name = "windows-result" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] [[package]] -name = "unsigned-varint" -version = "0.7.1" +name = "windows-strings" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] [[package]] -name = "unsigned-varint" -version = "0.8.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] [[package]] -name = "untrusted" -version = "0.7.1" +name = "windows-sys" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] [[package]] -name = "url" -version = "2.5.0" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "form_urlencoded", - "idna 0.5.0", - "percent-encoding", + "windows-targets 0.52.6", ] [[package]] -name = "utf-8" -version = "0.7.6" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] [[package]] -name = "uuid" -version = "1.6.1" +name = "windows-sys" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "getrandom", + "windows-targets 0.53.3", ] [[package]] -name = "vcpkg" -version = "0.2.15" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] [[package]] -name = "version_check" -version = "0.9.3" +name = "windows-targets" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] [[package]] -name = "void" -version = "1.0.2" +name = "windows-targets" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] [[package]] -name = "walkdir" -version = "2.3.3" +name = "windows-targets" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "same-file", - "winapi-util", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] -name = "want" -version = "0.3.0" +name = "windows-targets" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "log", - "try-lock", + "windows-link 0.1.3", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "windows_aarch64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] -name = "wasm-bindgen" -version = "0.2.82" +name = "windows_aarch64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" -dependencies = [ - "cfg-if 1.0.0", - "wasm-bindgen-macro", -] +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "wasm-bindgen-backend" -version = "0.2.82" +name = "windows_aarch64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 1.0.107", - "wasm-bindgen-shared", -] +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "wasm-bindgen-futures" -version = "0.4.25" +name = "windows_aarch64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" -dependencies = [ - "cfg-if 1.0.0", - "js-sys", - "wasm-bindgen", - "web-sys", -] +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" [[package]] -name = "wasm-bindgen-macro" -version = "0.2.82" +name = "windows_aarch64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.82" +name = "windows_aarch64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "wasm-bindgen-shared" -version = "0.2.82" +name = "windows_aarch64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "wasm-instrument" -version = "0.2.0" +name = "windows_aarch64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bca81f5279342b38b17d9acbf007a46ddeb73144e2bd5f0a21bfa9fc5d4ab3e" -dependencies = [ - "parity-wasm", -] +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" [[package]] -name = "wasm-streams" -version = "0.2.3" +name = "windows_i686_gnu" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] -name = "wasmparser" -version = "0.78.2" +name = "windows_i686_gnu" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "wasmtime" -version = "0.27.0" +name = "windows_i686_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b310b9d20fcf59385761d1ade7a3ef06aecc380e3d3172035b919eaf7465d9f7" -dependencies = [ - "anyhow", - "backtrace", - "bincode", - "cfg-if 1.0.0", - "cpp_demangle", - "indexmap 1.9.3", - "lazy_static", - "libc", - "log", - "paste", - "psm", - "region", - "rustc-demangle", - "serde", - "smallvec", - "target-lexicon", - "wasmparser", - "wasmtime-cache", - "wasmtime-environ", - "wasmtime-fiber", - "wasmtime-jit", - "wasmtime-profiling", - "wasmtime-runtime", - "wat", - "winapi", -] +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] -name = "wasmtime-cache" -version = "0.27.0" +name = "windows_i686_gnu" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d14d500d5c3dc5f5c097158feee123d64b3097f0d836a2a27dff9c761c73c843" -dependencies = [ - "anyhow", - "base64 0.13.1", - "bincode", - "directories-next", - "errno 0.2.7", - "file-per-thread-logger", - "libc", - "log", - "serde", - "sha2 0.9.5", - "toml 0.5.11", - "winapi", - "zstd", -] +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" [[package]] -name = "wasmtime-cranelift" -version = "0.27.0" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c525b39f062eada7db3c1298287b96dcb6e472b9f6b22501300b28d9fa7582f6" -dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-wasm", - "target-lexicon", - "wasmparser", - "wasmtime-environ", -] +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "wasmtime-debug" -version = "0.27.0" +name = "windows_i686_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d2a763e7a6fc734218e0e463196762a4f409c483063d81e0e85f96343b2e0a" -dependencies = [ - "anyhow", - "gimli 0.24.0", - "more-asserts", - "object 0.24.0", - "target-lexicon", - "thiserror", - "wasmparser", - "wasmtime-environ", -] +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" [[package]] -name = "wasmtime-environ" -version = "0.27.0" +name = "windows_i686_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64d0c2d881c31b0d65c1f2695e022d71eb60b9fbdd336aacca28208b58eac90" -dependencies = [ - "cfg-if 1.0.0", - "cranelift-codegen", - "cranelift-entity", - "cranelift-wasm", - "gimli 0.24.0", - "indexmap 1.9.3", - "log", - "more-asserts", - "serde", - "thiserror", - "wasmparser", -] +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] -name = "wasmtime-fiber" -version = "0.27.0" +name = "windows_i686_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a089d44cd7e2465d41a53b840a5b4fca1bf6d1ecfebc970eac9592b34ea5f0b3" -dependencies = [ - "cc", - "libc", - "winapi", -] +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "wasmtime-jit" -version = "0.27.0" +name = "windows_i686_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4539ea734422b7c868107e2187d7746d8affbcaa71916d72639f53757ad707" -dependencies = [ - "addr2line 0.15.2", - "anyhow", - "cfg-if 1.0.0", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", - "gimli 0.24.0", - "log", - "more-asserts", - "object 0.24.0", - "rayon", - "region", - "serde", - "target-lexicon", - "thiserror", - "wasmparser", - "wasmtime-cranelift", - "wasmtime-debug", - "wasmtime-environ", - "wasmtime-obj", - "wasmtime-profiling", - "wasmtime-runtime", - "winapi", -] +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "wasmtime-obj" -version = "0.27.0" +name = "windows_i686_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1a8ff85246d091828e2225af521a6208ed28c997bb5c39eb697366dc2e2f2b" -dependencies = [ - "anyhow", - "more-asserts", - "object 0.24.0", - "target-lexicon", - "wasmtime-debug", - "wasmtime-environ", -] +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" [[package]] -name = "wasmtime-profiling" -version = "0.27.0" +name = "windows_x86_64_gnu" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24364d522dcd67c897c8fffc42e5bdfc57207bbb6d7eeade0da9d4a7d70105b" -dependencies = [ - "anyhow", - "cfg-if 1.0.0", - "gimli 0.24.0", - "lazy_static", - "libc", - "object 0.24.0", - "scroll", - "serde", - "target-lexicon", - "wasmtime-environ", - "wasmtime-runtime", -] +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] -name = "wasmtime-runtime" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51e57976e8a19a18a18e002c6eb12e5769554204238e47ff155fda1809ef0f7" -dependencies = [ - "anyhow", - "backtrace", - "cc", - "cfg-if 1.0.0", - "indexmap 1.9.3", - "lazy_static", - "libc", - "log", - "mach", - "memoffset", - "more-asserts", - "rand", - "region", - "thiserror", - "wasmtime-environ", - "wasmtime-fiber", - "winapi", -] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "wast" -version = "37.0.0" +name = "windows_x86_64_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc7b9a76845047ded00e031754ff410afee0d50fbdf62b55bdeecd245063d68" -dependencies = [ - "leb128", -] +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "wat" -version = "1.0.39" +name = "windows_x86_64_gnu" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2cc8d9a69d1ab28a41d9149bb06bb927aba8fc9d56625f8b597a564c83f50" -dependencies = [ - "wast", -] +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" [[package]] -name = "web-sys" -version = "0.3.52" +name = "windows_x86_64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696" -dependencies = [ - "js-sys", - "wasm-bindgen", -] +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] -name = "web3" -version = "0.19.0-graph" -source = "git+https://github.com/graphprotocol/rust-web3?branch=graph-patches-onto-0.18#7f8eb6dfcc13a4186f9b42f91de950646bc4a833" -dependencies = [ - "arrayvec 0.7.4", - "base64 0.13.1", - "bytes", - "derive_more", - "ethabi", - "ethereum-types", - "futures 0.3.16", - "futures-timer", - "headers", - "hex", - "idna 0.2.3", - "jsonrpc-core", - "log", - "once_cell", - "parking_lot 0.12.1", - "pin-project", - "reqwest", - "rlp", - "secp256k1", - "serde", - "serde_json", - "soketto", - "tiny-keccak 2.0.2", - "tokio", - "tokio-stream", - "tokio-util 0.6.7", - "url", - "web3-async-native-tls", -] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "web3-async-native-tls" -version = "0.4.0" +name = "windows_x86_64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" -dependencies = [ - "native-tls", - "thiserror", - "tokio", - "url", -] +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "webpki" -version = "0.22.2" +name = "windows_x86_64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" -dependencies = [ - "ring", - "untrusted", -] +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" [[package]] -name = "which" -version = "4.2.2" +name = "windows_x86_64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" -dependencies = [ - "either", - "lazy_static", - "libc", -] +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] -name = "winapi" -version = "0.3.9" +name = "windows_x86_64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winapi-util" -version = "0.1.5" +name = "windows_x86_64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ - "winapi", + "memchr", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "winnow" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" [[package]] -name = "windows-sys" -version = "0.32.0" +name = "winnow" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" dependencies = [ - "windows_aarch64_msvc 0.32.0", - "windows_i686_gnu 0.32.0", - "windows_i686_msvc 0.32.0", - "windows_x86_64_gnu 0.32.0", - "windows_x86_64_msvc 0.32.0", + "memchr", ] [[package]] -name = "windows-sys" -version = "0.45.0" +name = "wiremock" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ - "windows-targets 0.42.2", + "assert-json-diff", + "base64", + "deadpool 0.12.3", + "futures 0.3.31", + "http 1.4.2", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", ] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" dependencies = [ - "windows-targets 0.48.0", + "wit-bindgen-rust-macro", ] [[package]] -name = "windows-targets" -version = "0.42.2" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "anyhow", + "heck 0.5.0", + "wit-parser 0.244.0", ] [[package]] -name = "windows-targets" -version = "0.48.0" +name = "wit-bindgen-rt" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "bitflags 2.11.1", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.14.0", + "prettyplease", + "syn 2.0.118", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.32.0" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.118", + "wit-bindgen-core", + "wit-bindgen-rust", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.244.0", + "wasm-metadata", + "wasmparser 0.244.0", + "wit-parser 0.244.0", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.14.0", + "log", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.244.0", +] [[package]] -name = "windows_i686_gnu" -version = "0.32.0" +name = "wit-parser" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "e960732e824fab95099971a09e638979347c94ca48568d3c854c945729196947" +dependencies = [ + "anyhow", + "hashbrown 0.17.0", + "id-arena", + "indexmap 2.14.0", + "log", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.251.0", +] [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] -name = "windows_i686_gnu" -version = "0.48.0" +name = "writeable" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] -name = "windows_i686_msvc" -version = "0.32.0" +name = "ws_stream_wasm" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" +dependencies = [ + "async_io_stream", + "futures 0.3.31", + "js-sys", + "log", + "pharos", + "rustc_version 0.4.0", + "send_wrapper", + "thiserror 2.0.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "wyz" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] [[package]] -name = "windows_i686_msvc" -version = "0.48.0" +name = "xxhash-rust" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "63658493314859b4dfdf3fb8c1defd61587839def09582db50b8a4e93afca6bb" [[package]] -name = "windows_x86_64_gnu" -version = "0.32.0" +name = "yansi" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "yoke" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] [[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" +name = "yoke-derive" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "zerocopy" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" +name = "zerocopy-derive" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.32.0" +name = "zerofrom" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "zerofrom-derive" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +name = "zeroize" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] [[package]] -name = "winnow" -version = "0.5.17" +name = "zeroize_derive" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ - "memchr", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "winreg" -version = "0.10.1" +name = "zerovec" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ - "winapi", + "yoke", + "zerofrom", + "zerovec-derive", ] [[package]] -name = "wyz" -version = "0.5.0" +name = "zerovec-derive" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ - "tap", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "xxhash-rust" -version = "0.8.5" +name = "zlib-rs" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074914ea4eec286eb8d1fd745768504f420a1f7b7919185682a4a267bed7d2e7" +checksum = "c745c48e1007337ed136dc99df34128b9faa6ed542d80a1c673cf55a6d7236c8" [[package]] -name = "yansi" -version = "0.5.1" +name = "zmij" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" -version = "0.6.1+zstd.1.4.9" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "3.0.1+zstd.1.4.9" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "1.4.20+zstd.1.4.9" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", - "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index c8dd5cf1999..a956a514a95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,20 +1,33 @@ [workspace] +resolver = "2" members = [ "core", - "chain/*", + "core/graphman", + "core/graphman_store", + "chain/common", + "chain/ethereum", + "chain/near", + "gnd", "graphql", "node", - "runtime/*", - "server/*", - "store/*", - "substreams/*", + "runtime/derive", + "runtime/test", + "runtime/wasm", + "server/graphman", + "server/http", + "server/index-node", + "server/json-rpc", + "server/metrics", + "store/postgres", + "store/test-store", "graph", "tests", + "graph/derive", ] [workspace.package] -version = "0.34.0" -edition = "2021" +version = "0.45.0" +edition = "2024" authors = ["The Graph core developers & contributors"] readme = "README.md" homepage = "https://thegraph.com" @@ -22,20 +35,107 @@ repository = "https://github.com/graphprotocol/graph-node" license = "MIT OR Apache-2.0" [workspace.dependencies] -prost = "0.11.9" -prost-types = "0.11.9" -tonic = { version = "0.8.3", features = ["tls-roots", "gzip"] } -tonic-build = { version = "0.8.4", features = ["prost"] } +alloy = { version = "2.0.5", features = ["dyn-abi", "json-abi", "full", "arbitrary", "json-rpc", "serde"] } +alloy-rpc-types = "2.0.0" +# rustls is pulled in transitively by alloy (aws_lc_rs) and object_store via +# reqwest (ring). With both providers linked, rustls 0.23 requires an explicit +# default provider to be installed before any TLS use. We install aws_lc_rs +# explicitly in each binary/test entry point. +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs"] } +anyhow = "1.0" +async-graphql = { version = "7.2.1", features = ["chrono"] } +async-graphql-axum = "7.2.1" +async-trait = "0.1.74" +axum = "0.8.9" +chrono = "0.4.44" +bs58 = "0.5.1" +clap = { version = "4.5.4", features = ["derive", "env", "wrap_help"] } +clap_complete = "4" +console = "0.16" +derive_more = { version = "2.1.1", default-features = false } +diesel = { version = "2.2.7", features = [ + "postgres", + "serde_json", + "numeric", + "r2d2", + "chrono", + "i-implement-a-third-party-backend-and-opt-into-breaking-changes", +] } +diesel-async = { version = "0.9.0", features = ["deadpool", "async-connection-wrapper", "tokio", "postgres"] } +diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } +diesel-dynamic-schema = { version = "0.2.3", features = ["postgres"] } +diesel_derives = "2.3.9" +diesel_migrations = "2.3.2" +env_logger = "0.11.10" +envconfig = "0.11.1" +git-testament = "0.2" +graph = { path = "./graph" } +graph-core = { path = "./core" } +graph-store-postgres = { path = "./store/postgres" } +graphman-server = { path = "./server/graphman" } +graphman = { path = "./core/graphman" } +graphman-store = { path = "./core/graphman_store" } +graphql-tools = "0.5.1" +indicatif = "0.18" +Inflector = "0.11.3" +itertools = "0.15.0" +lazy_static = "1.5.0" +prost = "0.14" +prost-types = "0.14" +redis = { version = "1.2.2", features = [ + "aio", + "connection-manager", + "tokio-comp", +] } +regex = "1.5.4" +reqwest = "0.12.23" +semver = { version = "1", features = ["serde"] } +serde = { version = "1.0.126", features = ["rc"] } +serde_derive = "1.0.125" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +serde_regex = "1.1.0" +serde_yaml = "0.9.21" +slog = { version = "2.8.2", features = ["release_max_level_trace", "max_level_trace"] } +slog-async = "2.5.0" +slog-term = "2.7.0" +sqlparser = { version = "0.62.0", features = ["visitor"] } +strum = { version = "0.28", features = ["derive"] } +syn = { version = "2.0.117", features = ["full"] } +test-store = { path = "./store/test-store" } +thiserror = "2.0.18" +deadpool = { version = "0.13", features = ["rt_tokio_1", "managed"] } +tokio = { version = "1.50.0", features = ["full"] } +tokio-stream = { version = "0.1.18", features = ["sync"] } +tokio-retry = "0.3.0" -# Incremental compilation on Rust 1.58 causes an ICE on build. As soon as graph node builds again, these can be removed. -[profile.test] -incremental = false +tonic = { version = "0.14", features = ["tls-native-roots", "gzip"] } +tonic-prost = "0.14" +tonic-prost-build = "0.14" +tower-http = { version = "0.7.0", features = ["cors"] } +tower = { version = "0.5.1", features = ["full"] } +wasmparser = "0.118.1" +wasmtime = { version = "46.0.0", features = ["async"] } +rand = { version = "0.9.2", features = ["os_rng"] } +prometheus = "0.14.0" +url = "2.5.8" -[profile.dev] -incremental = false +# Dependencies related to Amp subgraphs +ahash = "0.8.11" +arrow = { version = "=59.0.0" } +arrow-flight = { version = "=59.0.0", features = ["flight-sql-experimental"] } +parquet = { version = "=59.0.0" } +futures = "0.3.31" +half = "2.7.1" +indoc = "2.0.7" +lazy-regex = "3.6.0" +parking_lot = "0.12.4" +sqlparser-latest = { version = "0.57.0", package = "sqlparser", features = ["visitor"] } +tokio-util = "0.7.15" + +[workspace.lints.clippy] +too_many_arguments = "allow" +type_complexity = "allow" [profile.release] -lto = true opt-level = 's' strip = "debuginfo" - diff --git a/NEWS.md b/NEWS.md index 1470de669b2..8068ecbc9fa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,523 @@ # NEWS -## Unreleased +## v0.45.0 + +``` +$ docker pull graphprotocol/graph-node:v0.45.0 +``` + +### Breaking Changes + +- **`GRAPH_GETH_ETH_CALL_ERRORS` renamed to `GRAPH_RPC_ETH_CALL_ERRORS`.** The env var (and internal config field) controlling which `eth_call` error messages are treated as reverts is renamed for provider-neutral naming. Update it before upgrading, or the setting is silently ignored and defaults apply. ([#6487](https://github.com/graphprotocol/graph-node/pull/6487)) + +### What's New + +- **Postponed attribute index creation re-enabled, now built in the background.** `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` (introduced in v0.44.0 but inert — hardcoded off) now actually defers non-critical attribute indexes until a new deployment nears chain head (`GRAPH_POSTPONE_INDEXES_CREATION_THRESHOLD`, default 10000 blocks from head). Index creation now runs in a background task instead of blocking block processing, and interrupted `CREATE INDEX CONCURRENTLY` runs are detected and rebuilt on retry instead of being silently skipped. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608)) +- **`ethereum.decodeParams` host function.** Decodes calldata/event data whose top-level type is a dynamic tuple (e.g. Gnosis Safe's `execTransaction`), which `ethereum.decode` cannot handle since it expects a single ABI value with a leading offset word. Gated behind apiVersion 0.0.10; `ethereum.decode` itself is unchanged. ([#6649](https://github.com/graphprotocol/graph-node/pull/6649)) + +### Bug Fixes + +- Fixed `EthereumLogFilter` merging per-handler `receipt` flags with overwrite instead of OR semantics, so a handler declaring `receipt: true` could silently run with `event.receipt == null` if another handler on the same filter key declared `receipt: false`. ([#6558](https://github.com/graphprotocol/graph-node/pull/6558)) +- Fixed grafts silently succeeding when the graft block was below the base deployment's earliest available block, leaving the destination missing pre-graft entity versions and producing `unexpected null in handler` errors on every event; `Graft::validate` now rejects these upfront. ([#6610](https://github.com/graphprotocol/graph-node/pull/6610)) +- Fixed `_logs` queries returning an empty result (indistinguishable from a subgraph producing no logs) when no log store is configured; now returns a `NotSupported` error. ([#6637](https://github.com/graphprotocol/graph-node/pull/6637)) +- Fixed startup panicking when a shard is configured with `pool_size = 0`; such shards' chains are now skipped instead of erroring. ([#6648](https://github.com/graphprotocol/graph-node/pull/6648)) +- Fixed the Firehose extended-blocks check matching on `optimism-mainnet` instead of the `optimism` chain ID used by Firehose endpoints, so Optimism's extended block data went undetected. ([#6662](https://github.com/graphprotocol/graph-node/pull/6662)) +- Fixed fulltext (`tsvector`) search columns being silently dropped when copying or grafting a subgraph, leaving the destination's fulltext index empty. ([#6687](https://github.com/graphprotocol/graph-node/pull/6687)) +- Fixed Reth `StackUnderflow`/`StackOverflow`/`OpcodeNotFound` EVM halts being misclassified as non-deterministic — Reth surfaces them via its `Debug` format (e.g. `EVM error: StackUnderflow`), which didn't match graph-node's space-separated deterministic-error patterns, so subgraphs indexing via Reth-backed RPC providers stalled and retried instead of recording a deterministic revert. ([#6645](https://github.com/graphprotocol/graph-node/pull/6645)) + +### Graphman + +- Fixed `graphman dump` panicking on tables with more than ~2 GiB in a single string/binary column; batches are now split into byte-bounded slices before conversion to Arrow. ([#6646](https://github.com/graphprotocol/graph-node/pull/6646)) + +### gnd (Graph Node Dev) + +- Fixed `gnd add` producing a subgraph that failed `codegen`/`build` because it never wrote new event entities to `schema.graphql`; unified `add` and `init` onto one scaffolding path, which also fixes event-name collisions across data sources, overloaded-event disambiguation, small-integer type mapping, and reserved-word parameter names. ([#6660](https://github.com/graphprotocol/graph-node/pull/6660)) + +### Upgrade Notes + +- If you set `GRAPH_GETH_ETH_CALL_ERRORS`, rename it to `GRAPH_RPC_ETH_CALL_ERRORS` before upgrading. ([#6487](https://github.com/graphprotocol/graph-node/pull/6487)) +- `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` now actually takes effect (it was a no-op in v0.44.0). If you already set it expecting no behavior change, be aware new deployments will now defer some attribute indexes until near chain head. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608)) +- If a subgraph was grafted from a base block below the base deployment's earliest available block before this release, it may be missing pre-graft entity versions; re-create the graft from a valid earliest block. ([#6610](https://github.com/graphprotocol/graph-node/pull/6610)) + +### Contributors + +Thanks to all contributors for this release: @cargopete, @dimitrovmaksim, @fordN, @incrypto32, @lutter, @YaroShkvorets + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.44.0...v0.45.0 + +## v0.44.0 + +``` +$ docker pull graphprotocol/graph-node:v0.44.0 +``` + +### Critical Fix + +- **`EntityCache::load_related` returned wrong derived-collection membership on same-block parent reassignment.** Membership was decided against intermediate cache layers rather than the entity's final state, so entities reassigned to a new parent within the same block could be left in the old collection, omitted from the new one, or wrongly retained after a cross-handler revert. See Upgrade Notes. ([#6548](https://github.com/graphprotocol/graph-node/pull/6548)) + +### Breaking Changes + +- **Elasticsearch CLI flags removed.** `--elasticsearch-url`, `--elasticsearch-user`, `--elasticsearch-password` (and the matching `ELASTICSEARCH_*` env vars) are gone. Configure Elasticsearch under the new `[log_store]` section in `graph-node.toml`. See Upgrade Notes. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + +### What's New + +- **Query subgraph logs over GraphQL.** A new `_logs` field exposes subgraph-emitted logs (mapping `log.*` calls, runtime, system) with filters for level, timestamp range, and text search, plus pagination and `orderDirection`. The old Elasticsearch sink for subgraph logs is replaced by a unified `LogStore` abstraction with three backends configured under a new `[log_store]` section in `graph-node.toml`: File (JSON Lines), Loki, or Elasticsearch. Omit `[log_store]` to disable; logs still go to stdout/stderr. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + + + + +### Improvements + +- Revert entity versions during copy instead of in a separate post-copy pass, speeding up subgraph copy and graft. ([#6472](https://github.com/graphprotocol/graph-node/pull/6472)) + +### Bug Fixes + +- Fixed `trace_filter` deserialization failing on providers that omit `result.output` (notably Sonic), which caused repeated trace ingestion retries. Fixed upstream in alloy 2.0.5 ([alloy#3931](https://github.com/alloy-rs/alloy/pull/3931)) and picked up via the bump in this release. ([#6576](https://github.com/graphprotocol/graph-node/pull/6576)) +- Fixed runner panics on out-of-range `BigInt` values: `to_signed_u256` replaced by fallible `to_i256`, and `to_unsigned_u256` now errors on values `>= 2^256` instead of panicking. ([#6560](https://github.com/graphprotocol/graph-node/pull/6560)) +- Fixed `graphman chain change-shard` failing with a duplicate-key error on revert to the original shard. Backups now use unique names and revert reuses the existing `-old`. ([#6199](https://github.com/graphprotocol/graph-node/pull/6199)) + +### Upgrade Notes + +- v0.44.0 changes `EntityCache::load_related` results for same-block parent-reassignment edge cases ([#6548](https://github.com/graphprotocol/graph-node/pull/6548)). Subgraphs that read derived fields within handlers may diverge in POI from v0.43.0 on blocks exhibiting this pattern; resync from before the affected blocks if affected. The v0.44.0 result is canonical. +- If you set `--elasticsearch-*` flags or `ELASTICSEARCH_*` env vars, migrate to a `[log_store]` section in `graph-node.toml` before upgrading. See `docs/log-store.md`. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + +### Contributors + +Thanks to all contributors for this release: @erayack, @fordN, @incrypto32, @lutter + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.43.0...v0.44.0 + +## v0.43.0 + +### What's New + +- **`skipDuplicates` for immutable entities.** A new `skipDuplicates` parameter on the `@entity` directive (`@entity(immutable: true, skipDuplicates: true)`) silently skips duplicate inserts instead of failing the subgraph. This unblocks subgraph composition on Amp-powered subgraphs where SQL queries can produce the same entities across block ranges. ([#6458](https://github.com/graphprotocol/graph-node/pull/6458)) +- **Per-chain RPC settings via TOML config.** Chain-specific tuning parameters (`json_rpc_timeout`, `request_retries`, `max_block_range_size`, `polling_interval`, `block_batch_size`, etc.) can now be set per chain in `config.toml` instead of relying on global environment variables. Fully backwards compatible — env vars remain the fallback. ([#6459](https://github.com/graphprotocol/graph-node/pull/6459)) +- **RPC provider failover for block ingestor.** When the current RPC provider becomes unreachable during block ingestion polling, graph-node now automatically switches to a healthy alternative provider. ([#6430](https://github.com/graphprotocol/graph-node/pull/6430)) +- Warn on startup when running a debug build, with `[DEBUG-BUILD]` prefix on all log lines. ([#6488](https://github.com/graphprotocol/graph-node/pull/6488)) + +### Improvements + +- **Call cache eviction rewrite.** Replaced the old per-contract iteration approach with ctid-based join deletes against the `call_meta` table, significantly faster on large `call_cache` tables. The old `--ttl-max-contracts` flag has been replaced by `--max-contracts`, which computes an effective TTL cutoff instead of limiting per-invocation iteration. Eviction now also returns stats on the number of contracts and entries removed. ([#6476](https://github.com/graphprotocol/graph-node/pull/6476)) ([#6477](https://github.com/graphprotocol/graph-node/pull/6477)) +- Reduced unnecessary `eth_getBlockByHash` and `eth_getBlockByNumber` RPC calls by checking the block cache first in `block_pointer_from_number` and `fetch_full_block_with_rpc`. ([#6491](https://github.com/graphprotocol/graph-node/pull/6491), partially reverted by [#6537](https://github.com/graphprotocol/graph-node/pull/6537)) +- Header-only `ChainStore` query methods (`ancestor_block_ptr`, `block_parent_ptr`) skip deserializing the full block `data` JSONB column when only hash/number/parent are needed. ([#6456](https://github.com/graphprotocol/graph-node/pull/6456)) +- Batch checking for update attempts on immutable entities, reducing per-entity overhead. +- Node name is now used as the PostgreSQL `application_name` when `PGAPPNAME` is not set, making it easier to identify graph-node connections in `pg_stat_activity`. + +### Bug Fixes + +- Fixed dropped block trigger when `once` and `polling` filters match the same block — only one trigger type was firing. ([#6530](https://github.com/graphprotocol/graph-node/pull/6530)) +- Fixed block stream ignoring configured `endBlock` in two cases: the block-skip optimization bypassing the `max_end_block` check, and `max_end_block` being set to `None` when multiple data sources share the same `endBlock`. ([#6474](https://github.com/graphprotocol/graph-node/pull/6474)) +- Fixed GraphQL introspection not returning `isDeprecated: false` for `__InputValue`, which caused some client libraries to fail. ([#6475](https://github.com/graphprotocol/graph-node/pull/6475)) +- Fixed IPC provider connections failing when configured with `ipc://` or `file://` URLs — the URL was passed directly to the transport instead of extracting the file path. ([#6443](https://github.com/graphprotocol/graph-node/pull/6443)) +- Fixed `graphman config pools` not working due to hardcoded pool size override. ([#6444](https://github.com/graphprotocol/graph-node/pull/6444)) +- Fixed unfail retry mechanism stopping after the first attempt when the deployment head was still behind the error block. ([#6529](https://github.com/graphprotocol/graph-node/pull/6529)) + +### Note on `ethereum.decode()` whitespace handling + +The migration from `ethabi` to `alloy` in v0.42.0 ([#6063](https://github.com/graphprotocol/graph-node/pull/6063)) incidentally fixed a long-standing parsing bug in `ethabi` where type strings containing whitespace before a type name (e.g. `" address"` with a leading space) were silently decoded as `Uint(8)` instead of the intended type. `alloy` parses these correctly. + +Subgraphs that relied on the incorrect `Uint(8)` decoding to subsequently call `.toBigInt()` on what is actually an `Address` value will abort on v0.42.0+ with: + +``` +Mapping aborted ... Ethereum value is not an int or uint. +``` + +This is not a graph-node regression. Recompile the subgraph with the correct accessor (`.toAddress()` for addresses) to fix. See [#6461](https://github.com/graphprotocol/graph-node/issues/6461) for details. + +### gnd (Graph Node Dev) + +- `gnd indexer` command that delegates to `graph-indexer`, allowing indexer management (allocations, rules, cost models, status) directly through gnd. ([#6492](https://github.com/graphprotocol/graph-node/pull/6492)) +- `gnd deploy` now prompts for `--version-label` in interactive mode and requires it in non-interactive mode. ([#6532](https://github.com/graphprotocol/graph-node/pull/6532)) +- Test framework improvements: partial receipt.logs support, mock IPFS/Arweave clients for file data source testing. ([#6442](https://github.com/graphprotocol/graph-node/pull/6442)) + +### Contributors + +Thanks to all contributors for this release: @aayushbaluni, @dimitrovmaksim, @incrypto32, @isum, @lutter, @suntzu + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.42.1...v0.43.0 + +## v0.42.1 + +### Bug Fixes + +- Fixed multi-shard deployments becoming unresponsive due to `mirror_primary_tables` holding locks indefinitely, exhausting the connection pool. Added `statement_timeout` and `lock_timeout` to prevent stuck queries from cascading into a full lockup. ([#6446](https://github.com/graphprotocol/graph-node/pull/6446)) +- Fixed the `ingestor` setting in `[chains.*]` config sections having no effect — block ingestion ran regardless of its value. ([#6447](https://github.com/graphprotocol/graph-node/pull/6447)) + +## v0.42.0 + +### Breaking Changes + +- **Substreams support removed.** Substreams have been unsupported on the network for some time. All substreams-related code has been removed, simplifying the codebase significantly. Substreams-based subgraphs will no longer work. ([#6261](https://github.com/graphprotocol/graph-node/pull/6261)) +- **Migrated from rust-web3 to alloy.** The Ethereum RPC layer now uses the alloy crate instead of rust-web3. This should be transparent to most users, but chains that do not return EIP-2718 typed transaction fields may need the `no_eip2718` provider feature flag. ([#6063](https://github.com/graphprotocol/graph-node/pull/6063)) ([#6317](https://github.com/graphprotocol/graph-node/pull/6317)) + +### What's New + +- **Amp-powered subgraphs (experimental).** A new kind of subgraph powered by [Amp](https://github.com/edgeandnode/amp), the blockchain native database. Amp-powered subgraphs index data directly from Amp servers instead of processing blocks individually, reducing indexing time from days/weeks to minutes/hours. See `docs/amp-powered-subgraphs.md` for details. ([#6218](https://github.com/graphprotocol/graph-node/pull/6218)) ([#6293](https://github.com/graphprotocol/graph-node/pull/6293)) ([#6369](https://github.com/graphprotocol/graph-node/pull/6369)) +- **SQL query interface (experimental).** A new experimental SQL query interface allows querying subgraph data using SQL syntax via GraphQL. Documentation in `docs/implementation/sql-interface.md`. ([#6172](https://github.com/graphprotocol/graph-node/pull/6172)) +- **Async store.** All store and database interactions are now fully async using `diesel_async`, eliminating blocking database calls from tokio tasks. This improves throughput and reduces the risk of thread starvation under heavy load. ([#6194](https://github.com/graphprotocol/graph-node/pull/6194)) ([#6185](https://github.com/graphprotocol/graph-node/pull/6185)) +- **RPC request compression.** Configurable compression for outgoing JSON-RPC requests to upstream providers, supporting gzip, brotli, and deflate. Configured via provider `features` in the config file. ([#6084](https://github.com/graphprotocol/graph-node/pull/6084)) +- Query the current (partially filled) aggregation bucket with `current: include` in GraphQL requests. ([#6293](https://github.com/graphprotocol/graph-node/pull/6293)) +- Extended `first` and `last` aggregation support to `String`, `Bytes`, and entity reference types. ([#6225](https://github.com/graphprotocol/graph-node/pull/6225)) +- Automated account-like table optimization — an optional background job that detects and marks tables with high entity update ratios as account-like. Enabled via `GRAPH_STORE_ACCOUNT_LIKE_SCAN_INTERVAL_HOURS`, `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT`, and `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`. ([#6209](https://github.com/graphprotocol/graph-node/pull/6209)) +- Subgraph table sizes and row counts are now exposed in `SubgraphIndexingStatus` via the index-node API. ([#6201](https://github.com/graphprotocol/graph-node/pull/6201)) +- Configurable database setup timeout via `GRAPH_STORE_SETUP_TIMEOUT`. +- Option to disable the store call cache via `GRAPH_STORE_DISABLE_CALL_CACHE`, useful for indexers with locally hosted RPC nodes. ([#6214](https://github.com/graphprotocol/graph-node/pull/6214)) +- Docker: added profiling support to the `graph-node-debug` image with frame pointers and `linux-perf`. ([#6418](https://github.com/graphprotocol/graph-node/pull/6418)) + +### Improvements + +- Optimized WASM trigger instantiation by caching linker and compilation artifacts, reducing per-trigger overhead. ([#6364](https://github.com/graphprotocol/graph-node/pull/6364)) +- Optimized log filter matching and trigger allocation, reducing CPU usage during block processing. ([#6419](https://github.com/graphprotocol/graph-node/pull/6419)) +- Reduced unnecessary entity clones during block processing. ([#6362](https://github.com/graphprotocol/graph-node/pull/6362)) +- Replaced `std::sync::RwLock` with `parking_lot::RwLock` across multiple components for better performance under contention. +- Deadlock prevention under heavy load, especially during index node startup — connection pool and store semaphore improvements. ([#6224](https://github.com/graphprotocol/graph-node/pull/6224)) +- Raw subgraph manifests are now loaded from the store cache during startup, reducing IPFS dependency. ([#6223](https://github.com/graphprotocol/graph-node/pull/6223)) +- Deployment hashes are now logged when removing a subgraph. ([#6405](https://github.com/graphprotocol/graph-node/pull/6405)) + +### Bug Fixes + +- Fixed duplicate VID when multiple offchain triggers (file/IPFS data sources) fire in the same block, causing unique constraint violations. Also fixed in `ipfs.map()` callbacks. ([#6336](https://github.com/graphprotocol/graph-node/pull/6336)) ([#6416](https://github.com/graphprotocol/graph-node/pull/6416)) +- Fixed incorrect `nocase` filter generation for non-String types (e.g., `Bytes`), which caused SQL errors like `operator does not exist: bytea ~~* bytea`. ([#6351](https://github.com/graphprotocol/graph-node/pull/6351)) +- Fixed `can_copy_from` failures not fully rolling back deployment creation, leaving orphaned records. ([#6228](https://github.com/graphprotocol/graph-node/pull/6228)) +- Fixed multi-column index validation bug where column order mismatch caused valid indexes to be silently dropped during subgraph copy/graft. ([#6341](https://github.com/graphprotocol/graph-node/pull/6341)) +- Fixed grafting failure ("Unexpected null for non-null column") when source tables have too few rows for PostgreSQL to generate `histogram_bounds` statistics. ([#6275](https://github.com/graphprotocol/graph-node/pull/6275)) +- Fixed panic when converting a negative number to `U256` in the runtime — now returns an error instead. ([#6219](https://github.com/graphprotocol/graph-node/pull/6219)) +- Fixed `graphman copy` copying pruned `earliest_block_number` from source, which could leave the destination deployment in an invalid state. ([#6384](https://github.com/graphprotocol/graph-node/pull/6384)) +- Fixed calls returning `0x` being incorrectly cached in the call cache, preventing stale empty results from persisting. ([#6187](https://github.com/graphprotocol/graph-node/pull/6187)) + +### Graphman + +- **`graphman dump` and `graphman restore` (experimental).** New commands for exporting and importing subgraph data in Parquet format, enabling backup and migration of deployment data across shards. Supports incremental dumps and progress reporting. See `docs/dump.md` for details. ([#6397](https://github.com/graphprotocol/graph-node/pull/6397)) +- `graphman chain call-cache remove` now supports `--ttl-days` to remove stale call cache entries that haven't been accessed within a specified number of days. ([#6186](https://github.com/graphprotocol/graph-node/pull/6186)) + +### gnd (Graph Node Dev) + +- **[experimental]** Major expansion of `gnd` as a drop-in replacement for `graph-cli`: `init`, `add`, `codegen`, `build`, `publish`, `deploy`, `create`, `remove`, `auth`, `clean`, and `test` commands. ([#6282](https://github.com/graphprotocol/graph-node/pull/6282)) + +### Contributors + +Thanks to all contributors for this release: @DaMandal0rian, @dimitrovmaksim, @fubhy, @hudsonhrh, @incrypto32, @isum, @lutter, @shiyasmohd + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.41.2...v0.42.0 + +## v0.41.2 + +### Bug Fixes + +- Fixed entity corruption after rewinding pruned subgraphs — rewinding a subgraph with `history_blocks` (pruning) enabled could leave frequently-updated entities with no open version, causing deterministic `unexpected null` errors and making the subgraph unrecoverable without a full re-index ([#6340](https://github.com/graphprotocol/graph-node/pull/6340)) +- Fixed `graphman rewind` and `graphman truncate` targeting the wrong deployment when multiple instances of the same subgraph exist ([#6299](https://github.com/graphprotocol/graph-node/pull/6299)) +- Fixed batch write memoization bug where entity lookups could incorrectly return `None`, causing missed entity updates during indexing ([#6314](https://github.com/graphprotocol/graph-node/pull/6314)) +- Expanded deterministic RPC error handling for Reth and Erigon — added `vm execution error`, `invalidjump`, `notactivated`, and `invalidfeopcode` to recognized deterministic errors. Indexers using these clients no longer need to set `GRAPH_GETH_ETH_CALL_ERRORS` manually for these cases ([#6355](https://github.com/graphprotocol/graph-node/pull/6355)) + +**Full changelog:** + +## v0.41.1 + +### Bug Fixes + +- Fixed a regression in v0.41.0 where the indexing status endpoint would return an empty list when querying all subgraphs without specific deployment filters ([#6210](https://github.com/graphprotocol/graph-node/pull/6210)) + +**Full changelog:** + +## v0.41.0 + +### Bug Fixes + +- Fix non-deterministic handling of transaction receipts in event handlers; receipts are only processed when explicitly declared ([#6200](https://github.com/graphprotocol/graph-node/pull/6200)) + +### New Features + +- Added support for struct field access in declarative calls — enables accessing nested struct fields by field name using dot notation in manifest call declarations ([#6099](https://github.com/graphprotocol/graph-node/pull/6099)) +- New standalone `gnd` (Graph Node Dev) crate — a graph-node binary optimized for local development with minimal setup required ([#6167](https://github.com/graphprotocol/graph-node/pull/6167)) +- Extended IPFS usage metrics and logging capabilities ([#6058](https://github.com/graphprotocol/graph-node/pull/6058)) +- Extended support for additional IPFS content path formats ([#6058](https://github.com/graphprotocol/graph-node/pull/6058)) + +### Improvements + +- Deferred IPFS manifest fetching when starting subgraphs — fixes issue where slow IPFS responses could block assignment event processing ([#6170](https://github.com/graphprotocol/graph-node/pull/6170)) +- Added Content Identifier (CID) to IPFS retry logs for better debugging ([#6144](https://github.com/graphprotocol/graph-node/pull/6144)) +- Fixed handling of empty arrays in indexing status queries ([#6143](https://github.com/graphprotocol/graph-node/pull/6143)) +- Fixed case-insensitive array inputs in queries ([#6075](https://github.com/graphprotocol/graph-node/pull/6075)) +- Fixed panic when handling empty lists in `list_values` ([#6100](https://github.com/graphprotocol/graph-node/pull/6100)) +- Enhanced logging for subgraph assignment processing ([#6169](https://github.com/graphprotocol/graph-node/pull/6169)) +- Added logging for `MAX_BLOCKING_THREADS` configuration setting ([#6161](https://github.com/graphprotocol/graph-node/pull/6161)) + +**Full changelog:** + +## v0.40.1 + +### Improvements + +- Enhanced IPFS logging to include Content Identifiers (CIDs) in operation names for better debugging + +**Full changelog:** + +## v0.40.0 + +### Critical Bugfix + +- Fixed GraphQL query panic with empty arrays: Resolved a critical bug where GraphQL queries using `_in` filters with empty arrays would cause the graph-node to panic ([#6100](https://github.com/graphprotocol/graph-node/pull/6100)) + +### New Features + +- IPFS files can now be cached to disk using `GRAPH_IPFS_CACHE_LOCATION` — reduces IPFS requests after restarts ([#6031](https://github.com/graphprotocol/graph-node/pull/6031)) + +### Improvements + +- Speed up appending changes to batch operations ([#6025](https://github.com/graphprotocol/graph-node/pull/6025)) +- Improved backoff strategy for offchain data sources, configurable via `GRAPH_FDS_MAX_BACKOFF` ([#6043](https://github.com/graphprotocol/graph-node/pull/6043)) +- Better error messages for OR operator usage with column filters ([#6078](https://github.com/graphprotocol/graph-node/pull/6078)) +- Update Wasmtime version for improved WebAssembly performance ([#6050](https://github.com/graphprotocol/graph-node/pull/6050)) + +### Bug Fixes + +- Fixed subgraph composition sync failures with different ID types ([#6080](https://github.com/graphprotocol/graph-node/pull/6080)) +- Fixed pruning status reporting accuracy ([#6062](https://github.com/graphprotocol/graph-node/pull/6062)) + +**Full changelog:** + +## v0.39.1 + +### Critical Fix + +- Reverted `event.transactionLogIndex` behavior change from v0.38.0 that caused subgraph failures with duplicate ID errors and POI divergence between indexers ([#6042](https://github.com/graphprotocol/graph-node/pull/6042)) + +**Full changelog:** + +## v0.39.0 + +### Breaking Changes + +- **Database schema migration**: `subgraphs.subgraph_deployment` table split into `subgraphs.head` and `subgraphs.deployment`. External tools accessing this table will need updates. See [documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/implementation/metadata.md#subgraphshead) ([#6003](https://github.com/graphprotocol/graph-node/pull/6003)) +- **Arweave blockchain support removed** — Arweave subgraphs are no longer supported (file data sources remain unaffected) ([#5951](https://github.com/graphprotocol/graph-node/pull/5951)) +- **`graphman drop` removed** — Use `graphman remove` followed by `graphman unused record && graphman unused remove` ([#5974](https://github.com/graphprotocol/graph-node/pull/5974)) +- **Failed subgraphs are now paused instead of unassigned** ([#5971](https://github.com/graphprotocol/graph-node/pull/5971)) + +### New Features + +- Pruning status tracking with new graphman commands: `prune status`, `prune run`, `prune set` ([#5949](https://github.com/graphprotocol/graph-node/pull/5949)) +- Pruning timeout protection via `GRAPH_STORE_BATCH_TIMEOUT` ([#6002](https://github.com/graphprotocol/graph-node/pull/6002)) +- `graphman chain ingest` for manually ingesting specific blocks ([#5945](https://github.com/graphprotocol/graph-node/pull/5945)) +- Configurable IPFS retry limits via `GRAPH_IPFS_MAX_ATTEMPTS` and `GRAPH_IPFS_REQUEST_TIMEOUT` ([#5998](https://github.com/graphprotocol/graph-node/pull/5998)) + +### Bug Fixes + +- Fixed VID sequence naming to comply with PostgreSQL 63-character limit +- Fixed pruning of tables with VID sequences using CASCADE drops ([#5968](https://github.com/graphprotocol/graph-node/pull/5968)) +- Fixed numerical precision issues with large VID values ([#5970](https://github.com/graphprotocol/graph-node/pull/5970)) + +**Full changelog:** + +## v0.38.0 + +### What's new + +- A new `deployment_synced` metric is added [(#5816)](https://github.com/graphprotocol/graph-node/pull/5816) + that indicates whether a deployment has reached the chain head since it was deployed. + + **Possible values for the metric:** + - `0` - means that the deployment is not synced; + - `1` - means that the deployment is synced; + + _If a deployment is not running, the metric reports no value for that deployment._ + +## v0.37.0 + +### What's new + +- A new `deployment_status` metric is added [(#5720)](https://github.com/graphprotocol/graph-node/pull/5720) with the + following behavior: + - Once graph-node has figured out that it should index a deployment, `deployment_status` is set to `1` _(starting)_; + - When the block stream is created and blocks are ready to be processed, `deployment_status` is set to `2` _( + running)_; + - When a deployment is unassigned, `deployment_status` is set to `3` _(stopped)_; + - If a temporary or permanent failure occurs, `deployment_status` is set to `4` _(failed)_; + - If indexing manages to recover from a temporary failure, the `deployment_status` is set back to `2` _( + running)_; + +### Breaking changes + +- The `deployment_failed` metric is removed and the failures are reported by the new `deployment_status` + metric. [(#5720)](https://github.com/graphprotocol/graph-node/pull/5720) + +## v0.36.0 + +### Note on Firehose Extended Block Details + +By default, all Firehose providers are required to support extended block details, as this is the +safest option for a graph-node operator. Firehose providers that do not support extended block +details for enabled chains are considered invalid and will not be used. + +To disable checks for one or more chains, simply specify their names +in `GRAPH_NODE_FIREHOSE_DISABLE_EXTENDED_BLOCKS_FOR_CHAINS` as a comma separated list of chain +names. Graph Node defaults to an empty list, which means that this feature is enabled for all +chains. + +### What's new + +- Add support for substreams using 'index modules', 'block filters', 'store:sum_set'. [(#5463)](https://github.com/graphprotocol/graph-node/pull/5463) +- Implement new IPFS client [(#5600)](https://github.com/graphprotocol/graph-node/pull/5600) +- Add `timestamp` support to substreams. [(#5641)](https://github.com/graphprotocol/graph-node/pull/5641) +- Add graph-indexed header to query responses. [(#5710)](https://github.com/graphprotocol/graph-node/pull/5710) +- Use the new Firehose info endpoint. [(#5672)](https://github.com/graphprotocol/graph-node/pull/5672) +- Store `synced_at_block_number` when a deployment syncs. [(#5610)](https://github.com/graphprotocol/graph-node/pull/5610) +- Create nightly docker builds from master branch. [(#5400)](https://github.com/graphprotocol/graph-node/pull/5400) +- Make sure `transact_block_operations` does not go backwards. [(#5419)](https://github.com/graphprotocol/graph-node/pull/5419) +- Improve error message when store write fails. [(#5420)](https://github.com/graphprotocol/graph-node/pull/5420) +- Allow generating map of section nesting in debug builds. [(#5279)](https://github.com/graphprotocol/graph-node/pull/5279) +- Ensure substream module name is valid. [(#5424)](https://github.com/graphprotocol/graph-node/pull/5424) +- Improve error message when resolving references. [(#5385)](https://github.com/graphprotocol/graph-node/pull/5385) +- Check if subgraph head exists before trying to unfail. [(#5409)](https://github.com/graphprotocol/graph-node/pull/5409) +- Check for EIP 1898 support when checking block receipts support. [(#5406)](https://github.com/graphprotocol/graph-node/pull/5406) +- Use latest block hash for `check_block_receipts`. [(#5427)](https://github.com/graphprotocol/graph-node/pull/5427) +- Handle null blocks from Lotus. [(#5294)](https://github.com/graphprotocol/graph-node/pull/5294) +- Increase firehose grpc max decode size. [(#5483)](https://github.com/graphprotocol/graph-node/pull/5483) +- Improve Environment variable docs, rename `GRAPH_ETHEREUM_BLOCK_RECEIPTS_TIMEOUT` to `GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT`. [(#5468)](https://github.com/graphprotocol/graph-node/pull/5468) +- Remove provider checks at startup. [(#5337)](https://github.com/graphprotocol/graph-node/pull/5337) +- Track more features in subgraph features table. [(#5479)](https://github.com/graphprotocol/graph-node/pull/5479) +- Implement is_duplicate_of for substreams. [(#5482)](https://github.com/graphprotocol/graph-node/pull/5482) +- Add docs for `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION`. [(#5515)](https://github.com/graphprotocol/graph-node/pull/5515) +- Improve error message for missing template during grafting. [(#5464)](https://github.com/graphprotocol/graph-node/pull/5464) +- Enable "hard-coded" values in declarative eth_calls. [(#5498)](https://github.com/graphprotocol/graph-node/pull/5498) +- Respect causality region in derived fields. [(#5488)](https://github.com/graphprotocol/graph-node/pull/5488) +- Improve net_identifiers call with timeout. [(#5549)](https://github.com/graphprotocol/graph-node/pull/5549) +- Add arbitrum-sepolia chain ID to GRAPH_ETH_CALL_NO_GAS default value. [(#5504)](https://github.com/graphprotocol/graph-node/pull/5504) +- Disable genesis validation by default. [(#5565)](https://github.com/graphprotocol/graph-node/pull/5565) +- Timeout when trying to get `net_identifiers` at startup. [(#5568)](https://github.com/graphprotocol/graph-node/pull/5568) +- Only start substreams if no other block investor is available. [(#5569)](https://github.com/graphprotocol/graph-node/pull/5569) +- Allow running a single test case for integration tests. [(#5577)](https://github.com/graphprotocol/graph-node/pull/5577) +- Store timestamp when marking subgraph as synced. [(#5566)](https://github.com/graphprotocol/graph-node/pull/5566) +- Document missing env vars. [(#5580)](https://github.com/graphprotocol/graph-node/pull/5580) +- Return more features in status API. [(#5582)](https://github.com/graphprotocol/graph-node/pull/5582) +- Respect substreams datasource `startBlock`. [(#5617)](https://github.com/graphprotocol/graph-node/pull/5617) +- Update flagged dependencies. [(#5659)](https://github.com/graphprotocol/graph-node/pull/5659) +- Add more debug logs when subgraph is marked unhealthy. [(#5662)](https://github.com/graphprotocol/graph-node/pull/5662) +- Add config option for cache stores. [(#5716)](https://github.com/graphprotocol/graph-node/pull/5716) + +### Bug fixes + +- Add safety check when rewinding. [(#5423)](https://github.com/graphprotocol/graph-node/pull/5423) +- Fix rewind for deployments with multiple names. [(#5502)](https://github.com/graphprotocol/graph-node/pull/5502) +- Improve `graphman copy` performance [(#5425)](https://github.com/graphprotocol/graph-node/pull/5425) +- Fix retrieving chain info with graphman for some edge cases. [(#5516)](https://github.com/graphprotocol/graph-node/pull/5516) +- Improve `graphman restart` to handle multiple subgraph names for a deployment. [(#5674)](https://github.com/graphprotocol/graph-node/pull/5674) +- Improve adapter startup. [(#5503)](https://github.com/graphprotocol/graph-node/pull/5503) +- Detect Nethermind eth_call reverts. [(#5533)](https://github.com/graphprotocol/graph-node/pull/5533) +- Fix genesis block fetching for substreams. [(#5548)](https://github.com/graphprotocol/graph-node/pull/5548) +- Fix subgraph_resume being mislabelled as pause. [(#5588)](https://github.com/graphprotocol/graph-node/pull/5588) +- Make `SubgraphIndexingStatus.paused` nullable. [(#5551)](https://github.com/graphprotocol/graph-node/pull/5551) +- Fix a count aggregation bug. [(#5639)](https://github.com/graphprotocol/graph-node/pull/5639) +- Fix prost generated file. [(#5450)](https://github.com/graphprotocol/graph-node/pull/5450) +- Fix `deployment_head` metrics not progressing for substreams. [(#5522)](https://github.com/graphprotocol/graph-node/pull/5522) +- Enable graft validation checks in debug builds. [(#5584)](https://github.com/graphprotocol/graph-node/pull/5584) +- Use correct store when loading indexes for graft base. [(#5616)](https://github.com/graphprotocol/graph-node/pull/5616) +- Sanitise columns in SQL. [(#5578)](https://github.com/graphprotocol/graph-node/pull/5578) +- Truncate `subgraph_features` table before migrating. [(#5505)](https://github.com/graphprotocol/graph-node/pull/5505) +- Consistently apply max decode size. [(#5520)](https://github.com/graphprotocol/graph-node/pull/5520) +- Various docker packaging improvements [(#5709)](https://github.com/graphprotocol/graph-node/pull/5709) [(#5711)](https://github.com/graphprotocol/graph-node/pull/5711) [(#5712)](https://github.com/graphprotocol/graph-node/pull/5712) [(#5620)](https://github.com/graphprotocol/graph-node/pull/5620) [(#5621)](https://github.com/graphprotocol/graph-node/pull/5621) +- Retry IPFS requests on Cloudflare 521 Web Server Down. [(#5687)](https://github.com/graphprotocol/graph-node/pull/5687) +- Optimize IPFS retries. [(#5698)](https://github.com/graphprotocol/graph-node/pull/5698) +- Exclude full-text search columns from entity queries. [(#5693)](https://github.com/graphprotocol/graph-node/pull/5693) +- Do not allow multiple active runners for a subgraph. [(#5715)](https://github.com/graphprotocol/graph-node/pull/5715) +- Stop subgraphs passing max endBlock. [(#5583)](https://github.com/graphprotocol/graph-node/pull/5583) +- Do not repeat a rollup after restart in some corner cases. [(#5675)](https://github.com/graphprotocol/graph-node/pull/5675) + +### Graphman + +- Add command to update genesis block for a chain and to check genesis information against all providers. [(#5517)](https://github.com/graphprotocol/graph-node/pull/5517) +- Create GraphQL API to execute commands [(#5554)](https://github.com/graphprotocol/graph-node/pull/5554) +- Add graphman create/remove commands to GraphQL API. [(#5685)](https://github.com/graphprotocol/graph-node/pull/5685) + +### Contributors + +Thanks to all contributors for this release: @dwerner, @encalypto, @incrypto32, @isum, @leoyvens, @lutter, @mangas, @sduchesneau, @Shiyasmohd, @shuaibbapputty, @YaroShkvorets, @ziyadonji, @zorancv + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.35.1...v0.36.0 + +## v0.35.0 +### What's new + +- **Aggregations** - Declarative aggregations defined in the subgraph schema allow the developer to aggregate values on specific intervals using flexible aggregation functions. [(#5082)](https://github.com/graphprotocol/graph-node/pull/5082) [(#5184)](https://github.com/graphprotocol/graph-node/pull/5184) [(#5209)](https://github.com/graphprotocol/graph-node/pull/5209) [(#5242)](https://github.com/graphprotocol/graph-node/pull/5242) [(#5208)](https://github.com/graphprotocol/graph-node/pull/5208) +- **Add pause and resume to admin JSON-RPC API** - Adds support for explicit pausing and resuming of subgraph deployments with a field tracking the paused state in `indexerStatuses`. [(#5190)](https://github.com/graphprotocol/graph-node/pull/5190) +- **Support eth_getBalance calls in subgraph mappings** - Enables fetching the Eth balance of an address from the mappings using `ethereum.getBalance(address)`. [(#5202)](https://github.com/graphprotocol/graph-node/pull/5202) +- **Add parentHash to _meta query** - Particularly useful when polling for data each block to verify the sequence of blocks. [(#5232)](https://github.com/graphprotocol/graph-node/pull/5232) +- **Parallel execution of all top-level queries in a single query body** [(#5273)](https://github.com/graphprotocol/graph-node/pull/5273) +- The ElasticSearch index to which `graph-node` logs can now be configured with the `GRAPH_ELASTIC_SEARCH_INDEX` environment variable which defaults to `subgraph`. [(#5210)](https://github.com/graphprotocol/graph-node/pull/5210) +- Some small prefetch simplifications. [(#5132)](https://github.com/graphprotocol/graph-node/pull/5132) +- Migration changing the type of health column to text. [(#5077)](https://github.com/graphprotocol/graph-node/pull/5077) +- Disable eth_call_execution_time metric by default. [(#5164)](https://github.com/graphprotocol/graph-node/pull/5164) +- Call revert_state_to whenever blockstream is restarted. [(#5187)](https://github.com/graphprotocol/graph-node/pull/5187) +- Pruning performance improvement: only analyze when rebuilding. [(#5186)](https://github.com/graphprotocol/graph-node/pull/5186) +- Disallow grafts within the reorg threshold. [(#5135)](https://github.com/graphprotocol/graph-node/pull/5135) +- Optimize subgraph synced check-less. [(#5198)](https://github.com/graphprotocol/graph-node/pull/5198) +- Improve error log. [(#5217)](https://github.com/graphprotocol/graph-node/pull/5217) +- Update provider docs. [(#5216)](https://github.com/graphprotocol/graph-node/pull/5216) +- Downgrade 'Entity cache statistics' log to trace. [(#5241)](https://github.com/graphprotocol/graph-node/pull/5241) +- Do not clone MappingEventHandlers in match_and_decode. [(#5244)](https://github.com/graphprotocol/graph-node/pull/5244) +- Make batching conditional on caught-up status. [(#5252)](https://github.com/graphprotocol/graph-node/pull/5252) +- Remove hack in chain_head_listener. [(#5240)](https://github.com/graphprotocol/graph-node/pull/5240) +- Increase sleep time in write queue processing. [(#5266)](https://github.com/graphprotocol/graph-node/pull/5266) +- Memoize Batch.indirect_weight. [(#5276)](https://github.com/graphprotocol/graph-node/pull/5276) +- Optionally track detailed indexing gas metrics in csv. [(#5215)](https://github.com/graphprotocol/graph-node/pull/5215) +- store: Do not use prefix comparisons for primary keys. [(#5289)](https://github.com/graphprotocol/graph-node/pull/5289) + +### Graphman + +- Add ability to list removed unused deployment by id. [(#5152)](https://github.com/graphprotocol/graph-node/pull/5152) +- Add command to change block cache shard. [(#5169)](https://github.com/graphprotocol/graph-node/pull/5169) + +### Firehose and Substreams + +- **Add key-based authentication for Firehose/Substreams providers.** [(#5259)](https://github.com/graphprotocol/graph-node/pull/5259) +- Increase blockstream buffer size for substreams. [(#5182)](https://github.com/graphprotocol/graph-node/pull/5182) +- Improve substreams error handling. [(#5160)](https://github.com/graphprotocol/graph-node/pull/5160) +- Reset substreams/firehose block ingestor backoff. [(#5047)](https://github.com/graphprotocol/graph-node/pull/5047) + +### Bug Fixes + +- Fix graphiql issue when querying subgraph names with multiple path segments. [(#5136)](https://github.com/graphprotocol/graph-node/pull/5136) +- Fix change_health_column migration for sharded setup. [(#5183)](https://github.com/graphprotocol/graph-node/pull/5183) +- Fix conversion of BlockTime for NEAR. [(#5206)](https://github.com/graphprotocol/graph-node/pull/5206) +- Call revert_state_to to last good block instead of current block. [(#5195)](https://github.com/graphprotocol/graph-node/pull/5195) +- Fix Action::block_finished. [(#5218)](https://github.com/graphprotocol/graph-node/pull/5218) +- Fix runtime timeouts. [(#5236)](https://github.com/graphprotocol/graph-node/pull/5236) +- Remove panic from rewind and truncate. [(#5233)](https://github.com/graphprotocol/graph-node/pull/5233) +- Fix version stats for huge number of versions. [(#5261)](https://github.com/graphprotocol/graph-node/pull/5261) +- Fix _meta query failure due to incorrect selection set use. [(#5265)](https://github.com/graphprotocol/graph-node/pull/5265) + +### Major dependency upgrades + +- Update to diesel 2. [(#5002)](https://github.com/graphprotocol/graph-node/pull/5002) +- bump rust version. [(#4985)](https://github.com/graphprotocol/graph-node/pull/4985) + +### Contributors + +Thank you to all the contributors! `@incrypto32`, `@mangas`, `@lutter`, `@leoyvens`, `@zorancv`, `@YaroShkvorets`, `@seem-less` + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.34.1...v0.35.0 + + + +## v0.34.1 +## Bug fixes +- Fixed an issue that caused an increase in data size of /metrics endpoint of graph-node. [(#5161)](https://github.com/graphprotocol/graph-node/issues/5161) +- Fixed an issue that caused subgraphs with file data sources to skip non-deterministic errors that occurred in a file data source mapping handler. ## v0.34.0 ### What's New @@ -23,7 +540,7 @@ - Addressed a bug in the deduplication logic for Cosmos events, ensuring all distinct events are properly indexed and handled, especially when similar but not identical events occur within the same block. [(#5112)](https://github.com/graphprotocol/graph-node/pull/5112) - Fixed compatibility issues with ElasticSearch 8.X, ensuring proper log functionality. [(#5013)](https://github.com/graphprotocol/graph-node/pull/5013) -- Resolved an issue when rewinding data sources across multiple blocks. In rare cases, when a subgraph had been rewound by multiple blocks, data sources 'from the future' could have been left behind. This release adds a database migration that fixes that. With very unlucky timing this migration might miss some subgraphs, which will later lead to an error `assertion failed: self.hosts.last().and_then(|h| h.creation_block_number()) <= data_source.creation_block()`. Should that happen, the [migration script](https://github.com/graphprotocol/graph-node/blob/master/store/postgres/migrations/2024-01-05-170000_ds_corruption_fix_up/up.sql) should be rerun against the affected shard. [(#5083)](https://github.com/graphprotocol/graph-node/pull/5083) + - Resolved an issue when rewinding data sources across multiple blocks. In rare cases, when a subgraph had been rewound by multiple blocks, data sources 'from the future' could have been left behind. This release adds a database migration that fixes that. With very unlucky timing this migration might miss some subgraphs, which will later lead to an error `assertion failed: self.hosts.last().and_then(|h| h.creation_block_number()) <= data_source.creation_block()`. Should that happen, the [migration script](https://github.com/graphprotocol/graph-node/blob/master/store/postgres/migrations/2024-01-05-170000_ds_corruption_fix_up/up.sql) should be rerun against the affected shard. [(#5083)](https://github.com/graphprotocol/graph-node/pull/5083) - Increased the base backoff time for RPC, enhancing stability and reliability under load. [(#4984)](https://github.com/graphprotocol/graph-node/pull/4984) - Resolved an issue related to spawning offchain data sources from existing offchain data source mappings. [(#5051)](https://github.com/graphprotocol/graph-node/pull/5051)[(#5092)](https://github.com/graphprotocol/graph-node/pull/5092) - Resolved an issue where eth-call results for reverted calls were being cached in call cache. [(#4879)](https://github.com/graphprotocol/graph-node/pull/4879) @@ -244,8 +761,8 @@ Not Relevant @@ -953,7 +1470,7 @@ storage](./docs/config.md) and spread subgraph deployments, and the load coming from indexing and querying them across multiple independent Postgres databases. -**This feature is considered experimenatal. We encourage users to try this +**This feature is considered experimental. We encourage users to try this out in a test environment, but do not recommend it yet for production use** In particular, the details of how sharding is configured may change in backwards-incompatible ways in the future. diff --git a/README.md b/README.md index ff31fdad758..667c606a045 100644 --- a/README.md +++ b/README.md @@ -1,195 +1,157 @@ # Graph Node [![Build Status](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml/badge.svg)](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml?query=branch%3Amaster) -[![Getting Started Docs](https://img.shields.io/badge/docs-getting--started-brightgreen.svg)](docs/getting-started.md) +[![Docs](https://img.shields.io/badge/docs-graph--node-green.svg)](docs/) +[![Subgraphs](https://img.shields.io/badge/docs-subgraphs-green.svg)](https://thegraph.com/docs/en/subgraphs/quick-start/) -[The Graph](https://thegraph.com/) is a protocol for building decentralized applications (dApps) quickly on Ethereum and IPFS using GraphQL. +## Overview -Graph Node is an open source Rust implementation that event sources the Ethereum blockchain to deterministically update a data store that can be queried via the GraphQL endpoint. +[The Graph](https://thegraph.com/) is a decentralized protocol that organizes and distributes blockchain data across the leading Web3 networks. A key component of The Graph's tech stack is Graph Node. -For detailed instructions and more context, check out the [Getting Started Guide](docs/getting-started.md). +Before using `graph-node,` it is highly recommended that you read the [official Graph documentation](https://thegraph.com/docs/en/subgraphs/quick-start/) to understand Subgraphs, which are the central mechanism for extracting and organizing blockchain data. -## Quick Start +This guide is for: -### Prerequisites - -To build and run this project you need to have the following installed on your system: - -- Rust (latest stable) – [How to install Rust](https://www.rust-lang.org/en-US/install.html) - - Note that `rustfmt`, which is part of the default Rust installation, is a build-time requirement. -- PostgreSQL – [PostgreSQL Downloads](https://www.postgresql.org/download/) -- IPFS – [Installing IPFS](https://docs.ipfs.io/install/) -- Profobuf Compiler - [Installing Protobuf](https://grpc.io/docs/protoc-installation/) - -For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. +1. Subgraph developers who want to run `graph-node` locally to test their Subgraphs during development +2. Contributors who want to add features or fix bugs to `graph-node` itself -**Minimum Hardware Requirements:** +## Running `graph-node` from Docker images -- To build graph-node with `cargo`, 8GB RAM are required. +For subgraph developers, it is highly recommended to use prebuilt Docker +images to set up a local `graph-node` environment. Please read [these +instructions](./docker/README.md) to learn how to do that. -### Docker +## Running `graph-node` from source -The easiest way to run a Graph Node is to use the official Docker compose setup. This will start a Postgres database, IPFS node, and Graph Node. -[Follow the instructions here](./docker/README.md). +This is usually only needed for developers who want to contribute to `graph-node`. -### Running a Local Graph Node +### Prerequisites -This is a quick example to show a working Graph Node. It is a [subgraph for Gravatars](https://github.com/graphprotocol/example-subgraph). +To build and run this project, you need to have the following installed on your system: -1. Install IPFS and run `ipfs init` followed by `ipfs daemon`. -2. Install PostgreSQL and run `initdb -D .postgres -E UTF8 --locale=C` followed by `pg_ctl -D .postgres -l logfile start` and `createdb graph-node`. -3. If using Ubuntu, you may need to install additional packages: - - `sudo apt-get install -y clang libpq-dev libssl-dev pkg-config` -4. In the terminal, clone https://github.com/graphprotocol/example-subgraph, and install dependencies and generate types for contract ABIs: +- Rust (latest stable): Follow [How to install + Rust](https://rust-lang.org/tools/install/). Run `rustup install +stable` in _this directory_ to make sure all required components are + installed. The `graph-node` code assumes that the latest available + `stable` compiler is used. +- PostgreSQL: [PostgreSQL Downloads](https://www.postgresql.org/download/) lists + downloads for almost all operating systems. + - For OSX: We highly recommend [Postgres.app](https://postgresapp.com/). + - For Linux: Use the Postgres version that comes with the distribution. +- IPFS: [Installing IPFS](https://docs.ipfs.io/install/) +- Protobuf Compiler: [Installing Protobuf](https://grpc.io/docs/protoc-installation/) -``` -yarn -yarn codegen -``` - -5. In the terminal, clone https://github.com/graphprotocol/graph-node, and run `cargo build`. +For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. -Once you have all the dependencies set up, you can run the following: +### Create a database -``` -cargo run -p graph-node --release -- \ - --postgres-url postgresql://USERNAME[:PASSWORD]@localhost:5432/graph-node \ - --ethereum-rpc NETWORK_NAME:[CAPABILITIES]:URL \ - --ipfs 127.0.0.1:5001 -``` +Once Postgres is running, you need to issue the following commands to create a database +and configure it for use with `graph-node`. -Try your OS username as `USERNAME` and `PASSWORD`. For details on setting -the connection string, check the [Postgres -documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). -`graph-node` uses a few Postgres extensions. If the Postgres user with which -you run `graph-node` is a superuser, `graph-node` will enable these -extensions when it initializes the database. If the Postgres user is not a -superuser, you will need to create the extensions manually since only -superusers are allowed to do that. To create them you need to connect as a -superuser, which in many installations is the `postgres` user: +The name of the `SUPERUSER` depends on your installation, but is usually `postgres` or your username. ```bash - psql -q -X -U graph-node < <'; +create database "graph-node" with owner=graph template=template0 encoding='UTF8' locale='C'; create extension pg_trgm; -create extension pg_stat_statements; create extension btree_gist; create extension postgres_fdw; -grant usage on foreign data wrapper postgres_fdw to ; +grant usage on foreign data wrapper postgres_fdw to graph; EOF - ``` -This will also spin up a GraphiQL interface at `http://127.0.0.1:8000/`. - -6. With this Gravatar example, to get the subgraph working locally run: +For convenience, set the connection string to the database in an environment +variable, and save it, e.g., in `~/.bashrc`: -``` -yarn create-local +```bash +export POSTGRES_URL=postgresql://graph:@localhost:5432/graph-node ``` -Then you can deploy the subgraph: +Use the `POSTGRES_URL` from above to have `graph-node` connect to the +database. If you ever need to manually inspect the contents of your +database, you can do that by running `psql $POSTGRES_URL`. Running this +command is also a convenient way to check that the database is up and +running and that the connection string is correct. -``` -yarn deploy-local +### Build and Run `graph-node` + +Clone this repository and run this command at the root of the repository: + +```bash +export GRAPH_LOG=debug +cargo run -p graph-node --release -- \ + --postgres-url $POSTGRES_URL \ + --ethereum-rpc NETWORK_NAME:[CAPABILITIES]:URL \ + --ipfs 127.0.0.1:5001 ``` -This will build and deploy the subgraph to the Graph Node. It should start indexing the subgraph immediately. +The argument for `--ethereum-rpc` contains a network name (e.g. `mainnet`) and +a list of provider capabilities (e.g. `archive,traces`). The URL is the address +of the Ethereum node you want to connect to, usually a `https` URL, so that the +entire argument might be `mainnet:archive,traces:https://provider.io/some/path`. -### Command-Line Interface +When `graph-node` starts, it prints the various ports that it is listening on. +The most important of these is the GraphQL HTTP server, which by default +is at `http://localhost:8000`. You can use routes like `/subgraphs/name/` +and `/subgraphs/id/` to query subgraphs once you have deployed them. -``` -USAGE: - graph-node [FLAGS] [OPTIONS] --ethereum-ipc --ethereum-rpc --ethereum-ws --ipfs --postgres-url +### Deploying a Subgraph -FLAGS: - --debug Enable debug logging - -h, --help Prints help information - -V, --version Prints version information +Follow the [Subgraph deployment +guide](https://thegraph.com/docs/en/subgraphs/developing/introduction/). +After setting up `graph-cli` as described, you can deploy a Subgraph to your +local Graph Node instance. -OPTIONS: - --admin-port Port for the JSON-RPC admin server [default: 8020] - --elasticsearch-password - Password to use for Elasticsearch logging [env: ELASTICSEARCH_PASSWORD] +### Advanced Configuration - --elasticsearch-url - Elasticsearch service to write subgraph logs to [env: ELASTICSEARCH_URL=] +The command line arguments generally are all that is needed to run a +`graph-node` instance. For advanced uses, various aspects of `graph-node` +can further be configured through [environment +variables](https://github.com/graphprotocol/graph-node/blob/master/docs/environment-variables.md). - --elasticsearch-user User to use for Elasticsearch logging [env: ELASTICSEARCH_USER=] - --ethereum-ipc - Ethereum network name (e.g. 'mainnet'), optional comma-separated capabilities (eg full,archive), and an Ethereum IPC pipe, separated by a ':' +Very large `graph-node` instances can also be configured using a +[configuration file](./docs/config.md) That is usually only necessary when +the `graph-node` needs to connect to multiple chains or if the work of +indexing and querying needs to be split across [multiple databases](./docs/config.md). - --ethereum-polling-interval - How often to poll the Ethereum node for new blocks [env: ETHEREUM_POLLING_INTERVAL=] [default: 500] +#### Log Storage - --ethereum-rpc - Ethereum network name (e.g. 'mainnet'), optional comma-separated capabilities (eg 'full,archive'), and an Ethereum RPC URL, separated by a ':' +`graph-node` supports storing and querying subgraph logs through multiple backends: - --ethereum-ws - Ethereum network name (e.g. 'mainnet'), optional comma-separated capabilities (eg `full,archive), and an Ethereum WebSocket URL, separated by a ':' +- **File**: Local JSON Lines files (recommended for local development) +- **Elasticsearch**: Enterprise-grade search and analytics (for production) +- **Loki**: Grafana's log aggregation system (for production) +- **Disabled**: No log storage (default) - --node-id - A unique identifier for this node instance. Should have the same value between consecutive node restarts [default: default] +**Quick example (file-based logs for local development):** +```bash +mkdir -p ./graph-logs - --http-port Port for the GraphQL HTTP server [default: 8000] - --ipfs HTTP address of an IPFS node - --postgres-url Location of the Postgres database used for storing entities - --subgraph <[NAME:]IPFS_HASH> Name and IPFS hash of the subgraph manifest - --ws-port Port for the GraphQL WebSocket server [default: 8001] +cargo run -p graph-node --release -- \ + --postgres-url $POSTGRES_URL \ + --ethereum-rpc mainnet:archive:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend file \ + --log-store-file-dir ./graph-logs ``` -### Advanced Configuration - -The command line arguments generally are all that is needed to run a -`graph-node` instance. For advanced uses, various aspects of `graph-node` -can further be configured through [environment -variables](https://github.com/graphprotocol/graph-node/blob/master/docs/environment-variables.md). Very -large `graph-node` instances can also split the work of querying and -indexing across [multiple databases](./docs/config.md). - -## Project Layout - -- `node` — A local Graph Node. -- `graph` — A library providing traits for system components and types for - common data. -- `core` — A library providing implementations for core components, used by all - nodes. -- `chain/ethereum` — A library with components for obtaining data from - Ethereum. -- `graphql` — A GraphQL implementation with API schema generation, - introspection, and more. -- `mock` — A library providing mock implementations for all system components. -- `runtime/wasm` — A library for running WASM data-extraction scripts. -- `server/http` — A library providing a GraphQL server over HTTP. -- `store/postgres` — A Postgres store with a GraphQL-friendly interface - and audit logs. - -## Roadmap - -🔨 = In Progress - -🛠 = Feature complete. Additional testing required. - -✅ = Feature complete - - -| Feature | Status | -| ------- | :------: | -| **Ethereum** | | -| Indexing smart contract events | ✅ | -| Handle chain reorganizations | ✅ | -| **Mappings** | | -| WASM-based mappings| ✅ | -| TypeScript-to-WASM toolchain | ✅ | -| Autogenerated TypeScript types | ✅ | -| **GraphQL** | | -| Query entities by ID | ✅ | -| Query entity collections | ✅ | -| Pagination | ✅ | -| Filtering | ✅ | -| Block-based Filtering | ✅ | -| Entity relationships | ✅ | -| Subscriptions | ✅ | +Logs are queried via GraphQL at `http://localhost:8000/graphql`: +```graphql +query { + _logs(subgraphId: "QmYourSubgraphHash", level: ERROR, first: 10) { + timestamp + level + text + } +} +``` +**For complete documentation**, see the **[Log Store Guide](./docs/log-store.md)**, which covers: +- How to configure each backend (Elasticsearch, Loki, File) +- Complete GraphQL query examples +- Choosing the right backend for your use case +- Performance considerations and best practices ## Contributing diff --git a/chain/arweave/.gitignore b/chain/arweave/.gitignore deleted file mode 100644 index 97442b5f148..00000000000 --- a/chain/arweave/.gitignore +++ /dev/null @@ -1 +0,0 @@ -google.protobuf.rs \ No newline at end of file diff --git a/chain/arweave/Cargo.toml b/chain/arweave/Cargo.toml deleted file mode 100644 index f8b20147af9..00000000000 --- a/chain/arweave/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "graph-chain-arweave" -version.workspace = true -edition.workspace = true - -[build-dependencies] -tonic-build = { workspace = true } - -[dependencies] -base64-url = "2.0.2" -graph = { path = "../../graph" } -prost = { workspace = true } -prost-types = { workspace = true } -serde = "1.0" -sha2 = "0.10.8" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } - -[dev-dependencies] -diesel = { version = "1.4.7", features = ["postgres", "serde_json", "numeric", "r2d2"] } diff --git a/chain/arweave/build.rs b/chain/arweave/build.rs deleted file mode 100644 index ea8153e7bd1..00000000000 --- a/chain/arweave/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .out_dir("src/protobuf") - .compile(&["proto/arweave.proto"], &["proto"]) - .expect("Failed to compile Firehose Arweave proto(s)"); -} diff --git a/chain/arweave/proto/arweave.proto b/chain/arweave/proto/arweave.proto deleted file mode 100644 index b3a41a4a56a..00000000000 --- a/chain/arweave/proto/arweave.proto +++ /dev/null @@ -1,108 +0,0 @@ -syntax = "proto3"; - -package sf.arweave.type.v1; - -option go_package = "github.com/ChainSafe/firehose-arweave/pb/sf/arweave/type/v1;pbcodec"; - -message BigInt { - bytes bytes = 1; -} - -message Block { - // Firehose block version (unrelated to Arweave block version) - uint32 ver = 1; - // The block identifier - bytes indep_hash = 2; - // The nonce chosen to solve the mining problem - bytes nonce = 3; - // `indep_hash` of the previous block in the weave - bytes previous_block = 4; - // POSIX time of block discovery - uint64 timestamp = 5; - // POSIX time of the last difficulty retarget - uint64 last_retarget = 6; - // Mining difficulty; the number `hash` must be greater than. - BigInt diff = 7; - // How many blocks have passed since the genesis block - uint64 height = 8; - // Mining solution hash of the block; must satisfy the mining difficulty - bytes hash = 9; - // Merkle root of the tree of Merkle roots of block's transactions' data. - bytes tx_root = 10; - // Transactions contained within this block - repeated Transaction txs = 11; - // The root hash of the Merkle Patricia Tree containing - // all wallet (account) balances and the identifiers - // of the last transactions posted by them; if any. - bytes wallet_list = 12; - // (string or) Address of the account to receive the block rewards. Can also be unclaimed which is encoded as a null byte - bytes reward_addr = 13; - // Tags that a block producer can add to a block - repeated Tag tags = 14; - // Size of reward pool - BigInt reward_pool = 15; - // Size of the weave in bytes - BigInt weave_size = 16; - // Size of this block in bytes - BigInt block_size = 17; - // Required after the version 1.8 fork. Zero otherwise. - // The sum of the average number of hashes computed - // by the network to produce the past blocks including this one. - BigInt cumulative_diff = 18; - // Required after the version 1.8 fork. Null byte otherwise. - // The Merkle root of the block index - the list of {`indep_hash`; `weave_size`; `tx_root`} triplets - bytes hash_list_merkle = 20; - // The proof of access; Used after v2.4 only; set as defaults otherwise - ProofOfAccess poa = 21; -} - -// A succinct proof of access to a recall byte found in a TX -message ProofOfAccess { - // The recall byte option chosen; global offset of index byte - string option = 1; - // The path through the Merkle tree of transactions' `data_root`s; - // from the `data_root` being proven to the corresponding `tx_root` - bytes tx_path = 2; - // The path through the Merkle tree of identifiers of chunks of the - // corresponding transaction; from the chunk being proven to the - // corresponding `data_root`. - bytes data_path = 3; - // The data chunk. - bytes chunk = 4; -} - -message Transaction { - // 1 or 2 for v1 or v2 transactions. More allowable in the future - uint32 format = 1; - // The transaction identifier. - bytes id = 2; - // Either the identifier of the previous transaction from the same - // wallet or the identifier of one of the last ?MAX_TX_ANCHOR_DEPTH blocks. - bytes last_tx = 3; - // The public key the transaction is signed with. - bytes owner = 4; - // A list of arbitrary key-value pairs - repeated Tag tags = 5; - // The address of the recipient; if any. The SHA2-256 hash of the public key. - bytes target = 6; - // The amount of Winstons to send to the recipient; if any. - BigInt quantity = 7; - // The data to upload; if any. For v2 transactions; the field is optional - // - a fee is charged based on the `data_size` field; - // data may be uploaded any time later in chunks. - bytes data = 8; - // Size in bytes of the transaction data. - BigInt data_size = 9; - // The Merkle root of the Merkle tree of data chunks. - bytes data_root = 10; - // The signature. - bytes signature = 11; - // The fee in Winstons. - BigInt reward = 12; -} - - -message Tag { - bytes name = 1; - bytes value = 2; -} diff --git a/chain/arweave/src/adapter.rs b/chain/arweave/src/adapter.rs deleted file mode 100644 index 9b25016b4c6..00000000000 --- a/chain/arweave/src/adapter.rs +++ /dev/null @@ -1,262 +0,0 @@ -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::prelude::*; -use sha2::{Digest, Sha256}; -use std::collections::HashSet; - -const MATCH_ALL_WILDCARD: &str = ""; -// Size of sha256(pubkey) -const SHA256_LEN: usize = 32; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter { - pub(crate) block_filter: ArweaveBlockFilter, - pub(crate) transaction_filter: ArweaveTransactionFilter, -} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { - let TriggerFilter { - block_filter, - transaction_filter, - } = self; - - block_filter.extend(ArweaveBlockFilter::from_data_sources(data_sources.clone())); - transaction_filter.extend(ArweaveTransactionFilter::from_data_sources(data_sources)); - } - - fn node_capabilities(&self) -> bc::EmptyNodeCapabilities { - bc::EmptyNodeCapabilities::default() - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - vec![] - } -} - -/// ArweaveBlockFilter will match every block regardless of source being set. -/// see docs: https://thegraph.com/docs/en/supported-networks/arweave/ -#[derive(Clone, Debug, Default)] -pub(crate) struct ArweaveTransactionFilter { - owners_pubkey: HashSet>, - owners_sha: HashSet>, - match_all: bool, -} - -impl ArweaveTransactionFilter { - pub fn matches(&self, owner: &[u8]) -> bool { - if self.match_all { - return true; - } - - if owner.len() == SHA256_LEN { - return self.owners_sha.contains(owner); - } - - self.owners_pubkey.contains(owner) || self.owners_sha.contains(&sha256(owner)) - } - - pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { - let owners: Vec> = iter - .into_iter() - .filter(|data_source| { - data_source.source.owner.is_some() - && !data_source.mapping.transaction_handlers.is_empty() - }) - .map(|ds| match &ds.source.owner { - Some(str) if MATCH_ALL_WILDCARD.eq(str) => MATCH_ALL_WILDCARD.as_bytes().to_owned(), - owner => base64_url::decode(&owner.clone().unwrap_or_default()).unwrap_or_default(), - }) - .collect(); - - let (owners_sha, long) = owners - .into_iter() - .partition::>, _>(|owner| owner.len() == SHA256_LEN); - - let (owners_pubkey, wildcard) = long - .into_iter() - .partition::>, _>(|long| long.len() != MATCH_ALL_WILDCARD.len()); - - let match_all = !wildcard.is_empty(); - - let owners_sha: Vec> = owners_sha - .into_iter() - .chain::>>(owners_pubkey.iter().map(|long| sha256(long)).collect()) - .collect(); - - Self { - match_all, - owners_pubkey: HashSet::from_iter(owners_pubkey), - owners_sha: HashSet::from_iter(owners_sha), - } - } - - pub fn extend(&mut self, other: ArweaveTransactionFilter) { - let ArweaveTransactionFilter { - owners_pubkey, - owners_sha, - match_all, - } = self; - - owners_pubkey.extend(other.owners_pubkey); - owners_sha.extend(other.owners_sha); - *match_all = *match_all || other.match_all; - } -} - -/// ArweaveBlockFilter will match every block regardless of source being set. -/// see docs: https://thegraph.com/docs/en/supported-networks/arweave/ -#[derive(Clone, Debug, Default)] -pub(crate) struct ArweaveBlockFilter { - pub trigger_every_block: bool, -} - -impl ArweaveBlockFilter { - pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { - Self { - trigger_every_block: iter - .into_iter() - .any(|data_source| !data_source.mapping.block_handlers.is_empty()), - } - } - - pub fn extend(&mut self, other: ArweaveBlockFilter) { - self.trigger_every_block = self.trigger_every_block || other.trigger_every_block; - } -} - -fn sha256(bs: &[u8]) -> Vec { - let mut hasher = Sha256::new(); - hasher.update(bs); - let res = hasher.finalize(); - res.to_vec() -} - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use graph::{prelude::Link, semver::Version}; - - use crate::data_source::{DataSource, Mapping, Source, TransactionHandler}; - - use super::{ArweaveTransactionFilter, MATCH_ALL_WILDCARD}; - - const ARWEAVE_PUBKEY_EXAMPLE: &str = "x-62w7g2yKACOgP_d04bhG8IX-AWgPrxHl2JgZBDdNLfAsidiiAaoIZPeM8K5gGvl7-8QVk79YV4OC878Ey0gXi7Atj5BouRyXnFMjJcPVXVyBoYCBuG7rJDDmh4_Ilon6vVOuHVIZ47Vb0tcgsxgxdvVFC2mn9N_SBl23pbeICNJZYOH57kf36gicuV_IwYSdqlQ0HQ_psjmg8EFqO7xzvAMP5HKW3rqTrYZxbCew2FkM734ysWckT39TpDBPx3HrFOl6obUdQWkHNOeKyzcsKFDywNgVWZOb89CYU7JFYlwX20io39ZZv0UJUOEFNjtVHkT_s0_A2O9PltsrZLLlQXZUuYASdbAPD2g_qXfhmPBZ0SXPWCDY-UVwVN1ncwYmk1F_i35IA8kAKsajaltD2wWDQn9g5mgJAWWn2xhLqkbwGbdwQMRD0-0eeuy1uzCooJQCC_bPJksoqkYwB9SGOjkayf4r4oZ2QDY4FicCsswz4Od_gud30ZWyHjWgqGzSFYFzawDBS1Gr_nu_q5otFrv20ZGTxYqGsLHWq4VHs6KjsQvzgBjfyb0etqHQEPJJmbQmY3LSogR4bxdReUHhj2EK9xIB-RKzDvDdL7fT5K0V9MjbnC2uktA0VjLlvwJ64_RhbQhxdp_zR39r-zyCXT-brPEYW1-V7Ey9K3XUE"; - const ARWEAVE_SHA_EXAMPLE: &str = "ahLxjCMCHr1ZE72VDDoaK4IKiLUUpeuo8t-M6y23DXw"; - - #[test] - fn transaction_filter_wildcard_matches_all() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(base64_url::encode(MATCH_ALL_WILDCARD)), 10), - new_datasource(Some(base64_url::encode("owner")), 10), - new_datasource(Some(ARWEAVE_PUBKEY_EXAMPLE.into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let filter = ArweaveTransactionFilter::from_data_sources(dss); - assert_eq!(true, filter.matches("asdas".as_bytes())) - } - - #[test] - fn transaction_filter_match() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(ARWEAVE_PUBKEY_EXAMPLE.into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let filter = ArweaveTransactionFilter::from_data_sources(dss); - assert_eq!(false, filter.matches("asdas".as_bytes())); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_SHA_EXAMPLE).expect("failed to parse sha example") - ) - ); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_PUBKEY_EXAMPLE).expect("failed to parse PK example") - ) - ) - } - - #[test] - fn transaction_filter_extend_match() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(ARWEAVE_SHA_EXAMPLE.into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let filter = ArweaveTransactionFilter::from_data_sources(dss); - assert_eq!(false, filter.matches("asdas".as_bytes())); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_SHA_EXAMPLE).expect("failed to parse sha example") - ) - ); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_PUBKEY_EXAMPLE).expect("failed to parse PK example") - ) - ) - } - - #[test] - fn transaction_filter_extend_wildcard_matches_all() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(MATCH_ALL_WILDCARD.into()), 10), - new_datasource(Some("owner".into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let mut filter = ArweaveTransactionFilter::default(); - - filter.extend(ArweaveTransactionFilter::from_data_sources(dss)); - assert_eq!(true, filter.matches("asdas".as_bytes())); - assert_eq!(true, filter.matches(ARWEAVE_PUBKEY_EXAMPLE.as_bytes())); - assert_eq!(true, filter.matches(ARWEAVE_SHA_EXAMPLE.as_bytes())) - } - - fn new_datasource(owner: Option, start_block: i32) -> DataSource { - DataSource { - kind: "".into(), - network: None, - name: "".into(), - source: Source { - owner, - start_block, - end_block: None, - }, - mapping: Mapping { - api_version: Version::new(1, 2, 3), - language: "".into(), - entities: vec![], - block_handlers: vec![], - transaction_handlers: vec![TransactionHandler { - handler: "my_handler".into(), - }], - runtime: Arc::new(vec![]), - link: Link { link: "".into() }, - }, - context: Arc::new(None), - creation_block: None, - } - } -} diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs deleted file mode 100644 index b827c78948a..00000000000 --- a/chain/arweave/src/chain.rs +++ /dev/null @@ -1,374 +0,0 @@ -use graph::anyhow; -use graph::blockchain::client::ChainClient; -use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; -use graph::blockchain::{ - BasicBlockchainBuilder, Block, BlockIngestor, BlockchainBuilder, BlockchainKind, - EmptyNodeCapabilities, NoopRuntimeAdapter, -}; -use graph::cheap_clone::CheapClone; -use graph::components::store::DeploymentCursorTracker; -use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::env::EnvVars; -use graph::firehose::FirehoseEndpoint; -use graph::prelude::MetricsRegistry; -use graph::substreams::Clock; -use graph::{ - blockchain::{ - block_stream::{ - BlockStreamEvent, BlockWithTriggers, FirehoseError, - FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, - }, - firehose_block_stream::FirehoseBlockStream, - BlockHash, BlockPtr, Blockchain, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - firehose::{self as firehose, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, -}; -use prost::Message; -use std::sync::Arc; - -use crate::adapter::TriggerFilter; -use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; -use crate::trigger::{self, ArweaveTrigger}; -use crate::{ - codec, - data_source::{DataSource, UnresolvedDataSource}, -}; -use graph::blockchain::block_stream::{BlockStream, BlockStreamMapper, FirehoseCursor}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: String, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: arweave") - } -} - -impl BlockchainBuilder for BasicBlockchainBuilder { - fn build(self, _config: &Arc) -> Chain { - Chain { - logger_factory: self.logger_factory, - name: self.name, - client: Arc::new(ChainClient::::new_firehose(self.firehose_endpoints)), - chain_store: self.chain_store, - metrics_registry: self.metrics_registry, - } - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Arweave; - - type Client = (); - type Block = codec::Block; - - type DataSource = DataSource; - - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = crate::trigger::ArweaveTrigger; - - type MappingTrigger = crate::trigger::ArweaveTrigger; - - type TriggerFilter = crate::adapter::TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn triggers_adapter( - &self, - _loc: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = TriggersAdapter {}; - Ok(Arc::new(adapter)) - } - - fn is_refetch_block_required(&self) -> bool { - false - } - - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - async fn new_block_stream( - &self, - deployment: DeploymentLocator, - store: impl DeploymentCursorTracker, - start_blocks: Vec, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = self - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); - - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - self.chain_client(), - store.block_ptr(), - store.firehose_cursor(), - firehose_mapper, - start_blocks, - logger, - self.metrics_registry.clone(), - ))) - } - - fn chain_store(&self) -> Arc { - self.chain_store.clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - self.client - .firehose_endpoint()? - .block_ptr_for_number::(logger, number) - .await - .map_err(Into::into) - } - - fn runtime_adapter(&self) -> Arc> { - Arc::new(NoopRuntimeAdapter::default()) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } - - fn block_ingestor(&self) -> anyhow::Result> { - let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), - self.chain_client(), - self.logger_factory - .component_logger("ArweaveFirehoseBlockIngestor", None), - self.name.clone(), - ); - Ok(Box::new(ingestor)) - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result>, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &TriggerFilter, - ) -> Result, Error> { - // TODO: Find the best place to introduce an `Arc` and avoid this clone. - let shared_block = Arc::new(block.clone()); - - let TriggerFilter { - block_filter, - transaction_filter, - } = filter; - - let txs = block - .clone() - .txs - .into_iter() - .filter(|tx| transaction_filter.matches(&tx.owner)) - .map(|tx| trigger::TransactionWithBlockPtr { - tx: Arc::new(tx), - block: shared_block.clone(), - }) - .collect::>(); - - let mut trigger_data: Vec<_> = txs - .into_iter() - .map(|tx| ArweaveTrigger::Transaction(Arc::new(tx))) - .collect(); - - if block_filter.trigger_every_block { - trigger_data.push(ArweaveTrigger::Block(shared_block.cheap_clone())); - } - - Ok(BlockWithTriggers::new(block, trigger_data, logger)) - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - panic!("Should never be called since FirehoseBlockStream cannot resolve it") - } - - /// Panics if `block` is genesis. - /// But that's ok since this is only called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - // FIXME (Arweave): Might not be necessary for Arweave support for now - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 48]), - number: block.number.saturating_sub(1), - })) - } -} - -pub struct FirehoseMapper { - adapter: Arc>, - filter: Arc, -} - -#[async_trait] -impl BlockStreamMapper for FirehoseMapper { - fn decode_block(&self, output: Option<&[u8]>) -> Result, Error> { - let block = match output { - Some(block) => codec::Block::decode(block)?, - None => anyhow::bail!("Arweave mapper is expected to always have a block"), - }; - - Ok(Some(block)) - } - - async fn block_with_triggers( - &self, - logger: &Logger, - block: codec::Block, - ) -> Result, Error> { - self.adapter - .triggers_in_block(logger, block, self.filter.as_ref()) - .await - } - async fn handle_substreams_block( - &self, - _logger: &Logger, - _clock: Clock, - _cursor: FirehoseCursor, - _block: Vec, - ) -> Result, Error> { - unimplemented!() - } -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - fn trigger_filter(&self) -> &TriggerFilter { - self.filter.as_ref() - } - - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down stuct that would decode only a few fields and ignore all the rest. - // unwrap: Input cannot be None so output will be error or block. - let block = self.decode_block(Some(&any_block.value.as_ref()))?.unwrap(); - - use ForkStep::*; - match step { - StepNew => Ok(BlockStreamEvent::ProcessBlock( - self.block_with_triggers(&logger, block).await?, - FirehoseCursor::from(response.cursor.clone()), - )), - - StepUndo => { - let parent_ptr = block - .parent_ptr() - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - StepFinal => { - panic!("irreversible step is not handled and should not be requested in the Firehose request") - } - - StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - // # FIXME - // - // the final block of arweave is itself in the current implementation - async fn final_block_ptr_for( - &self, - _logger: &Logger, - _endpoint: &Arc, - block: &codec::Block, - ) -> Result { - Ok(block.ptr()) - } -} diff --git a/chain/arweave/src/codec.rs b/chain/arweave/src/codec.rs deleted file mode 100644 index 3df276f309b..00000000000 --- a/chain/arweave/src/codec.rs +++ /dev/null @@ -1,45 +0,0 @@ -#[rustfmt::skip] -#[path = "protobuf/sf.arweave.r#type.v1.rs"] -mod pbcodec; - -use graph::{ - blockchain::Block as BlockchainBlock, - blockchain::{BlockPtr, BlockTime}, - prelude::BlockNumber, -}; - -pub use pbcodec::*; - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - BlockNumber::try_from(self.height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr { - hash: self.indep_hash.clone().into(), - number: self.number(), - } - } - - fn parent_ptr(&self) -> Option { - if self.height == 0 { - return None; - } - - Some(BlockPtr { - hash: self.previous_block.clone().into(), - number: self.number().saturating_sub(1), - }) - } - - fn timestamp(&self) -> BlockTime { - BlockTime::since_epoch(i64::try_from(self.timestamp).unwrap(), 0) - } -} - -impl AsRef<[u8]> for BigInt { - fn as_ref(&self) -> &[u8] { - self.bytes.as_ref() - } -} diff --git a/chain/arweave/src/data_source.rs b/chain/arweave/src/data_source.rs deleted file mode 100644 index b5db65ee096..00000000000 --- a/chain/arweave/src/data_source.rs +++ /dev/null @@ -1,407 +0,0 @@ -use graph::anyhow::Context; -use graph::blockchain::{Block, TriggerWithHandler}; -use graph::components::store::StoredDynamicDataSource; -use graph::data::subgraph::DataSourceContext; -use graph::prelude::SubgraphManifestValidationError; -use graph::{ - anyhow::{anyhow, Error}, - blockchain::{self, Blockchain}, - prelude::{ - async_trait, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, - LinkResolver, Logger, - }, - semver, -}; -use std::collections::HashSet; -use std::sync::Arc; - -use crate::chain::Chain; -use crate::trigger::ArweaveTrigger; - -pub const ARWEAVE_KIND: &str = "arweave"; -const BLOCK_HANDLER_KIND: &str = "block"; -const TRANSACTION_HANDLER_KIND: &str = "transaction"; -/// Runtime representation of a data source. -#[derive(Clone, Debug)] -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info(_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!("Arweave subgraphs do not support templates")) - } - - // FIXME - // - // need to decode the base64url encoding? - fn address(&self) -> Option<&[u8]> { - self.source.owner.as_ref().map(String::as_bytes) - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn handler_kinds(&self) -> HashSet<&str> { - let mut kinds = HashSet::new(); - - if self.handler_for_block().is_some() { - kinds.insert(BLOCK_HANDLER_KIND); - } - - if self.handler_for_transaction().is_some() { - kinds.insert(TRANSACTION_HANDLER_KIND); - } - - kinds - } - - fn end_block(&self) -> Option { - self.source.end_block - } - - fn match_and_decode( - &self, - trigger: &::TriggerData, - block: &Arc<::Block>, - _logger: &Logger, - ) -> Result>, Error> { - if self.source.start_block > block.number() { - return Ok(None); - } - - let handler = match trigger { - // A block trigger matches if a block handler is present. - ArweaveTrigger::Block(_) => match self.handler_for_block() { - Some(handler) => &handler.handler, - None => return Ok(None), - }, - // A transaction trigger matches if a transaction handler is present. - ArweaveTrigger::Transaction(_) => match self.handler_for_transaction() { - Some(handler) => &handler.handler, - None => return Ok(None), - }, - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.cheap_clone(), - handler.clone(), - block.ptr(), - block.timestamp(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - self.creation_block - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - context, - - // The creation block is ignored for detection duplicate data sources. - // Contract ABI equality is implicit in `source` and `mapping.abis` equality. - creation_block: _, - } = self; - - // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs - // used at runtime but not needed to define uniqueness; each runtime host should be for a - // unique data source. - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.block_handlers == other.mapping.block_handlers - && context == &other.context - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - // FIXME (Arweave): Implement me! - todo!() - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - // FIXME (Arweave): Implement me correctly - todo!() - } - - fn validate(&self) -> Vec { - let mut errors = Vec::new(); - - if self.kind != ARWEAVE_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - ARWEAVE_KIND, - self.kind - )) - } - - // Validate that there is a `source` address if there are transaction handlers - let no_source_address = self.address().is_none(); - let has_transaction_handlers = !self.mapping.transaction_handlers.is_empty(); - if no_source_address && has_transaction_handlers { - errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); - }; - - // Validate that there are no more than one of both block handlers and transaction handlers - if self.mapping.block_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated block handlers")); - } - if self.mapping.transaction_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated transaction handlers")); - } - - errors - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } -} - -impl DataSource { - fn from_manifest( - kind: String, - network: Option, - name: String, - source: Source, - mapping: Mapping, - context: Option, - ) -> Result { - // Data sources in the manifest are created "before genesis" so they have no creation block. - let creation_block = None; - - Ok(DataSource { - kind, - network, - name, - source, - mapping, - context: Arc::new(context), - creation_block, - }) - } - - fn handler_for_block(&self) -> Option<&MappingBlockHandler> { - self.mapping.block_handlers.first() - } - - fn handler_for_transaction(&self) -> Option<&TransactionHandler> { - self.mapping.transaction_handlers.first() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSource { - kind, - network, - name, - source, - mapping, - context, - } = self; - - let mapping = mapping.resolve(resolver, logger).await.with_context(|| { - format!( - "failed to resolve data source {} with source_address {:?} and start_block {}", - name, - base64_url::encode(&source.owner.clone().unwrap_or_default()), - source.start_block - ) - })?; - - DataSource::from_manifest(kind, network, name, source, mapping, context) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSourceTemplate { - kind, - network, - name, - mapping, - } = self; - - let mapping = mapping - .resolve(resolver, logger) - .await - .with_context(|| format!("failed to resolve data source template {}", name))?; - - Ok(DataSourceTemplate { - kind, - network, - name, - mapping, - }) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn name(&self) -> &str { - &self.name - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } - - fn manifest_idx(&self) -> u32 { - unreachable!("arweave does not support dynamic data sources") - } - - fn kind(&self) -> &str { - &self.kind - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - pub api_version: String, - pub language: String, - pub entities: Vec, - #[serde(default)] - pub block_handlers: Vec, - #[serde(default)] - pub transaction_handlers: Vec, - pub file: Link, -} - -impl UnresolvedMapping { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let UnresolvedMapping { - api_version, - language, - entities, - block_handlers, - transaction_handlers, - file: link, - } = self; - - let api_version = semver::Version::parse(&api_version)?; - - let module_bytes = resolver - .cat(logger, &link) - .await - .with_context(|| format!("failed to resolve mapping {}", link.link))?; - - Ok(Mapping { - api_version, - language, - entities, - block_handlers, - transaction_handlers, - runtime: Arc::new(module_bytes), - link, - }) - } -} - -#[derive(Clone, Debug)] -pub struct Mapping { - pub api_version: semver::Version, - pub language: String, - pub entities: Vec, - pub block_handlers: Vec, - pub transaction_handlers: Vec, - pub runtime: Arc>, - pub link: Link, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct TransactionHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct Source { - // A data source that does not have an owner can only have block handlers. - pub(crate) owner: Option, - #[serde(default)] - pub(crate) start_block: BlockNumber, - pub(crate) end_block: Option, -} diff --git a/chain/arweave/src/lib.rs b/chain/arweave/src/lib.rs deleted file mode 100644 index 77e63bc51ab..00000000000 --- a/chain/arweave/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod adapter; -mod chain; -mod codec; -mod data_source; -mod runtime; -mod trigger; - -pub use crate::chain::Chain; -pub use codec::Block; diff --git a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs deleted file mode 100644 index fba41614f1b..00000000000 --- a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs +++ /dev/null @@ -1,146 +0,0 @@ -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BigInt { - #[prost(bytes = "vec", tag = "1")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - /// Firehose block version (unrelated to Arweave block version) - #[prost(uint32, tag = "1")] - pub ver: u32, - /// The block identifier - #[prost(bytes = "vec", tag = "2")] - pub indep_hash: ::prost::alloc::vec::Vec, - /// The nonce chosen to solve the mining problem - #[prost(bytes = "vec", tag = "3")] - pub nonce: ::prost::alloc::vec::Vec, - /// `indep_hash` of the previous block in the weave - #[prost(bytes = "vec", tag = "4")] - pub previous_block: ::prost::alloc::vec::Vec, - /// POSIX time of block discovery - #[prost(uint64, tag = "5")] - pub timestamp: u64, - /// POSIX time of the last difficulty retarget - #[prost(uint64, tag = "6")] - pub last_retarget: u64, - /// Mining difficulty; the number `hash` must be greater than. - #[prost(message, optional, tag = "7")] - pub diff: ::core::option::Option, - /// How many blocks have passed since the genesis block - #[prost(uint64, tag = "8")] - pub height: u64, - /// Mining solution hash of the block; must satisfy the mining difficulty - #[prost(bytes = "vec", tag = "9")] - pub hash: ::prost::alloc::vec::Vec, - /// Merkle root of the tree of Merkle roots of block's transactions' data. - #[prost(bytes = "vec", tag = "10")] - pub tx_root: ::prost::alloc::vec::Vec, - /// Transactions contained within this block - #[prost(message, repeated, tag = "11")] - pub txs: ::prost::alloc::vec::Vec, - /// The root hash of the Merkle Patricia Tree containing - /// all wallet (account) balances and the identifiers - /// of the last transactions posted by them; if any. - #[prost(bytes = "vec", tag = "12")] - pub wallet_list: ::prost::alloc::vec::Vec, - /// (string or) Address of the account to receive the block rewards. Can also be unclaimed which is encoded as a null byte - #[prost(bytes = "vec", tag = "13")] - pub reward_addr: ::prost::alloc::vec::Vec, - /// Tags that a block producer can add to a block - #[prost(message, repeated, tag = "14")] - pub tags: ::prost::alloc::vec::Vec, - /// Size of reward pool - #[prost(message, optional, tag = "15")] - pub reward_pool: ::core::option::Option, - /// Size of the weave in bytes - #[prost(message, optional, tag = "16")] - pub weave_size: ::core::option::Option, - /// Size of this block in bytes - #[prost(message, optional, tag = "17")] - pub block_size: ::core::option::Option, - /// Required after the version 1.8 fork. Zero otherwise. - /// The sum of the average number of hashes computed - /// by the network to produce the past blocks including this one. - #[prost(message, optional, tag = "18")] - pub cumulative_diff: ::core::option::Option, - /// Required after the version 1.8 fork. Null byte otherwise. - /// The Merkle root of the block index - the list of {`indep_hash`; `weave_size`; `tx_root`} triplets - #[prost(bytes = "vec", tag = "20")] - pub hash_list_merkle: ::prost::alloc::vec::Vec, - /// The proof of access; Used after v2.4 only; set as defaults otherwise - #[prost(message, optional, tag = "21")] - pub poa: ::core::option::Option, -} -/// A succinct proof of access to a recall byte found in a TX -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProofOfAccess { - /// The recall byte option chosen; global offset of index byte - #[prost(string, tag = "1")] - pub option: ::prost::alloc::string::String, - /// The path through the Merkle tree of transactions' `data_root`s; - /// from the `data_root` being proven to the corresponding `tx_root` - #[prost(bytes = "vec", tag = "2")] - pub tx_path: ::prost::alloc::vec::Vec, - /// The path through the Merkle tree of identifiers of chunks of the - /// corresponding transaction; from the chunk being proven to the - /// corresponding `data_root`. - #[prost(bytes = "vec", tag = "3")] - pub data_path: ::prost::alloc::vec::Vec, - /// The data chunk. - #[prost(bytes = "vec", tag = "4")] - pub chunk: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transaction { - /// 1 or 2 for v1 or v2 transactions. More allowable in the future - #[prost(uint32, tag = "1")] - pub format: u32, - /// The transaction identifier. - #[prost(bytes = "vec", tag = "2")] - pub id: ::prost::alloc::vec::Vec, - /// Either the identifier of the previous transaction from the same - /// wallet or the identifier of one of the last ?MAX_TX_ANCHOR_DEPTH blocks. - #[prost(bytes = "vec", tag = "3")] - pub last_tx: ::prost::alloc::vec::Vec, - /// The public key the transaction is signed with. - #[prost(bytes = "vec", tag = "4")] - pub owner: ::prost::alloc::vec::Vec, - /// A list of arbitrary key-value pairs - #[prost(message, repeated, tag = "5")] - pub tags: ::prost::alloc::vec::Vec, - /// The address of the recipient; if any. The SHA2-256 hash of the public key. - #[prost(bytes = "vec", tag = "6")] - pub target: ::prost::alloc::vec::Vec, - /// The amount of Winstons to send to the recipient; if any. - #[prost(message, optional, tag = "7")] - pub quantity: ::core::option::Option, - /// The data to upload; if any. For v2 transactions; the field is optional - /// - a fee is charged based on the `data_size` field; - /// data may be uploaded any time later in chunks. - #[prost(bytes = "vec", tag = "8")] - pub data: ::prost::alloc::vec::Vec, - /// Size in bytes of the transaction data. - #[prost(message, optional, tag = "9")] - pub data_size: ::core::option::Option, - /// The Merkle root of the Merkle tree of data chunks. - #[prost(bytes = "vec", tag = "10")] - pub data_root: ::prost::alloc::vec::Vec, - /// The signature. - #[prost(bytes = "vec", tag = "11")] - pub signature: ::prost::alloc::vec::Vec, - /// The fee in Winstons. - #[prost(message, optional, tag = "12")] - pub reward: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tag { - #[prost(bytes = "vec", tag = "1")] - pub name: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "2")] - pub value: ::prost::alloc::vec::Vec, -} diff --git a/chain/arweave/src/runtime/abi.rs b/chain/arweave/src/runtime/abi.rs deleted file mode 100644 index 616cfa70b8c..00000000000 --- a/chain/arweave/src/runtime/abi.rs +++ /dev/null @@ -1,191 +0,0 @@ -use crate::codec; -use crate::trigger::TransactionWithBlockPtr; -use graph::runtime::gas::GasCounter; -use graph::runtime::{asc_new, AscHeap, AscPtr, HostExportError, ToAscObj}; -use graph_runtime_wasm::asc_abi::class::{Array, Uint8Array}; - -pub(crate) use super::generated::*; - -impl ToAscObj for codec::Tag { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTag { - name: asc_new(heap, self.name.as_slice(), gas)?, - value: asc_new(heap, self.value.as_slice(), gas)?, - }) - } -} - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content = self - .iter() - .map(|x| asc_new(heap, x.as_slice(), gas)) - .collect::>, _>>()?; - Ok(AscTransactionArray(Array::new(&content, heap, gas)?)) - } -} - -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content = self - .iter() - .map(|x| asc_new(heap, x, gas)) - .collect::, _>>()?; - Ok(AscTagArray(Array::new(&content, heap, gas)?)) - } -} - -impl ToAscObj for codec::ProofOfAccess { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscProofOfAccess { - option: asc_new(heap, &self.option, gas)?, - tx_path: asc_new(heap, self.tx_path.as_slice(), gas)?, - data_path: asc_new(heap, self.data_path.as_slice(), gas)?, - chunk: asc_new(heap, self.chunk.as_slice(), gas)?, - }) - } -} - -impl ToAscObj for codec::Transaction { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTransaction { - format: self.format, - id: asc_new(heap, self.id.as_slice(), gas)?, - last_tx: asc_new(heap, self.last_tx.as_slice(), gas)?, - owner: asc_new(heap, self.owner.as_slice(), gas)?, - tags: asc_new(heap, &self.tags, gas)?, - target: asc_new(heap, self.target.as_slice(), gas)?, - quantity: asc_new( - heap, - self.quantity - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - data: asc_new(heap, self.data.as_slice(), gas)?, - data_size: asc_new( - heap, - self.data_size - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - data_root: asc_new(heap, self.data_root.as_slice(), gas)?, - signature: asc_new(heap, self.signature.as_slice(), gas)?, - reward: asc_new( - heap, - self.reward.as_ref().map(|b| b.as_ref()).unwrap_or_default(), - gas, - )?, - }) - } -} - -impl ToAscObj for codec::Block { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscBlock { - indep_hash: asc_new(heap, self.indep_hash.as_slice(), gas)?, - nonce: asc_new(heap, self.nonce.as_slice(), gas)?, - previous_block: asc_new(heap, self.previous_block.as_slice(), gas)?, - timestamp: self.timestamp, - last_retarget: self.last_retarget, - diff: asc_new( - heap, - self.diff.as_ref().map(|b| b.as_ref()).unwrap_or_default(), - gas, - )?, - height: self.height, - hash: asc_new(heap, self.hash.as_slice(), gas)?, - tx_root: asc_new(heap, self.tx_root.as_slice(), gas)?, - txs: asc_new( - heap, - &self - .txs - .iter() - .map(|tx| tx.id.clone()) - .collect::>>(), - gas, - )?, - wallet_list: asc_new(heap, self.wallet_list.as_slice(), gas)?, - reward_addr: asc_new(heap, self.reward_addr.as_slice(), gas)?, - tags: asc_new(heap, &self.tags, gas)?, - reward_pool: asc_new( - heap, - self.reward_pool - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - weave_size: asc_new( - heap, - self.weave_size - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - block_size: asc_new( - heap, - self.block_size - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - cumulative_diff: asc_new( - heap, - self.cumulative_diff - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - hash_list_merkle: asc_new(heap, self.hash_list_merkle.as_slice(), gas)?, - poa: self - .poa - .as_ref() - .map(|poa| asc_new(heap, poa, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - }) - } -} - -impl ToAscObj for TransactionWithBlockPtr { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTransactionWithBlockPtr { - tx: asc_new(heap, &self.tx.as_ref(), gas)?, - block: asc_new(heap, self.block.as_ref(), gas)?, - }) - } -} diff --git a/chain/arweave/src/runtime/generated.rs b/chain/arweave/src/runtime/generated.rs deleted file mode 100644 index e8a10fdb158..00000000000 --- a/chain/arweave/src/runtime/generated.rs +++ /dev/null @@ -1,128 +0,0 @@ -use graph::runtime::{AscIndexId, AscPtr, AscType, DeterministicHostError, IndexForAscTypeId}; -use graph::semver::Version; -use graph_runtime_derive::AscType; -use graph_runtime_wasm::asc_abi::class::{Array, AscString, Uint8Array}; - -#[repr(C)] -#[derive(AscType, Default)] -pub struct AscBlock { - pub timestamp: u64, - pub last_retarget: u64, - pub height: u64, - pub indep_hash: AscPtr, - pub nonce: AscPtr, - pub previous_block: AscPtr, - pub diff: AscPtr, - pub hash: AscPtr, - pub tx_root: AscPtr, - pub txs: AscPtr, - pub wallet_list: AscPtr, - pub reward_addr: AscPtr, - pub tags: AscPtr, - pub reward_pool: AscPtr, - pub weave_size: AscPtr, - pub block_size: AscPtr, - pub cumulative_diff: AscPtr, - pub hash_list_merkle: AscPtr, - pub poa: AscPtr, -} - -impl AscIndexId for AscBlock { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveBlock; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscProofOfAccess { - pub option: AscPtr, - pub tx_path: AscPtr, - pub data_path: AscPtr, - pub chunk: AscPtr, -} - -impl AscIndexId for AscProofOfAccess { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveProofOfAccess; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscTransaction { - pub format: u32, - pub id: AscPtr, - pub last_tx: AscPtr, - pub owner: AscPtr, - pub tags: AscPtr, - pub target: AscPtr, - pub quantity: AscPtr, - pub data: AscPtr, - pub data_size: AscPtr, - pub data_root: AscPtr, - pub signature: AscPtr, - pub reward: AscPtr, -} - -impl AscIndexId for AscTransaction { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransaction; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscTag { - pub name: AscPtr, - pub value: AscPtr, -} - -impl AscIndexId for AscTag { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTag; -} - -#[repr(C)] -pub struct AscTransactionArray(pub(crate) Array>); - -impl AscType for AscTransactionArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -impl AscIndexId for AscTransactionArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransactionArray; -} - -#[repr(C)] -pub struct AscTagArray(pub(crate) Array>); - -impl AscType for AscTagArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -impl AscIndexId for AscTagArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTagArray; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscTransactionWithBlockPtr { - pub tx: AscPtr, - pub block: AscPtr, -} - -impl AscIndexId for AscTransactionWithBlockPtr { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransactionWithBlockPtr; -} diff --git a/chain/arweave/src/runtime/mod.rs b/chain/arweave/src/runtime/mod.rs deleted file mode 100644 index 31e18de7dd8..00000000000 --- a/chain/arweave/src/runtime/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod abi; - -mod generated; diff --git a/chain/arweave/src/trigger.rs b/chain/arweave/src/trigger.rs deleted file mode 100644 index 871dd5e11dd..00000000000 --- a/chain/arweave/src/trigger.rs +++ /dev/null @@ -1,152 +0,0 @@ -use graph::blockchain::Block; -use graph::blockchain::MappingTriggerTrait; -use graph::blockchain::TriggerData; -use graph::cheap_clone::CheapClone; -use graph::prelude::web3::types::H256; -use graph::prelude::BlockNumber; -use graph::runtime::asc_new; -use graph::runtime::gas::GasCounter; -use graph::runtime::AscHeap; -use graph::runtime::AscPtr; -use graph::runtime::HostExportError; -use graph_runtime_wasm::module::ToAscPtr; -use std::{cmp::Ordering, sync::Arc}; - -use crate::codec; - -// Logging the block is too verbose, so this strips the block from the trigger for Debug. -impl std::fmt::Debug for ArweaveTrigger { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[derive(Debug)] - pub enum MappingTriggerWithoutBlock { - Block, - Transaction(Arc), - } - - let trigger_without_block = match self { - ArweaveTrigger::Block(_) => MappingTriggerWithoutBlock::Block, - ArweaveTrigger::Transaction(tx) => { - MappingTriggerWithoutBlock::Transaction(tx.tx.clone()) - } - }; - - write!(f, "{:?}", trigger_without_block) - } -} - -impl ToAscPtr for ArweaveTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - Ok(match self { - ArweaveTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - ArweaveTrigger::Transaction(tx) => asc_new(heap, tx.as_ref(), gas)?.erase(), - }) - } -} - -#[derive(Clone)] -pub enum ArweaveTrigger { - Block(Arc), - Transaction(Arc), -} - -impl CheapClone for ArweaveTrigger { - fn cheap_clone(&self) -> ArweaveTrigger { - match self { - ArweaveTrigger::Block(block) => ArweaveTrigger::Block(block.cheap_clone()), - ArweaveTrigger::Transaction(tx) => ArweaveTrigger::Transaction(tx.cheap_clone()), - } - } -} - -impl PartialEq for ArweaveTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, - (Self::Transaction(a_tx), Self::Transaction(b_tx)) => a_tx.tx.id == b_tx.tx.id, - _ => false, - } - } -} - -impl Eq for ArweaveTrigger {} - -impl ArweaveTrigger { - pub fn block_number(&self) -> BlockNumber { - match self { - ArweaveTrigger::Block(block) => block.number(), - ArweaveTrigger::Transaction(tx) => tx.block.number(), - } - } - - pub fn block_hash(&self) -> H256 { - match self { - ArweaveTrigger::Block(block) => block.ptr().hash_as_h256(), - ArweaveTrigger::Transaction(tx) => tx.block.ptr().hash_as_h256(), - } - } - - fn error_context(&self) -> std::string::String { - match self { - ArweaveTrigger::Block(..) => { - format!("Block #{} ({})", self.block_number(), self.block_hash()) - } - ArweaveTrigger::Transaction(tx) => { - format!( - "Tx #{}, block #{}({})", - base64_url::encode(&tx.tx.id), - self.block_number(), - self.block_hash() - ) - } - } - } -} - -impl Ord for ArweaveTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - // Keep the order when comparing two block triggers - (Self::Block(..), Self::Block(..)) => Ordering::Equal, - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Greater, - (_, Self::Block(..)) => Ordering::Less, - - // Execution outcomes have no intrinsic ordering information so we keep the order in - // which they are included in the `txs` field of `Block`. - (Self::Transaction(..), Self::Transaction(..)) => Ordering::Equal, - } - } -} - -impl PartialOrd for ArweaveTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl TriggerData for ArweaveTrigger { - fn error_context(&self) -> String { - self.error_context() - } - - fn address_match(&self) -> Option<&[u8]> { - None - } -} - -impl MappingTriggerTrait for ArweaveTrigger { - fn error_context(&self) -> String { - self.error_context() - } -} - -pub struct TransactionWithBlockPtr { - // REVIEW: Do we want to actually also have those two below behind an `Arc` wrapper? - pub tx: Arc, - pub block: Arc, -} diff --git a/chain/common/Cargo.toml b/chain/common/Cargo.toml index cd2e2775960..1f945ea984e 100644 --- a/chain/common/Cargo.toml +++ b/chain/common/Cargo.toml @@ -7,6 +7,9 @@ edition.workspace = true [dependencies] protobuf = "3.0.2" -protobuf-parse = "3.3.0" +protobuf-parse = "3.7.2" anyhow = "1" -heck = "0.4" +heck = "0.5" + +[lints] +workspace = true diff --git a/chain/common/src/lib.rs b/chain/common/src/lib.rs index b8f2ae47eb4..395dd440c84 100644 --- a/chain/common/src/lib.rs +++ b/chain/common/src/lib.rs @@ -2,13 +2,13 @@ use std::collections::HashMap; use std::fmt::Debug; use anyhow::Error; -use protobuf::descriptor::field_descriptor_proto::Label; -use protobuf::descriptor::field_descriptor_proto::Type; +use protobuf::Message; +use protobuf::UnknownValueRef; use protobuf::descriptor::DescriptorProto; use protobuf::descriptor::FieldDescriptorProto; use protobuf::descriptor::OneofDescriptorProto; -use protobuf::Message; -use protobuf::UnknownValueRef; +use protobuf::descriptor::field_descriptor_proto::Label; +use protobuf::descriptor::field_descriptor_proto::Type; use std::convert::From; use std::path::Path; diff --git a/chain/common/tests/test-acme.rs b/chain/common/tests/test-acme.rs index 554e4ecbd5c..2945fea337e 100644 --- a/chain/common/tests/test-acme.rs +++ b/chain/common/tests/test-acme.rs @@ -7,8 +7,8 @@ fn check_repeated_type_ok() { let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) + .values() + .flat_map(|t| t.fields.iter()) .filter(|t| t.is_array) .map(|t| t.type_name.clone()) .collect::>(); @@ -63,7 +63,7 @@ fn required_ok() { !f.required, "Transaction.events field should NOT be required!" ), - _ => assert!(false, "Unexpected message field [{}]!", f.name), + _ => panic!("Unexpected message field [{}]!", f.name), }; }); } diff --git a/chain/cosmos/Cargo.toml b/chain/cosmos/Cargo.toml deleted file mode 100644 index d1a4e219866..00000000000 --- a/chain/cosmos/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "graph-chain-cosmos" -version.workspace = true -edition = "2018" - -[build-dependencies] -tonic-build = { workspace = true } -graph-chain-common = { path = "../common" } - -[dependencies] -graph = { path = "../../graph" } -prost = { workspace = true } -prost-types = { workspace = true } -serde = "1.0" -anyhow = "1.0" -semver = "1.0.21" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/cosmos/build.rs b/chain/cosmos/build.rs deleted file mode 100644 index c618d3b466d..00000000000 --- a/chain/cosmos/build.rs +++ /dev/null @@ -1,54 +0,0 @@ -const PROTO_FILE: &str = "proto/cosmos.proto"; - -fn main() { - println!("cargo:rerun-if-changed=proto"); - - let types = - graph_chain_common::parse_proto_file(PROTO_FILE).expect("Unable to parse proto file!"); - - let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) - .filter(|t| t.is_array) - .map(|t| t.type_name.clone()) - .collect::>(); - - let mut builder = tonic_build::configure().out_dir("src/protobuf"); - - for (name, ptype) in types { - //generate Asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_asc_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - //generate data index id - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_network_type_id(Cosmos)]", - ); - - //generate conversion from rust type to asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_from_rust_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - if array_types.contains(&ptype.name) { - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_array_type(Cosmos)]", - ); - } - } - - builder - .compile(&[PROTO_FILE], &["proto"]) - .expect("Failed to compile Firehose Cosmos proto(s)"); -} diff --git a/chain/cosmos/proto/cosmos.proto b/chain/cosmos/proto/cosmos.proto deleted file mode 100644 index c32502da1e9..00000000000 --- a/chain/cosmos/proto/cosmos.proto +++ /dev/null @@ -1,368 +0,0 @@ -syntax = "proto3"; - -package sf.cosmos.type.v1; - -option go_package = "github.com/figment-networks/proto-cosmos/pb/sf/cosmos/type/v1;pbcosmos"; - -import "google/protobuf/descriptor.proto"; -import "google/protobuf/any.proto"; -import "gogoproto/gogo.proto"; -import "cosmos_proto/cosmos.proto"; -import "firehose/annotations.proto"; - -message Block { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; - EvidenceList evidence = 2 [(gogoproto.nullable) = false]; - Commit last_commit = 3; - ResponseBeginBlock result_begin_block = 4 [(firehose.required) = true]; - ResponseEndBlock result_end_block = 5 [(firehose.required) = true]; - repeated TxResult transactions = 7; - repeated Validator validator_updates = 8; -} - -// HeaderOnlyBlock is a standard [Block] structure where all other fields are -// removed so that hydrating that object from a [Block] bytes payload will -// drastically reduce allocated memory required to hold the full block. -// -// This can be used to unpack a [Block] when only the [Header] information -// is required and greatly reduce required memory. -message HeaderOnlyBlock { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; -} - -message EventData { - Event event = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3; -} - -message TransactionData { - TxResult tx = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; -} - -message MessageData { - google.protobuf.Any message = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3 [(firehose.required) = true]; -} - -message TransactionContext { - bytes hash = 1; - uint32 index = 2; - uint32 code = 3; - int64 gas_wanted = 4; - int64 gas_used = 5; -} - -message Header { - Consensus version = 1 [(gogoproto.nullable) = false]; - string chain_id = 2 [(gogoproto.customname) = "ChainID"]; - uint64 height = 3; - Timestamp time = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - BlockID last_block_id = 5 [(firehose.required) = true, (gogoproto.nullable) = false]; - bytes last_commit_hash = 6; - bytes data_hash = 7; - bytes validators_hash = 8; - bytes next_validators_hash = 9; - bytes consensus_hash = 10; - bytes app_hash = 11; - bytes last_results_hash = 12; - bytes evidence_hash = 13; - bytes proposer_address = 14; - bytes hash = 15; -} - -message Consensus { - option (gogoproto.equal) = true; - - uint64 block = 1; - uint64 app = 2; -} - -message Timestamp { - int64 seconds = 1; - int32 nanos = 2; -} - -message BlockID { - bytes hash = 1; - PartSetHeader part_set_header = 2 [(gogoproto.nullable) = false]; -} - -message PartSetHeader { - uint32 total = 1; - bytes hash = 2; -} - -message EvidenceList { - repeated Evidence evidence = 1 [(gogoproto.nullable) = false]; -} - -message Evidence { - oneof sum { - DuplicateVoteEvidence duplicate_vote_evidence = 1; - LightClientAttackEvidence light_client_attack_evidence = 2; - } -} - -message DuplicateVoteEvidence { - EventVote vote_a = 1; - EventVote vote_b = 2; - int64 total_voting_power = 3; - int64 validator_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message EventVote { - SignedMsgType event_vote_type = 1 [json_name = "type"]; - uint64 height = 2; - int32 round = 3; - BlockID block_id = 4 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes validator_address = 6; - int32 validator_index = 7; - bytes signature = 8; -} - -enum SignedMsgType { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - SIGNED_MSG_TYPE_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "UnknownType"]; - SIGNED_MSG_TYPE_PREVOTE = 1 [(gogoproto.enumvalue_customname) = "PrevoteType"]; - SIGNED_MSG_TYPE_PRECOMMIT = 2 [(gogoproto.enumvalue_customname) = "PrecommitType"]; - SIGNED_MSG_TYPE_PROPOSAL = 32 [(gogoproto.enumvalue_customname) = "ProposalType"]; -} - -message LightClientAttackEvidence { - LightBlock conflicting_block = 1; - int64 common_height = 2; - repeated Validator byzantine_validators = 3; - int64 total_voting_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message LightBlock { - SignedHeader signed_header = 1; - ValidatorSet validator_set = 2; -} - -message SignedHeader { - Header header = 1; - Commit commit = 2; -} - -message Commit { - int64 height = 1; - int32 round = 2; - BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; -} - -message CommitSig { - BlockIDFlag block_id_flag = 1; - bytes validator_address = 2; - Timestamp timestamp = 3 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes signature = 4; -} - -enum BlockIDFlag { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - BLOCK_ID_FLAG_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "BlockIDFlagUnknown"]; - BLOCK_ID_FLAG_ABSENT = 1 [(gogoproto.enumvalue_customname) = "BlockIDFlagAbsent"]; - BLOCK_ID_FLAG_COMMIT = 2 [(gogoproto.enumvalue_customname) = "BlockIDFlagCommit"]; - BLOCK_ID_FLAG_NIL = 3 [(gogoproto.enumvalue_customname) = "BlockIDFlagNil"]; -} - -message ValidatorSet { - repeated Validator validators = 1; - Validator proposer = 2; - int64 total_voting_power = 3; -} - -message Validator { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 voting_power = 3; - int64 proposer_priority = 4; -} - -message PublicKey { - option (gogoproto.compare) = true; - option (gogoproto.equal) = true; - - oneof sum { - bytes ed25519 = 1; - bytes secp256k1 = 2; - } -} - -message ResponseBeginBlock { - repeated Event events = 1 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; -} - -message Event { - string event_type = 1 [json_name = "type"]; - repeated EventAttribute attributes = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"]; -} - -message EventAttribute { - string key = 1; - string value = 2; - bool index = 3; -} - -message ResponseEndBlock { - repeated ValidatorUpdate validator_updates = 1; - ConsensusParams consensus_param_updates = 2; - repeated Event events = 3; -} - -message ValidatorUpdate { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 power = 3; -} - -message ConsensusParams { - BlockParams block = 1 [(gogoproto.nullable) = false]; - EvidenceParams evidence = 2 [(gogoproto.nullable) = false]; - ValidatorParams validator = 3 [(gogoproto.nullable) = false]; - VersionParams version = 4 [(gogoproto.nullable) = false]; -} - -message BlockParams { - int64 max_bytes = 1; - int64 max_gas = 2; -} - -message EvidenceParams { - int64 max_age_num_blocks = 1; - Duration max_age_duration = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; - int64 max_bytes = 3; -} - -message Duration { - int64 seconds = 1; - int32 nanos = 2; -} - -message ValidatorParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - repeated string pub_key_types = 1; -} - -message VersionParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - uint64 app_version = 1; -} - -message TxResult { - uint64 height = 1; - uint32 index = 2; - Tx tx = 3 [(firehose.required) = true]; - ResponseDeliverTx result = 4 [(firehose.required) = true]; - bytes hash = 5; -} - -message Tx { - TxBody body = 1 [(firehose.required) = true]; - AuthInfo auth_info = 2; - repeated bytes signatures = 3; -} - -message TxBody { - repeated google.protobuf.Any messages = 1; - string memo = 2; - uint64 timeout_height = 3; - repeated google.protobuf.Any extension_options = 1023; - repeated google.protobuf.Any non_critical_extension_options = 2047; -} - -message Any { - string type_url = 1; - bytes value = 2; -} - -message AuthInfo { - repeated SignerInfo signer_infos = 1; - Fee fee = 2; - Tip tip = 3; -} - -message SignerInfo { - google.protobuf.Any public_key = 1; - ModeInfo mode_info = 2; - uint64 sequence = 3; -} - -message ModeInfo { - oneof sum { - ModeInfoSingle single = 1; - ModeInfoMulti multi = 2; - } -} - -message ModeInfoSingle { - SignMode mode = 1; -} - -enum SignMode { - SIGN_MODE_UNSPECIFIED = 0; - SIGN_MODE_DIRECT = 1; - SIGN_MODE_TEXTUAL = 2; - SIGN_MODE_LEGACY_AMINO_JSON = 127; -} - -message ModeInfoMulti { - CompactBitArray bitarray = 1; - repeated ModeInfo mode_infos = 2; -} - -message CompactBitArray { - option (gogoproto.goproto_stringer) = false; - - uint32 extra_bits_stored = 1; - bytes elems = 2; -} - -message Fee { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - uint64 gas_limit = 2; - string payer = 3 [(cosmos_proto.scalar) = "cosmos.AddressString"]; - string granter = 4 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message Coin { - option (gogoproto.equal) = true; - - string denom = 1; - string amount = 2 [(gogoproto.customtype) = "Int", (gogoproto.nullable) = false]; -} - -message Tip { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - string tipper = 2 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message ResponseDeliverTx { - uint32 code = 1; - bytes data = 2; - string log = 3; - string info = 4; - int64 gas_wanted = 5; - int64 gas_used = 6; - repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; - string codespace = 8; -} - -message ValidatorSetUpdates { - repeated Validator validator_updates = 1; -} diff --git a/chain/cosmos/proto/cosmos_proto/cosmos.proto b/chain/cosmos/proto/cosmos_proto/cosmos.proto deleted file mode 100644 index 5c63b86f063..00000000000 --- a/chain/cosmos/proto/cosmos_proto/cosmos.proto +++ /dev/null @@ -1,97 +0,0 @@ -syntax = "proto3"; -package cosmos_proto; - -import "google/protobuf/descriptor.proto"; - -option go_package = "github.com/cosmos/cosmos-proto;cosmos_proto"; - -extend google.protobuf.MessageOptions { - - // implements_interface is used to indicate the type name of the interface - // that a message implements so that it can be used in google.protobuf.Any - // fields that accept that interface. A message can implement multiple - // interfaces. Interfaces should be declared using a declare_interface - // file option. - repeated string implements_interface = 93001; -} - -extend google.protobuf.FieldOptions { - - // accepts_interface is used to annotate that a google.protobuf.Any - // field accepts messages that implement the specified interface. - // Interfaces should be declared using a declare_interface file option. - string accepts_interface = 93001; - - // scalar is used to indicate that this field follows the formatting defined - // by the named scalar which should be declared with declare_scalar. Code - // generators may choose to use this information to map this field to a - // language-specific type representing the scalar. - string scalar = 93002; -} - -extend google.protobuf.FileOptions { - - // declare_interface declares an interface type to be used with - // accepts_interface and implements_interface. Interface names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given interface type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/interfaces.proto in the file descriptor set. - repeated InterfaceDescriptor declare_interface = 793021; - - // declare_scalar declares a scalar type to be used with - // the scalar field option. Scalar names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given scalar type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/scalars.proto in the file descriptor set. - repeated ScalarDescriptor declare_scalar = 793022; -} - -// InterfaceDescriptor describes an interface type to be used with -// accepts_interface and implements_interface and declared by declare_interface. -message InterfaceDescriptor { - - // name is the name of the interface. It should be a short-name (without - // a period) such that the fully qualified name of the interface will be - // package.name, ex. for the package a.b and interface named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the interface and its - // purpose. - string description = 2; -} - -// ScalarDescriptor describes an scalar type to be used with -// the scalar field option and declared by declare_scalar. -// Scalars extend simple protobuf built-in types with additional -// syntax and semantics, for instance to represent big integers. -// Scalars should ideally define an encoding such that there is only one -// valid syntactical representation for a given semantic meaning, -// i.e. the encoding should be deterministic. -message ScalarDescriptor { - - // name is the name of the scalar. It should be a short-name (without - // a period) such that the fully qualified name of the scalar will be - // package.name, ex. for the package a.b and scalar named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the scalar and its - // encoding format. For instance a big integer or decimal scalar should - // specify precisely the expected encoding format. - string description = 2; - - // field_type is the type of field with which this scalar can be used. - // Scalars can be used with one and only one type of field so that - // encoding standards and simple and clear. Currently only string and - // bytes fields are supported for scalars. - repeated ScalarType field_type = 3; -} - -enum ScalarType { - SCALAR_TYPE_UNSPECIFIED = 0; - SCALAR_TYPE_STRING = 1; - SCALAR_TYPE_BYTES = 2; -} diff --git a/chain/cosmos/proto/firehose/annotations.proto b/chain/cosmos/proto/firehose/annotations.proto deleted file mode 100644 index 1476c1ab08d..00000000000 --- a/chain/cosmos/proto/firehose/annotations.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package firehose; - -option go_package = "github.com/streamingfast/pbgo/sf/firehose/v1;pbfirehose"; - -import "google/protobuf/descriptor.proto"; - -extend google.protobuf.FieldOptions { - optional bool required = 77001; -} diff --git a/chain/cosmos/proto/gogoproto/gogo.proto b/chain/cosmos/proto/gogoproto/gogo.proto deleted file mode 100644 index 49e78f99fe5..00000000000 --- a/chain/cosmos/proto/gogoproto/gogo.proto +++ /dev/null @@ -1,145 +0,0 @@ -// Protocol Buffers for Go with Gadgets -// -// Copyright (c) 2013, The GoGo Authors. All rights reserved. -// http://github.com/gogo/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -package gogoproto; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "GoGoProtos"; -option go_package = "github.com/gogo/protobuf/gogoproto"; - -extend google.protobuf.EnumOptions { - optional bool goproto_enum_prefix = 62001; - optional bool goproto_enum_stringer = 62021; - optional bool enum_stringer = 62022; - optional string enum_customname = 62023; - optional bool enumdecl = 62024; -} - -extend google.protobuf.EnumValueOptions { - optional string enumvalue_customname = 66001; -} - -extend google.protobuf.FileOptions { - optional bool goproto_getters_all = 63001; - optional bool goproto_enum_prefix_all = 63002; - optional bool goproto_stringer_all = 63003; - optional bool verbose_equal_all = 63004; - optional bool face_all = 63005; - optional bool gostring_all = 63006; - optional bool populate_all = 63007; - optional bool stringer_all = 63008; - optional bool onlyone_all = 63009; - - optional bool equal_all = 63013; - optional bool description_all = 63014; - optional bool testgen_all = 63015; - optional bool benchgen_all = 63016; - optional bool marshaler_all = 63017; - optional bool unmarshaler_all = 63018; - optional bool stable_marshaler_all = 63019; - - optional bool sizer_all = 63020; - - optional bool goproto_enum_stringer_all = 63021; - optional bool enum_stringer_all = 63022; - - optional bool unsafe_marshaler_all = 63023; - optional bool unsafe_unmarshaler_all = 63024; - - optional bool goproto_extensions_map_all = 63025; - optional bool goproto_unrecognized_all = 63026; - optional bool gogoproto_import = 63027; - optional bool protosizer_all = 63028; - optional bool compare_all = 63029; - optional bool typedecl_all = 63030; - optional bool enumdecl_all = 63031; - - optional bool goproto_registration = 63032; - optional bool messagename_all = 63033; - - optional bool goproto_sizecache_all = 63034; - optional bool goproto_unkeyed_all = 63035; -} - -extend google.protobuf.MessageOptions { - optional bool goproto_getters = 64001; - optional bool goproto_stringer = 64003; - optional bool verbose_equal = 64004; - optional bool face = 64005; - optional bool gostring = 64006; - optional bool populate = 64007; - optional bool stringer = 67008; - optional bool onlyone = 64009; - - optional bool equal = 64013; - optional bool description = 64014; - optional bool testgen = 64015; - optional bool benchgen = 64016; - optional bool marshaler = 64017; - optional bool unmarshaler = 64018; - optional bool stable_marshaler = 64019; - - optional bool sizer = 64020; - - optional bool unsafe_marshaler = 64023; - optional bool unsafe_unmarshaler = 64024; - - optional bool goproto_extensions_map = 64025; - optional bool goproto_unrecognized = 64026; - - optional bool protosizer = 64028; - optional bool compare = 64029; - - optional bool typedecl = 64030; - - optional bool messagename = 64033; - - optional bool goproto_sizecache = 64034; - optional bool goproto_unkeyed = 64035; -} - -extend google.protobuf.FieldOptions { - optional bool nullable = 65001; - optional bool embed = 65002; - optional string customtype = 65003; - optional string customname = 65004; - optional string jsontag = 65005; - optional string moretags = 65006; - optional string casttype = 65007; - optional string castkey = 65008; - optional string castvalue = 65009; - - optional bool stdtime = 65010; - optional bool stdduration = 65011; - optional bool wktpointer = 65012; - - optional string castrepeated = 65013; -} diff --git a/chain/cosmos/src/adapter.rs b/chain/cosmos/src/adapter.rs deleted file mode 100644 index 746c91e2e07..00000000000 --- a/chain/cosmos/src/adapter.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::collections::HashSet; - -use prost::Message; -use prost_types::Any; - -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::firehose::EventTypeFilter; -use graph::prelude::*; - -const EVENT_TYPE_FILTER_TYPE_URL: &str = - "type.googleapis.com/sf.cosmos.transform.v1.EventTypeFilter"; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter { - pub(crate) event_type_filter: CosmosEventTypeFilter, - pub(crate) block_filter: CosmosBlockFilter, -} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { - self.event_type_filter - .extend_from_data_sources(data_sources.clone()); - self.block_filter.extend_from_data_sources(data_sources); - } - - fn node_capabilities(&self) -> bc::EmptyNodeCapabilities { - bc::EmptyNodeCapabilities::default() - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - if self.block_filter.trigger_every_block { - return vec![]; - } - - if self.event_type_filter.event_types.is_empty() { - return vec![]; - } - - let filter = EventTypeFilter { - event_types: Vec::from_iter(self.event_type_filter.event_types), - }; - - vec![Any { - type_url: EVENT_TYPE_FILTER_TYPE_URL.to_string(), - value: filter.encode_to_vec(), - }] - } -} - -pub type EventType = String; - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosEventTypeFilter { - pub event_types: HashSet, -} - -impl CosmosEventTypeFilter { - pub(crate) fn matches(&self, event_type: &EventType) -> bool { - self.event_types.contains(event_type) - } - - fn extend_from_data_sources<'a>(&mut self, data_sources: impl Iterator) { - self.event_types.extend( - data_sources.flat_map(|data_source| data_source.events().map(ToString::to_string)), - ); - } -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosBlockFilter { - pub trigger_every_block: bool, -} - -impl CosmosBlockFilter { - fn extend_from_data_sources<'a>( - &mut self, - mut data_sources: impl Iterator, - ) { - if !self.trigger_every_block { - self.trigger_every_block = data_sources.any(DataSource::has_block_handler); - } - } -} - -#[cfg(test)] -mod test { - use graph::blockchain::TriggerFilter as _; - - use super::*; - - #[test] - fn test_trigger_filters() { - let cases = [ - (TriggerFilter::test_new(false, &[]), None), - (TriggerFilter::test_new(true, &[]), None), - (TriggerFilter::test_new(true, &["event_1", "event_2"]), None), - ( - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - Some(event_type_filter_with(&["event_1", "event_2", "event_3"])), - ), - ]; - - for (trigger_filter, expected_filter) in cases { - let firehose_filter = trigger_filter.to_firehose_filter(); - let decoded_filter = decode_filter(firehose_filter); - - assert_eq!(decoded_filter.is_some(), expected_filter.is_some()); - - if let (Some(mut expected_filter), Some(mut decoded_filter)) = - (expected_filter, decoded_filter) - { - // event types may be in different order - expected_filter.event_types.sort(); - decoded_filter.event_types.sort(); - - assert_eq!(decoded_filter, expected_filter); - } - } - } - - impl TriggerFilter { - pub(crate) fn test_new(trigger_every_block: bool, event_types: &[&str]) -> TriggerFilter { - TriggerFilter { - event_type_filter: CosmosEventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - }, - block_filter: CosmosBlockFilter { - trigger_every_block, - }, - } - } - } - - fn event_type_filter_with(event_types: &[&str]) -> EventTypeFilter { - EventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - } - } - - fn decode_filter(proto_filters: Vec) -> Option { - assert!(proto_filters.len() <= 1); - - let proto_filter = proto_filters.get(0)?; - - assert_eq!(proto_filter.type_url, EVENT_TYPE_FILTER_TYPE_URL); - - let firehose_filter = EventTypeFilter::decode(&*proto_filter.value) - .expect("Could not decode EventTypeFilter from protobuf Any"); - - Some(firehose_filter) - } -} diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs deleted file mode 100644 index 5cc1a6f9b1b..00000000000 --- a/chain/cosmos/src/chain.rs +++ /dev/null @@ -1,677 +0,0 @@ -use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; -use graph::blockchain::BlockIngestor; -use graph::env::EnvVars; -use graph::prelude::MetricsRegistry; -use graph::substreams::Clock; -use std::sync::Arc; - -use graph::blockchain::block_stream::{BlockStreamMapper, FirehoseCursor}; -use graph::blockchain::client::ChainClient; -use graph::blockchain::{BasicBlockchainBuilder, BlockchainBuilder, NoopRuntimeAdapter}; -use graph::cheap_clone::CheapClone; -use graph::components::store::DeploymentCursorTracker; -use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::{ - blockchain::{ - block_stream::{ - BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseError, - FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, - }, - firehose_block_stream::FirehoseBlockStream, - Block as _, BlockHash, BlockPtr, Blockchain, BlockchainKind, EmptyNodeCapabilities, - IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - firehose::{self, FirehoseEndpoint, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, -}; -use prost::Message; - -use crate::data_source::{ - DataSource, DataSourceTemplate, EventOrigin, UnresolvedDataSource, UnresolvedDataSourceTemplate, -}; -use crate::trigger::CosmosTrigger; -use crate::{codec, TriggerFilter}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: String, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: cosmos") - } -} - -impl BlockchainBuilder for BasicBlockchainBuilder { - fn build(self, _config: &Arc) -> Chain { - Chain { - logger_factory: self.logger_factory, - name: self.name, - client: Arc::new(ChainClient::new_firehose(self.firehose_endpoints)), - chain_store: self.chain_store, - metrics_registry: self.metrics_registry, - } - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Cosmos; - - type Client = (); - type Block = codec::Block; - - type DataSource = DataSource; - - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = CosmosTrigger; - - type MappingTrigger = CosmosTrigger; - - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn is_refetch_block_required(&self) -> bool { - false - } - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - fn triggers_adapter( - &self, - _loc: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = TriggersAdapter {}; - Ok(Arc::new(adapter)) - } - - async fn new_block_stream( - &self, - deployment: DeploymentLocator, - store: impl DeploymentCursorTracker, - start_blocks: Vec, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = self - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); - - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - self.chain_client(), - store.block_ptr(), - store.firehose_cursor(), - firehose_mapper, - start_blocks, - logger, - self.metrics_registry.clone(), - ))) - } - - fn chain_store(&self) -> Arc { - self.chain_store.cheap_clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint()?; - - firehose_endpoint - .block_ptr_for_number::(logger, number) - .await - .map_err(Into::into) - } - - fn runtime_adapter(&self) -> Arc> { - Arc::new(NoopRuntimeAdapter::default()) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } - - fn block_ingestor(&self) -> anyhow::Result> { - let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), - self.chain_client(), - self.logger_factory - .component_logger("CosmosFirehoseBlockIngestor", None), - self.name.clone(), - ); - Ok(Box::new(ingestor)) - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result>, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &TriggerFilter, - ) -> Result, Error> { - let shared_block = Arc::new(block.clone()); - - let header_only_block = codec::HeaderOnlyBlock::from(&block); - - let mut triggers: Vec<_> = shared_block - .begin_block_events()? - .cloned() - // FIXME (Cosmos): Optimize. Should use an Arc instead of cloning the - // block. This is not currently possible because EventData is automatically - // generated. - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::BeginBlock, - ) - }) - .chain(shared_block.transactions().flat_map(|tx| { - tx.result - .as_ref() - .unwrap() - .events - .iter() - .filter_map(|e| { - filter_event_trigger( - filter, - e.clone(), - &header_only_block, - Some(build_tx_context(tx)), - EventOrigin::DeliverTx, - ) - }) - .collect::>() - })) - .chain( - shared_block - .end_block_events()? - .cloned() - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::EndBlock, - ) - }), - ) - .collect(); - - triggers.extend(shared_block.transactions().cloned().flat_map(|tx_result| { - let mut triggers: Vec<_> = Vec::new(); - if let Some(tx) = tx_result.tx.clone() { - if let Some(tx_body) = tx.body { - triggers.extend(tx_body.messages.into_iter().map(|message| { - CosmosTrigger::with_message( - message, - header_only_block.clone(), - build_tx_context(&tx_result), - ) - })); - } - } - triggers.push(CosmosTrigger::with_transaction( - tx_result, - header_only_block.clone(), - )); - triggers - })); - - if filter.block_filter.trigger_every_block { - triggers.push(CosmosTrigger::Block(shared_block.cheap_clone())); - } - - Ok(BlockWithTriggers::new(block, triggers, logger)) - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - /// Panics if `block` is genesis. - /// But that's ok since this is only called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -/// Returns a new event trigger only if the given event matches the event filter. -fn filter_event_trigger( - filter: &TriggerFilter, - event: codec::Event, - block: &codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, -) -> Option { - if filter.event_type_filter.matches(&event.event_type) { - Some(CosmosTrigger::with_event( - event, - block.clone(), - tx_context, - origin, - )) - } else { - None - } -} - -fn build_tx_context(tx: &codec::TxResult) -> codec::TransactionContext { - codec::TransactionContext { - hash: tx.hash.clone(), - index: tx.index, - code: tx.result.as_ref().unwrap().code, - gas_wanted: tx.result.as_ref().unwrap().gas_wanted, - gas_used: tx.result.as_ref().unwrap().gas_used, - } -} - -pub struct FirehoseMapper { - adapter: Arc>, - filter: Arc, -} - -#[async_trait] -impl BlockStreamMapper for FirehoseMapper { - fn decode_block(&self, output: Option<&[u8]>) -> Result, Error> { - let block = match output { - Some(block) => crate::Block::decode(block)?, - None => anyhow::bail!("cosmos mapper is expected to always have a block"), - }; - - Ok(Some(block)) - } - - async fn block_with_triggers( - &self, - logger: &Logger, - block: crate::Block, - ) -> Result, Error> { - self.adapter - .triggers_in_block(logger, block, self.filter.as_ref()) - .await - } - - async fn handle_substreams_block( - &self, - _logger: &Logger, - _clock: Clock, - _cursor: FirehoseCursor, - _block: Vec, - ) -> Result, Error> { - unimplemented!() - } -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - fn trigger_filter(&self) -> &TriggerFilter { - self.filter.as_ref() - } - - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down struct that would decode only a few fields and ignore all the rest. - // unwrap: Input cannot be None so output will be error or block. - let block = self.decode_block(Some(any_block.value.as_ref()))?.unwrap(); - - match step { - ForkStep::StepNew => Ok(BlockStreamEvent::ProcessBlock( - self.block_with_triggers(logger, block).await?, - FirehoseCursor::from(response.cursor.clone()), - )), - - ForkStep::StepUndo => { - let parent_ptr = block - .parent_ptr() - .map_err(FirehoseError::from)? - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - ForkStep::StepFinal => { - panic!( - "final step is not handled and should not be requested in the Firehose request" - ) - } - - ForkStep::StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - async fn final_block_ptr_for( - &self, - logger: &Logger, - endpoint: &Arc, - block: &codec::Block, - ) -> Result { - // Cosmos provides instant block finality. - self.block_ptr_for_number(logger, endpoint, block.number()) - .await - } -} - -#[cfg(test)] -mod test { - use graph::prelude::{ - slog::{o, Discard, Logger}, - tokio, - }; - - use super::*; - - use codec::{ - Block, Event, Header, HeaderOnlyBlock, ResponseBeginBlock, ResponseDeliverTx, - ResponseEndBlock, TxResult, - }; - - #[tokio::test] - async fn test_trigger_filters() { - let adapter = TriggersAdapter {}; - let logger = Logger::root(Discard, o!()); - - let block_with_events = Block::test_with_event_types( - vec!["begin_event_1", "begin_event_2", "begin_event_3"], - vec!["tx_event_1", "tx_event_2", "tx_event_3"], - vec!["end_event_1", "end_event_2", "end_event_3"], - ); - - let header_only_block = HeaderOnlyBlock::from(&block_with_events); - - let cases = [ - ( - Block::test_new(), - TriggerFilter::test_new(false, &[]), - vec![], - ), - ( - Block::test_new(), - TriggerFilter::test_new(true, &[]), - vec![CosmosTrigger::Block(Arc::new(Block::test_new()))], - ), - ( - Block::test_new(), - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - vec![], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(false, &["begin_event_3", "tx_event_3", "end_event_3"]), - vec![ - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_3"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[2])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_3"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(true, &["begin_event_3", "tx_event_2", "end_event_1"]), - vec![ - CosmosTrigger::Block(Arc::new(block_with_events.clone())), - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_2"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[1])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_1"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ]; - - for (block, trigger_filter, expected_triggers) in cases { - let triggers = adapter - .triggers_in_block(&logger, block, &trigger_filter) - .await - .expect("failed to get triggers in block"); - - assert_eq!( - triggers.trigger_data.len(), - expected_triggers.len(), - "Expected trigger list to contain exactly {:?}, but it didn't: {:?}", - expected_triggers, - triggers.trigger_data - ); - - // they may not be in the same order - for trigger in expected_triggers { - assert!( - triggers.trigger_data.contains(&trigger), - "Expected trigger list to contain {:?}, but it only contains: {:?}", - trigger, - triggers.trigger_data - ); - } - } - } - - impl Block { - fn test_new() -> Block { - Block::test_with_event_types(vec![], vec![], vec![]) - } - - fn test_with_event_types( - begin_event_types: Vec<&str>, - tx_event_types: Vec<&str>, - end_event_types: Vec<&str>, - ) -> Block { - Block { - header: Some(Header { - version: None, - chain_id: "test".to_string(), - height: 1, - time: None, - last_block_id: None, - last_commit_hash: vec![], - data_hash: vec![], - validators_hash: vec![], - next_validators_hash: vec![], - consensus_hash: vec![], - app_hash: vec![], - last_results_hash: vec![], - evidence_hash: vec![], - proposer_address: vec![], - hash: vec![], - }), - evidence: None, - last_commit: None, - result_begin_block: Some(ResponseBeginBlock { - events: begin_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - result_end_block: Some(ResponseEndBlock { - validator_updates: vec![], - consensus_param_updates: None, - events: end_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - transactions: tx_event_types - .into_iter() - .map(TxResult::test_with_event_type) - .collect(), - validator_updates: vec![], - } - } - } - - impl Event { - fn test_with_type(event_type: &str) -> Event { - Event { - event_type: event_type.to_string(), - attributes: vec![], - } - } - } - - impl TxResult { - fn test_with_event_type(event_type: &str) -> TxResult { - TxResult { - height: 1, - index: 1, - tx: None, - result: Some(ResponseDeliverTx { - code: 1, - data: vec![], - log: "".to_string(), - info: "".to_string(), - gas_wanted: 1, - gas_used: 1, - codespace: "".to_string(), - events: vec![Event::test_with_type(event_type)], - }), - hash: vec![], - } - } - } -} diff --git a/chain/cosmos/src/codec.rs b/chain/cosmos/src/codec.rs deleted file mode 100644 index bdc05c1b820..00000000000 --- a/chain/cosmos/src/codec.rs +++ /dev/null @@ -1,198 +0,0 @@ -pub(crate) use crate::protobuf::pbcodec::*; - -use graph::blockchain::{Block as BlockchainBlock, BlockTime}; -use graph::{ - blockchain::BlockPtr, - prelude::{anyhow::anyhow, BlockNumber, Error}, -}; - -use std::convert::TryFrom; - -impl Block { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn begin_block_events(&self) -> Result, Error> { - let events = self - .result_begin_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_begin_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn end_block_events(&self) -> Result, Error> { - let events = self - .result_end_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_end_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn transactions(&self) -> impl Iterator { - self.transactions.iter() - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: Block) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a Block> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a Block) -> Result { - let header = b.header()?; - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } - - fn timestamp(&self) -> BlockTime { - let time = self.header().unwrap().time.as_ref().unwrap(); - BlockTime::since_epoch(time.seconds, time.nanos as u32) - } -} - -impl HeaderOnlyBlock { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl From<&Block> for HeaderOnlyBlock { - fn from(b: &Block) -> HeaderOnlyBlock { - HeaderOnlyBlock { - header: b.header.clone(), - } - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: HeaderOnlyBlock) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a HeaderOnlyBlock> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a HeaderOnlyBlock) -> Result { - let header = b.header()?; - - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for HeaderOnlyBlock { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } - - fn timestamp(&self) -> BlockTime { - let time = self.header().unwrap().time.as_ref().unwrap(); - BlockTime::since_epoch(time.seconds, time.nanos as u32) - } -} - -impl EventData { - pub fn event(&self) -> Result<&Event, Error> { - self.event - .as_ref() - .ok_or_else(|| anyhow!("event data missing event field")) - } - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("event data missing block field")) - } -} - -impl TransactionData { - pub fn tx_result(&self) -> Result<&TxResult, Error> { - self.tx - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing tx field")) - } - - pub fn response_deliver_tx(&self) -> Result<&ResponseDeliverTx, Error> { - self.tx_result()? - .result - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing result field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing block field")) - } -} - -impl MessageData { - pub fn message(&self) -> Result<&prost_types::Any, Error> { - self.message - .as_ref() - .ok_or_else(|| anyhow!("message data missing message field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("message data missing block field")) - } -} diff --git a/chain/cosmos/src/data_source.rs b/chain/cosmos/src/data_source.rs deleted file mode 100644 index f8827cefba3..00000000000 --- a/chain/cosmos/src/data_source.rs +++ /dev/null @@ -1,722 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{Context, Error, Result}; - -use graph::{ - blockchain::{self, Block, Blockchain, TriggerWithHandler}, - components::store::StoredDynamicDataSource, - data::subgraph::DataSourceContext, - prelude::{ - anyhow, async_trait, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, - LinkResolver, Logger, - }, -}; - -use crate::chain::Chain; -use crate::codec; -use crate::trigger::CosmosTrigger; - -pub const COSMOS_KIND: &str = "cosmos"; -const BLOCK_HANDLER_KIND: &str = "block"; -const EVENT_HANDLER_KIND: &str = "event"; -const TRANSACTION_HANDLER_KIND: &str = "transaction"; -const MESSAGE_HANDLER_KIND: &str = "message"; - -const DYNAMIC_DATA_SOURCE_ERROR: &str = "Cosmos subgraphs do not support dynamic data sources"; -const TEMPLATE_ERROR: &str = "Cosmos subgraphs do not support templates"; - -/// Runtime representation of a data source. -// Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens -// of thousands of data sources in memory at once. -#[derive(Clone, Debug)] -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } - - fn address(&self) -> Option<&[u8]> { - None - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn handler_kinds(&self) -> HashSet<&str> { - let mut kinds = HashSet::new(); - - let Mapping { - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - .. - } = &self.mapping; - - if !block_handlers.is_empty() { - kinds.insert(BLOCK_HANDLER_KIND); - } - - if !event_handlers.is_empty() { - kinds.insert(EVENT_HANDLER_KIND); - } - - if !transaction_handlers.is_empty() { - kinds.insert(TRANSACTION_HANDLER_KIND); - } - - if !message_handlers.is_empty() { - kinds.insert(MESSAGE_HANDLER_KIND); - } - - kinds - } - - fn end_block(&self) -> Option { - self.source.end_block - } - - fn match_and_decode( - &self, - trigger: &::TriggerData, - block: &Arc<::Block>, - _logger: &Logger, - ) -> Result>> { - if self.source.start_block > block.number() { - return Ok(None); - } - - let handler = match trigger { - CosmosTrigger::Block(_) => match self.handler_for_block() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Event { event_data, origin } => { - match self.handler_for_event(event_data.event()?, *origin) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - - CosmosTrigger::Transaction(_) => match self.handler_for_transaction() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Message(message_data) => { - match self.handler_for_message(message_data.message()?) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.cheap_clone(), - handler, - block.ptr(), - block.timestamp(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - self.creation_block - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - context, - - // The creation block is ignored for detection duplicate data sources. - // Contract ABI equality is implicit in `source` and `mapping.abis` equality. - creation_block: _, - } = self; - - // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs - // used at runtime but not needed to define uniqueness; each runtime host should be for a - // unique data source. - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.block_handlers == other.mapping.block_handlers - && mapping.event_handlers == other.mapping.event_handlers - && mapping.transaction_handlers == other.mapping.transaction_handlers - && mapping.message_handlers == other.mapping.message_handlers - && context == &other.context - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR); - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) - } - - fn validate(&self) -> Vec { - let mut errors = Vec::new(); - - if self.kind != COSMOS_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - COSMOS_KIND, - self.kind - )) - } - - // Ensure there is only one block handler - if self.mapping.block_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated block handlers")); - } - - // Ensure there is only one transaction handler - if self.mapping.transaction_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated transaction handlers")); - } - - // Ensure that each event type + origin filter combination has only one handler - - // group handler origin filters by event type - let mut event_types = HashMap::with_capacity(self.mapping.event_handlers.len()); - for event_handler in self.mapping.event_handlers.iter() { - let origins = event_types - .entry(&event_handler.event) - // 3 is the maximum number of valid handlers for an event type (1 for each origin) - .or_insert(HashSet::with_capacity(3)); - - // insert returns false if value was already in the set - if !origins.insert(event_handler.origin) { - errors.push(multiple_origin_err( - &event_handler.event, - event_handler.origin, - )) - } - } - - // Ensure each event type either has: - // 1 handler with no origin filter - // OR - // 1 or more handlers with origin filter - for (event_type, origins) in event_types.iter() { - if origins.len() > 1 && !origins.iter().all(Option::is_some) { - errors.push(combined_origins_err(event_type)) - } - } - - // Ensure each message handlers is unique - let mut message_type_urls = HashSet::with_capacity(self.mapping.message_handlers.len()); - for message_handler in self.mapping.message_handlers.iter() { - if !message_type_urls.insert(message_handler.message.clone()) { - errors.push(duplicate_url_type(&message_handler.message)) - } - } - - errors - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } -} - -impl DataSource { - fn from_manifest( - kind: String, - network: Option, - name: String, - source: Source, - mapping: Mapping, - context: Option, - ) -> Result { - // Data sources in the manifest are created "before genesis" so they have no creation block. - let creation_block = None; - - Ok(DataSource { - kind, - network, - name, - source, - mapping, - context: Arc::new(context), - creation_block, - }) - } - - fn handler_for_block(&self) -> Option { - self.mapping.block_handlers.first().cloned() - } - - fn handler_for_transaction(&self) -> Option { - self.mapping.transaction_handlers.first().cloned() - } - - fn handler_for_message(&self, message: &::prost_types::Any) -> Option { - self.mapping - .message_handlers - .iter() - .find(|handler| handler.message == message.type_url) - .cloned() - } - - fn handler_for_event( - &self, - event: &codec::Event, - event_origin: EventOrigin, - ) -> Option { - self.mapping - .event_handlers - .iter() - .find(|handler| { - let event_type_matches = event.event_type == handler.event; - - if let Some(handler_origin) = handler.origin { - event_type_matches && event_origin == handler_origin - } else { - event_type_matches - } - }) - .cloned() - } - - pub(crate) fn has_block_handler(&self) -> bool { - !self.mapping.block_handlers.is_empty() - } - - /// Return an iterator over all event types from event handlers. - pub(crate) fn events(&self) -> impl Iterator { - self.mapping - .event_handlers - .iter() - .map(|handler| handler.event.as_str()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSource { - kind, - network, - name, - source, - mapping, - context, - } = self; - - let mapping = mapping.resolve(resolver, logger).await.with_context(|| { - format!( - "failed to resolve data source {} with source {}", - name, source.start_block - ) - })?; - - DataSource::from_manifest(kind, network, name, source, mapping, context) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - async fn resolve( - self, - _resolver: &Arc, - _logger: &Logger, - _manifest_idx: u32, - ) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn name(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn api_version(&self) -> semver::Version { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn runtime(&self) -> Option>> { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn manifest_idx(&self) -> u32 { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn kind(&self) -> &str { - &self.kind - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - pub api_version: String, - pub language: String, - pub entities: Vec, - #[serde(default)] - pub block_handlers: Vec, - #[serde(default)] - pub event_handlers: Vec, - #[serde(default)] - pub transaction_handlers: Vec, - #[serde(default)] - pub message_handlers: Vec, - pub file: Link, -} - -impl UnresolvedMapping { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let UnresolvedMapping { - api_version, - language, - entities, - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - file: link, - } = self; - - let api_version = semver::Version::parse(&api_version)?; - - let module_bytes = resolver - .cat(logger, &link) - .await - .with_context(|| format!("failed to resolve mapping {}", link.link))?; - - Ok(Mapping { - api_version, - language, - entities, - block_handlers: block_handlers.clone(), - event_handlers: event_handlers.clone(), - transaction_handlers: transaction_handlers.clone(), - message_handlers: message_handlers.clone(), - runtime: Arc::new(module_bytes), - link, - }) - } -} - -#[derive(Clone, Debug)] -pub struct Mapping { - pub api_version: semver::Version, - pub language: String, - pub entities: Vec, - pub block_handlers: Vec, - pub event_handlers: Vec, - pub transaction_handlers: Vec, - pub message_handlers: Vec, - pub runtime: Arc>, - pub link: Link, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingEventHandler { - pub event: String, - pub origin: Option, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingTransactionHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingMessageHandler { - pub message: String, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Source { - #[serde(default)] - pub start_block: BlockNumber, - pub(crate) end_block: Option, -} - -#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Deserialize)] -pub enum EventOrigin { - BeginBlock, - DeliverTx, - EndBlock, -} - -fn multiple_origin_err(event_type: &str, origin: Option) -> Error { - let origin_err_name = match origin { - Some(origin) => format!("{:?}", origin), - None => "no".to_string(), - }; - - anyhow!( - "data source has multiple {} event handlers with {} origin", - event_type, - origin_err_name, - ) -} - -fn combined_origins_err(event_type: &str) -> Error { - anyhow!( - "data source has combined origin and no-origin {} event handlers", - event_type - ) -} - -fn duplicate_url_type(message: &str) -> Error { - anyhow!( - "data source has more than one message handler for message {} ", - message - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - use graph::blockchain::DataSource as _; - - #[test] - fn test_event_handlers_origin_validation() { - let cases = [ - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_3", None), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::EndBlock)), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - ]), - vec![ - multiple_origin_err("event_1", None), - combined_origins_err("event_2"), - multiple_origin_err("event_3", Some(EventOrigin::EndBlock)), - ], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - #[test] - fn test_message_handlers_duplicate() { - let cases = [ - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_1".to_string(), - }, - ]), - vec![], - ), - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - ]), - vec![duplicate_url_type("message_0")], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - impl DataSource { - fn with_event_handlers(event_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { - start_block: 1, - end_block: None, - }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers, - transaction_handlers: vec![], - message_handlers: vec![], - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - - fn with_message_handlers(message_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { - start_block: 1, - end_block: None, - }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers: vec![], - transaction_handlers: vec![], - message_handlers, - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - } - - impl MappingEventHandler { - fn with_origin(event_type: &str, origin: Option) -> MappingEventHandler { - MappingEventHandler { - event: event_type.to_string(), - origin, - handler: "handler".to_string(), - } - } - } -} diff --git a/chain/cosmos/src/lib.rs b/chain/cosmos/src/lib.rs deleted file mode 100644 index 5d0bf16d050..00000000000 --- a/chain/cosmos/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -mod adapter; -pub mod chain; -pub mod codec; -mod data_source; -mod protobuf; -pub mod runtime; -mod trigger; - -// ETHDEP: These concrete types should probably not be exposed. -pub use data_source::{DataSource, DataSourceTemplate}; - -pub use crate::adapter::TriggerFilter; -pub use crate::chain::Chain; - -pub use protobuf::pbcodec; -pub use protobuf::pbcodec::Block; diff --git a/chain/cosmos/src/protobuf/.gitignore b/chain/cosmos/src/protobuf/.gitignore deleted file mode 100644 index 96786948080..00000000000 --- a/chain/cosmos/src/protobuf/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/google.protobuf.rs -/gogoproto.rs -/cosmos_proto.rs -/firehose.rs diff --git a/chain/cosmos/src/protobuf/mod.rs b/chain/cosmos/src/protobuf/mod.rs deleted file mode 100644 index c3292e66c4b..00000000000 --- a/chain/cosmos/src/protobuf/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[rustfmt::skip] -#[path = "sf.cosmos.r#type.v1.rs"] -pub mod pbcodec; - -pub use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, AscString, Uint8Array}; - -pub use crate::runtime::abi::*; -pub use pbcodec::*; diff --git a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs b/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs deleted file mode 100644 index d60de8086b1..00000000000 --- a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs +++ /dev/null @@ -1,838 +0,0 @@ -#[graph_runtime_derive::generate_asc_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub last_commit: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result_begin_block: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub result_end_block: ::core::option::Option, - #[prost(message, repeated, tag = "7")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "8")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are -/// removed so that hydrating that object from a \[Block\] bytes payload will -/// drastically reduce allocated memory required to hold the full block. -/// -/// This can be used to unpack a \[Block\] when only the \[Header\] information -/// is required and greatly reduce required memory. -#[graph_runtime_derive::generate_asc_type(__required__{header:Header})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{header:Header})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, -} -#[graph_runtime_derive::generate_asc_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventData { - #[prost(message, optional, tag = "1")] - pub event: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionData { - #[prost(message, optional, tag = "1")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MessageData { - #[prost(message, optional, tag = "1")] - pub message: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionContext { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(uint32, tag = "3")] - pub code: u32, - #[prost(int64, tag = "4")] - pub gas_wanted: i64, - #[prost(int64, tag = "5")] - pub gas_used: i64, -} -#[graph_runtime_derive::generate_asc_type(__required__{last_block_id:BlockID})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{last_block_id:BlockID})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Header { - #[prost(message, optional, tag = "1")] - pub version: ::core::option::Option, - #[prost(string, tag = "2")] - pub chain_id: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub height: u64, - #[prost(message, optional, tag = "4")] - pub time: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub last_block_id: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub last_commit_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "7")] - pub data_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "8")] - pub validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "9")] - pub next_validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "10")] - pub consensus_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "11")] - pub app_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "12")] - pub last_results_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "13")] - pub evidence_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "14")] - pub proposer_address: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "15")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Consensus { - #[prost(uint64, tag = "1")] - pub block: u64, - #[prost(uint64, tag = "2")] - pub app: u64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Timestamp { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockId { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub part_set_header: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PartSetHeader { - #[prost(uint32, tag = "1")] - pub total: u32, - #[prost(bytes = "vec", tag = "2")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceList { - #[prost(message, repeated, tag = "1")] - pub evidence: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Evidence { - #[prost(oneof = "evidence::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `Evidence`. -pub mod evidence { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - DuplicateVoteEvidence(super::DuplicateVoteEvidence), - #[prost(message, tag = "2")] - LightClientAttackEvidence(super::LightClientAttackEvidence), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DuplicateVoteEvidence { - #[prost(message, optional, tag = "1")] - pub vote_a: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub vote_b: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, - #[prost(int64, tag = "4")] - pub validator_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventVote { - #[prost(enumeration = "SignedMsgType", tag = "1")] - pub event_vote_type: i32, - #[prost(uint64, tag = "2")] - pub height: u64, - #[prost(int32, tag = "3")] - pub round: i32, - #[prost(message, optional, tag = "4")] - pub block_id: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(int32, tag = "7")] - pub validator_index: i32, - #[prost(bytes = "vec", tag = "8")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightClientAttackEvidence { - #[prost(message, optional, tag = "1")] - pub conflicting_block: ::core::option::Option, - #[prost(int64, tag = "2")] - pub common_height: i64, - #[prost(message, repeated, tag = "3")] - pub byzantine_validators: ::prost::alloc::vec::Vec, - #[prost(int64, tag = "4")] - pub total_voting_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightBlock { - #[prost(message, optional, tag = "1")] - pub signed_header: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub validator_set: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedHeader { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub commit: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Commit { - #[prost(int64, tag = "1")] - pub height: i64, - #[prost(int32, tag = "2")] - pub round: i32, - #[prost(message, optional, tag = "3")] - pub block_id: ::core::option::Option, - #[prost(message, repeated, tag = "4")] - pub signatures: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CommitSig { - #[prost(enumeration = "BlockIdFlag", tag = "1")] - pub block_id_flag: i32, - #[prost(bytes = "vec", tag = "2")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "3")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "4")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSet { - #[prost(message, repeated, tag = "1")] - pub validators: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub proposer: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Validator { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub voting_power: i64, - #[prost(int64, tag = "4")] - pub proposer_priority: i64, -} -#[graph_runtime_derive::generate_asc_type(sum{ed25519:Vec, secp256k1:Vec})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(sum{ed25519:Vec, secp256k1:Vec})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PublicKey { - #[prost(oneof = "public_key::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `PublicKey`. -pub mod public_key { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(bytes, tag = "1")] - Ed25519(::prost::alloc::vec::Vec), - #[prost(bytes, tag = "2")] - Secp256k1(::prost::alloc::vec::Vec), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseBeginBlock { - #[prost(message, repeated, tag = "1")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Event { - #[prost(string, tag = "1")] - pub event_type: ::prost::alloc::string::String, - #[prost(message, repeated, tag = "2")] - pub attributes: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventAttribute { - #[prost(string, tag = "1")] - pub key: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub value: ::prost::alloc::string::String, - #[prost(bool, tag = "3")] - pub index: bool, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseEndBlock { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub consensus_param_updates: ::core::option::Option, - #[prost(message, repeated, tag = "3")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorUpdate { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ConsensusParams { - #[prost(message, optional, tag = "1")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub validator: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub version: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockParams { - #[prost(int64, tag = "1")] - pub max_bytes: i64, - #[prost(int64, tag = "2")] - pub max_gas: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceParams { - #[prost(int64, tag = "1")] - pub max_age_num_blocks: i64, - #[prost(message, optional, tag = "2")] - pub max_age_duration: ::core::option::Option, - #[prost(int64, tag = "3")] - pub max_bytes: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Duration { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorParams { - #[prost(string, repeated, tag = "1")] - pub pub_key_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct VersionParams { - #[prost(uint64, tag = "1")] - pub app_version: u64, -} -#[graph_runtime_derive::generate_asc_type(__required__{tx:Tx, result:ResponseDeliverTx})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:Tx, - result:ResponseDeliverTx} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxResult { - #[prost(uint64, tag = "1")] - pub height: u64, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result: ::core::option::Option, - #[prost(bytes = "vec", tag = "5")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type(__required__{body:TxBody})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{body:TxBody})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tx { - #[prost(message, optional, tag = "1")] - pub body: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub auth_info: ::core::option::Option, - #[prost(bytes = "vec", repeated, tag = "3")] - pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxBody { - #[prost(message, repeated, tag = "1")] - pub messages: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(string, tag = "2")] - pub memo: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub timeout_height: u64, - #[prost(message, repeated, tag = "1023")] - pub extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(message, repeated, tag = "2047")] - pub non_critical_extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Any { - #[prost(string, tag = "1")] - pub type_url: ::prost::alloc::string::String, - #[prost(bytes = "vec", tag = "2")] - pub value: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AuthInfo { - #[prost(message, repeated, tag = "1")] - pub signer_infos: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub fee: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tip: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignerInfo { - #[prost(message, optional, tag = "1")] - pub public_key: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub mode_info: ::core::option::Option, - #[prost(uint64, tag = "3")] - pub sequence: u64, -} -#[graph_runtime_derive::generate_asc_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfo { - #[prost(oneof = "mode_info::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `ModeInfo`. -pub mod mode_info { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - Single(super::ModeInfoSingle), - #[prost(message, tag = "2")] - Multi(super::ModeInfoMulti), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoSingle { - #[prost(enumeration = "SignMode", tag = "1")] - pub mode: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoMulti { - #[prost(message, optional, tag = "1")] - pub bitarray: ::core::option::Option, - #[prost(message, repeated, tag = "2")] - pub mode_infos: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CompactBitArray { - #[prost(uint32, tag = "1")] - pub extra_bits_stored: u32, - #[prost(bytes = "vec", tag = "2")] - pub elems: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Fee { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "2")] - pub gas_limit: u64, - #[prost(string, tag = "3")] - pub payer: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub granter: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Coin { - #[prost(string, tag = "1")] - pub denom: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub amount: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tip { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(string, tag = "2")] - pub tipper: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseDeliverTx { - #[prost(uint32, tag = "1")] - pub code: u32, - #[prost(bytes = "vec", tag = "2")] - pub data: ::prost::alloc::vec::Vec, - #[prost(string, tag = "3")] - pub log: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub info: ::prost::alloc::string::String, - #[prost(int64, tag = "5")] - pub gas_wanted: i64, - #[prost(int64, tag = "6")] - pub gas_used: i64, - #[prost(message, repeated, tag = "7")] - pub events: ::prost::alloc::vec::Vec, - #[prost(string, tag = "8")] - pub codespace: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSetUpdates { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignedMsgType { - Unknown = 0, - Prevote = 1, - Precommit = 2, - Proposal = 32, -} -impl SignedMsgType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignedMsgType::Unknown => "SIGNED_MSG_TYPE_UNKNOWN", - SignedMsgType::Prevote => "SIGNED_MSG_TYPE_PREVOTE", - SignedMsgType::Precommit => "SIGNED_MSG_TYPE_PRECOMMIT", - SignedMsgType::Proposal => "SIGNED_MSG_TYPE_PROPOSAL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGNED_MSG_TYPE_UNKNOWN" => Some(Self::Unknown), - "SIGNED_MSG_TYPE_PREVOTE" => Some(Self::Prevote), - "SIGNED_MSG_TYPE_PRECOMMIT" => Some(Self::Precommit), - "SIGNED_MSG_TYPE_PROPOSAL" => Some(Self::Proposal), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum BlockIdFlag { - Unknown = 0, - Absent = 1, - Commit = 2, - Nil = 3, -} -impl BlockIdFlag { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - BlockIdFlag::Unknown => "BLOCK_ID_FLAG_UNKNOWN", - BlockIdFlag::Absent => "BLOCK_ID_FLAG_ABSENT", - BlockIdFlag::Commit => "BLOCK_ID_FLAG_COMMIT", - BlockIdFlag::Nil => "BLOCK_ID_FLAG_NIL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "BLOCK_ID_FLAG_UNKNOWN" => Some(Self::Unknown), - "BLOCK_ID_FLAG_ABSENT" => Some(Self::Absent), - "BLOCK_ID_FLAG_COMMIT" => Some(Self::Commit), - "BLOCK_ID_FLAG_NIL" => Some(Self::Nil), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignMode { - Unspecified = 0, - Direct = 1, - Textual = 2, - LegacyAminoJson = 127, -} -impl SignMode { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignMode::Unspecified => "SIGN_MODE_UNSPECIFIED", - SignMode::Direct => "SIGN_MODE_DIRECT", - SignMode::Textual => "SIGN_MODE_TEXTUAL", - SignMode::LegacyAminoJson => "SIGN_MODE_LEGACY_AMINO_JSON", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGN_MODE_UNSPECIFIED" => Some(Self::Unspecified), - "SIGN_MODE_DIRECT" => Some(Self::Direct), - "SIGN_MODE_TEXTUAL" => Some(Self::Textual), - "SIGN_MODE_LEGACY_AMINO_JSON" => Some(Self::LegacyAminoJson), - _ => None, - } - } -} diff --git a/chain/cosmos/src/runtime/abi.rs b/chain/cosmos/src/runtime/abi.rs deleted file mode 100644 index af9260b63be..00000000000 --- a/chain/cosmos/src/runtime/abi.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::protobuf::*; -use graph::runtime::HostExportError; -pub use graph::semver::Version; - -pub use graph::runtime::{ - asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, - DeterministicHostError, IndexForAscTypeId, ToAscObj, -}; -/* -TODO: AscBytesArray seem to be generic to all chains, but AscIndexId pins it to Cosmos -****************** this can be moved to runtime graph/runtime/src/asc_heap.rs, but IndexForAscTypeId::CosmosBytesArray ****** -*/ -pub struct AscBytesArray(pub Array>); - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self - .iter() - .map(|x| asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(x), gas)) - .collect(); - - Ok(AscBytesArray(Array::new(&content?, heap, gas)?)) - } -} - -//this can be moved to runtime -impl AscType for AscBytesArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -//we will have to keep this chain specific (Inner/Outer) -impl AscIndexId for AscBytesArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::CosmosBytesArray; -} - -/************************************************************************** */ -// this can be moved to runtime - prost_types::Any -impl ToAscObj for prost_types::Any { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscAny { - type_url: asc_new(heap, &self.type_url, gas)?, - value: asc_new( - heap, - &graph_runtime_wasm::asc_abi::class::Bytes(&self.value), - gas, - )?, - ..Default::default() - }) - } -} - -//this can be moved to runtime - prost_types::Any -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - - Ok(AscAnyArray(Array::new(&content?, heap, gas)?)) - } -} diff --git a/chain/cosmos/src/runtime/mod.rs b/chain/cosmos/src/runtime/mod.rs deleted file mode 100644 index 78d10d4d7d3..00000000000 --- a/chain/cosmos/src/runtime/mod.rs +++ /dev/null @@ -1,348 +0,0 @@ -pub mod abi; - -#[cfg(test)] -mod test { - use crate::protobuf::*; - - use graph::semver::Version; - - /// A macro that takes an ASC struct value definition and calls AscBytes methods to check that - /// memory layout is padded properly. - macro_rules! assert_asc_bytes { - ($struct_name:ident { - $($field:ident : $field_value:expr),+ - $(,)? // trailing - }) => { - let value = $struct_name { - $($field: $field_value),+ - }; - - // just call the function. it will panic on misalignments - let asc_bytes = value.to_asc_bytes().unwrap(); - - let value_004 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 4)).unwrap(); - let value_005 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 5)).unwrap(); - - // turn the values into bytes again to verify that they are the same as the original - // because these types usually don't implement PartialEq - assert_eq!( - asc_bytes, - value_004.to_asc_bytes().unwrap(), - "Expected {} v0.0.4 asc bytes to be the same", - stringify!($struct_name) - ); - assert_eq!( - asc_bytes, - value_005.to_asc_bytes().unwrap(), - "Expected {} v0.0.5 asc bytes to be the same", - stringify!($struct_name) - ); - }; - } - - #[test] - fn test_asc_type_alignment() { - // TODO: automatically generate these tests for each struct in derive(AscType) macro - - assert_asc_bytes!(AscBlock { - header: new_asc_ptr(), - evidence: new_asc_ptr(), - last_commit: new_asc_ptr(), - result_begin_block: new_asc_ptr(), - result_end_block: new_asc_ptr(), - transactions: new_asc_ptr(), - validator_updates: new_asc_ptr(), - }); - - assert_asc_bytes!(AscHeaderOnlyBlock { - header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventData { - event: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionData { - tx: new_asc_ptr(), - block: new_asc_ptr(), - }); - - assert_asc_bytes!(AscMessageData { - message: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionContext { - hash: new_asc_ptr(), - index: 20, - code: 20, - gas_wanted: 20, - gas_used: 20, - }); - - assert_asc_bytes!(AscHeader { - version: new_asc_ptr(), - chain_id: new_asc_ptr(), - height: 20, - time: new_asc_ptr(), - last_block_id: new_asc_ptr(), - last_commit_hash: new_asc_ptr(), - data_hash: new_asc_ptr(), - validators_hash: new_asc_ptr(), - next_validators_hash: new_asc_ptr(), - consensus_hash: new_asc_ptr(), - app_hash: new_asc_ptr(), - last_results_hash: new_asc_ptr(), - evidence_hash: new_asc_ptr(), - proposer_address: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscConsensus { block: 0, app: 0 }); - - assert_asc_bytes!(AscTimestamp { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscBlockId { - hash: new_asc_ptr(), - part_set_header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscPartSetHeader { - total: 20, - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidenceList { - evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidence { - duplicate_vote_evidence: new_asc_ptr(), - light_client_attack_evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscDuplicateVoteEvidence { - vote_a: new_asc_ptr(), - vote_b: new_asc_ptr(), - total_voting_power: 20, - validator_power: 20, - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventVote { - event_vote_type: 20, - height: 20, - round: 20, - block_id: new_asc_ptr(), - timestamp: new_asc_ptr(), - validator_address: new_asc_ptr(), - validator_index: 20, - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightClientAttackEvidence { - conflicting_block: new_asc_ptr(), - common_height: 20, - total_voting_power: 20, - byzantine_validators: new_asc_ptr(), - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightBlock { - signed_header: new_asc_ptr(), - validator_set: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignedHeader { - header: new_asc_ptr(), - commit: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommit { - height: 20, - round: 20, - block_id: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommitSig { - block_id_flag: 20, - validator_address: new_asc_ptr(), - timestamp: new_asc_ptr(), - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSet { - validators: new_asc_ptr(), - proposer: new_asc_ptr(), - total_voting_power: 20, - }); - - assert_asc_bytes!(AscValidator { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - voting_power: 20, - proposer_priority: 20, - }); - - assert_asc_bytes!(AscPublicKey { - ed25519: new_asc_ptr(), - secp256k1: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseBeginBlock { - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvent { - event_type: new_asc_ptr(), - attributes: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventAttribute { - key: new_asc_ptr(), - value: new_asc_ptr(), - index: true, - }); - - assert_asc_bytes!(AscResponseEndBlock { - validator_updates: new_asc_ptr(), - consensus_param_updates: new_asc_ptr(), - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorUpdate { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - power: 20, - }); - - assert_asc_bytes!(AscConsensusParams { - block: new_asc_ptr(), - evidence: new_asc_ptr(), - validator: new_asc_ptr(), - version: new_asc_ptr(), - }); - - assert_asc_bytes!(AscBlockParams { - max_bytes: 20, - max_gas: 20, - }); - - assert_asc_bytes!(AscEvidenceParams { - max_age_num_blocks: 20, - max_age_duration: new_asc_ptr(), - max_bytes: 20, - }); - - assert_asc_bytes!(AscDuration { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscValidatorParams { - pub_key_types: new_asc_ptr(), - }); - - assert_asc_bytes!(AscVersionParams { app_version: 20 }); - - assert_asc_bytes!(AscTxResult { - height: 20, - index: 20, - tx: new_asc_ptr(), - result: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTx { - body: new_asc_ptr(), - auth_info: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTxBody { - messages: new_asc_ptr(), - memo: new_asc_ptr(), - timeout_height: 20, - extension_options: new_asc_ptr(), - non_critical_extension_options: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAny { - type_url: new_asc_ptr(), - value: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAuthInfo { - signer_infos: new_asc_ptr(), - fee: new_asc_ptr(), - tip: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignerInfo { - public_key: new_asc_ptr(), - mode_info: new_asc_ptr(), - sequence: 20, - }); - - assert_asc_bytes!(AscModeInfo { - single: new_asc_ptr(), - multi: new_asc_ptr(), - }); - - assert_asc_bytes!(AscModeInfoSingle { mode: 20 }); - - assert_asc_bytes!(AscModeInfoMulti { - bitarray: new_asc_ptr(), - mode_infos: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCompactBitArray { - extra_bits_stored: 20, - elems: new_asc_ptr(), - }); - - assert_asc_bytes!(AscFee { - amount: new_asc_ptr(), - gas_limit: 20, - payer: new_asc_ptr(), - granter: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCoin { - denom: new_asc_ptr(), - amount: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTip { - amount: new_asc_ptr(), - tipper: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseDeliverTx { - code: 20, - data: new_asc_ptr(), - log: new_asc_ptr(), - info: new_asc_ptr(), - gas_wanted: 20, - gas_used: 20, - events: new_asc_ptr(), - codespace: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSetUpdates { - validator_updates: new_asc_ptr(), - }); - } - - // non-null AscPtr - fn new_asc_ptr() -> AscPtr { - AscPtr::new(12) - } -} diff --git a/chain/cosmos/src/trigger.rs b/chain/cosmos/src/trigger.rs deleted file mode 100644 index 0f692a695f9..00000000000 --- a/chain/cosmos/src/trigger.rs +++ /dev/null @@ -1,365 +0,0 @@ -use std::{cmp::Ordering, sync::Arc}; - -use graph::blockchain::{Block, BlockHash, MappingTriggerTrait, TriggerData}; -use graph::cheap_clone::CheapClone; -use graph::prelude::{BlockNumber, Error}; -use graph::runtime::HostExportError; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr}; -use graph_runtime_wasm::module::ToAscPtr; - -use crate::codec; -use crate::data_source::EventOrigin; - -// Logging the block is too verbose, so this strips the block from the trigger for Debug. -impl std::fmt::Debug for CosmosTrigger { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[derive(Debug)] - pub enum MappingTriggerWithoutBlock<'e> { - Block, - Event { - event_type: &'e str, - origin: EventOrigin, - }, - Transaction, - Message, - } - - let trigger_without_block = match self { - CosmosTrigger::Block(_) => MappingTriggerWithoutBlock::Block, - CosmosTrigger::Event { event_data, origin } => MappingTriggerWithoutBlock::Event { - event_type: &event_data.event().map_err(|_| std::fmt::Error)?.event_type, - origin: *origin, - }, - CosmosTrigger::Transaction(_) => MappingTriggerWithoutBlock::Transaction, - CosmosTrigger::Message(_) => MappingTriggerWithoutBlock::Message, - }; - - write!(f, "{:?}", trigger_without_block) - } -} - -impl ToAscPtr for CosmosTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - Ok(match self { - CosmosTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - CosmosTrigger::Event { event_data, .. } => { - asc_new(heap, event_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Transaction(transaction_data) => { - asc_new(heap, transaction_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Message(message_data) => { - asc_new(heap, message_data.as_ref(), gas)?.erase() - } - }) - } -} - -#[derive(Clone)] -pub enum CosmosTrigger { - Block(Arc), - Event { - event_data: Arc, - origin: EventOrigin, - }, - Transaction(Arc), - Message(Arc), -} - -impl CheapClone for CosmosTrigger {} - -impl PartialEq for CosmosTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, - ( - Self::Event { - event_data: a_event_data, - origin: a_origin, - }, - Self::Event { - event_data: b_event_data, - origin: b_origin, - }, - ) => { - if let (Ok(a_event), Ok(b_event)) = (a_event_data.event(), b_event_data.event()) { - let mut attributes_a = a_event.attributes.clone(); - attributes_a.sort_by(|a, b| a.key.cmp(&b.key)); - - let mut attributes_b = b_event.attributes.clone(); - attributes_b.sort_by(|a, b| a.key.cmp(&b.key)); - - a_event.event_type == b_event.event_type - && a_origin == b_origin - && attributes_a == attributes_b - } else { - false - } - } - (Self::Transaction(a_ptr), Self::Transaction(b_ptr)) => a_ptr == b_ptr, - (Self::Message(a_ptr), Self::Message(b_ptr)) => a_ptr == b_ptr, - _ => false, - } - } -} - -impl Eq for CosmosTrigger {} - -impl CosmosTrigger { - pub(crate) fn with_event( - event: codec::Event, - block: codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, - ) -> CosmosTrigger { - CosmosTrigger::Event { - event_data: Arc::new(codec::EventData { - event: Some(event), - block: Some(block), - tx: tx_context, - }), - origin, - } - } - - pub(crate) fn with_transaction( - tx_result: codec::TxResult, - block: codec::HeaderOnlyBlock, - ) -> CosmosTrigger { - CosmosTrigger::Transaction(Arc::new(codec::TransactionData { - tx: Some(tx_result), - block: Some(block), - })) - } - - pub(crate) fn with_message( - message: ::prost_types::Any, - block: codec::HeaderOnlyBlock, - tx_context: codec::TransactionContext, - ) -> CosmosTrigger { - CosmosTrigger::Message(Arc::new(codec::MessageData { - message: Some(message), - block: Some(block), - tx: Some(tx_context), - })) - } - - pub fn block_number(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.number()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.number()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.number()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.number()), - } - } - - pub fn block_hash(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.hash()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.hash()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.hash()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.hash()), - } - } - - fn error_context(&self) -> std::string::String { - match self { - CosmosTrigger::Block(..) => { - if let (Ok(block_number), Ok(block_hash)) = (self.block_number(), self.block_hash()) - { - format!("block #{block_number}, hash {block_hash}") - } else { - "block".to_string() - } - } - CosmosTrigger::Event { event_data, origin } => { - if let (Ok(event), Ok(block_number), Ok(block_hash)) = - (event_data.event(), self.block_number(), self.block_hash()) - { - format!( - "event type {}, origin: {:?}, block #{block_number}, hash {block_hash}", - event.event_type, origin, - ) - } else { - "event".to_string() - } - } - CosmosTrigger::Transaction(transaction_data) => { - if let (Ok(block_number), Ok(block_hash), Ok(response_deliver_tx)) = ( - self.block_number(), - self.block_hash(), - transaction_data.response_deliver_tx(), - ) { - format!( - "block #{block_number}, hash {block_hash}, transaction log: {}", - response_deliver_tx.log - ) - } else { - "transaction".to_string() - } - } - CosmosTrigger::Message(message_data) => { - if let (Ok(message), Ok(block_number), Ok(block_hash)) = ( - message_data.message(), - self.block_number(), - self.block_hash(), - ) { - format!( - "message type {}, block #{block_number}, hash {block_hash}", - message.type_url, - ) - } else { - "message".to_string() - } - } - } - } -} - -impl Ord for CosmosTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - // Events have no intrinsic ordering information, so we keep the order in - // which they are included in the `events` field - (Self::Event { .. }, Self::Event { .. }) => Ordering::Equal, - - // Keep the order when comparing two message triggers - (Self::Message(..), Self::Message(..)) => Ordering::Equal, - - // Transactions are ordered by their index inside the block - (Self::Transaction(a), Self::Transaction(b)) => { - if let (Ok(a_tx_result), Ok(b_tx_result)) = (a.tx_result(), b.tx_result()) { - a_tx_result.index.cmp(&b_tx_result.index) - } else { - Ordering::Equal - } - } - - // Keep the order when comparing two block triggers - (Self::Block(..), Self::Block(..)) => Ordering::Equal, - - // Event triggers always come first - (Self::Event { .. }, _) => Ordering::Greater, - (_, Self::Event { .. }) => Ordering::Less, - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Less, - (_, Self::Block(..)) => Ordering::Greater, - - // Message triggers before Transaction triggers - (Self::Message(..), Self::Transaction(..)) => Ordering::Greater, - (Self::Transaction(..), Self::Message(..)) => Ordering::Less, - } - } -} - -impl PartialOrd for CosmosTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl TriggerData for CosmosTrigger { - fn error_context(&self) -> String { - self.error_context() - } - - fn address_match(&self) -> Option<&[u8]> { - None - } -} - -impl MappingTriggerTrait for CosmosTrigger { - fn error_context(&self) -> String { - self.error_context() - } -} -#[cfg(test)] -mod tests { - use crate::codec::TxResult; - - use super::*; - - #[test] - fn test_cosmos_trigger_ordering() { - let event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let other_event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let other_message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let transaction_trigger = CosmosTrigger::Transaction(Arc::::new( - codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 1, - ..Default::default() - }), - }, - )); - let other_transaction_trigger = CosmosTrigger::Transaction( - Arc::::new(codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 2, - ..Default::default() - }), - }), - ); - let block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - let other_block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - - assert_eq!(event_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&message_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&other_event_trigger), Ordering::Equal); - - assert_eq!(message_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&other_message_trigger), Ordering::Equal); - assert_eq!(message_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(transaction_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!( - transaction_trigger.cmp(&other_transaction_trigger), - Ordering::Less - ); - assert_eq!( - other_transaction_trigger.cmp(&transaction_trigger), - Ordering::Greater - ); - assert_eq!(transaction_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(transaction_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(block_trigger.cmp(&other_block_trigger), Ordering::Equal); - assert_eq!(block_trigger.cmp(&transaction_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&event_trigger), Ordering::Less); - } -} diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml index 84bb1192903..a9c9d42a124 100644 --- a/chain/ethereum/Cargo.toml +++ b/chain/ethereum/Cargo.toml @@ -4,29 +4,32 @@ version.workspace = true edition.workspace = true [dependencies] -envconfig = "0.10.0" -futures = "0.1.21" -http = "0.2.4" +async-trait = { workspace = true } +envconfig = { workspace = true } jsonrpc-core = "18.0.0" graph = { path = "../../graph" } -lazy_static = "1.2.0" -serde = "1.0" +serde = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } -dirs-next = "2.0" anyhow = "1.0" -tiny-keccak = "1.5.0" hex = "0.4.3" -semver = "1.0.21" +semver = { workspace = true } +thiserror = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tower = { workspace = true } -itertools = "0.12.0" +itertools = "0.15.0" graph-runtime-wasm = { path = "../../runtime/wasm" } graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] -base64 = "0.20.0" -uuid = { version = "1.6.1", features = ["v4"] } +base64 = "0" [build-dependencies] -tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } + +[lints] +workspace = true diff --git a/chain/ethereum/build.rs b/chain/ethereum/build.rs index 8ccae67aa92..f9d8eb0101b 100644 --- a/chain/ethereum/build.rs +++ b/chain/ethereum/build.rs @@ -1,8 +1,9 @@ fn main() { println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() + tonic_prost_build::configure() .out_dir("src/protobuf") - .compile(&["proto/ethereum.proto"], &["proto"]) + .protoc_arg("--experimental_allow_proto3_optional") + .compile_protos(&["proto/ethereum.proto"], &["proto"]) .expect("Failed to compile Firehose Ethereum proto(s)"); } diff --git a/chain/ethereum/examples/firehose.rs b/chain/ethereum/examples/firehose.rs index fbdfaa1f141..3dd21859653 100644 --- a/chain/ethereum/examples/firehose.rs +++ b/chain/ethereum/examples/firehose.rs @@ -2,14 +2,14 @@ use anyhow::Error; use graph::{ endpoint::EndpointMetrics, env::env_var, - firehose::SubgraphLimit, + firehose::{self, FirehoseEndpoint, SubgraphLimit}, log::logger, - prelude::{prost, tokio, tonic, MetricsRegistry}, - {firehose, firehose::FirehoseEndpoint}, + prelude::{MetricsRegistry, prost, tokio, tonic}, }; use graph_chain_ethereum::codec; use hex::ToHex; use prost::Message; +use std::slice; use std::sync::Arc; use tonic::Streaming; @@ -26,7 +26,7 @@ async fn main() -> Result<(), Error> { let host = "https://api.streamingfast.io:443".to_string(); let metrics = Arc::new(EndpointMetrics::new( logger, - &[host.clone()], + slice::from_ref(&host), Arc::new(MetricsRegistry::mock()), )); @@ -34,6 +34,7 @@ async fn main() -> Result<(), Error> { "firehose", &host, token, + None, false, false, SubgraphLimit::Unlimited, @@ -44,16 +45,19 @@ async fn main() -> Result<(), Error> { println!("Connecting to the stream!"); let mut stream: Streaming = match firehose .clone() - .stream_blocks(firehose::Request { - start_block_num: 12369739, - stop_block_num: 12369739, - cursor: match &cursor { - Some(c) => c.clone(), - None => String::from(""), + .stream_blocks( + firehose::Request { + start_block_num: 12369739, + stop_block_num: 12369739, + cursor: match &cursor { + Some(c) => c.clone(), + None => String::from(""), + }, + final_blocks_only: false, + ..Default::default() }, - final_blocks_only: false, - ..Default::default() - }) + &firehose::ConnectionHeaders::new(), + ) .await { Ok(s) => s, diff --git a/chain/ethereum/proto/ethereum.proto b/chain/ethereum/proto/ethereum.proto index 3c9f7378c7d..50c10f921f0 100644 --- a/chain/ethereum/proto/ethereum.proto +++ b/chain/ethereum/proto/ethereum.proto @@ -2,64 +2,144 @@ syntax = "proto3"; package sf.ethereum.type.v2; -option go_package = "github.com/streamingfast/sf-ethereum/types/pb/sf/ethereum/type/v2;pbeth"; +option go_package = "github.com/streamingfast/firehose-ethereum/types/pb/sf/ethereum/type/v2;pbeth"; import "google/protobuf/timestamp.proto"; +// Block is the representation of the tracing of a block in the Ethereum +// blockchain. A block is a collection of [TransactionTrace] that are grouped +// together and processed as an atomic unit. Each [TransactionTrace] is composed +// of a series of [Call] (a.k.a internal transactions) and there is also at +// least one call per transaction a.k.a the root call which essentially has the +// same parameters as the transaction itself (e.g. `from`, `to`, `gas`, `value`, +// etc.). +// +// The exact tracing method used to build the block must be checked against +// [DetailLevel] field. There is two levels of details available, `BASE` and +// `EXTENDED`. The `BASE` level has been extracted using archive node RPC calls +// and will contain only the block header, transaction receipts and event logs. +// Refers to the Firehose service provider to know which blocks are offered on +// each network. +// +// The `EXTENDED` level has been extracted using the Firehose tracer and all +// fields are available in this Protobuf. +// +// The Ethereum block model is used across many chains which means that it +// happen that certain fields are not available in one chain but are available +// in another. Each field should be documented when necesssary if it's available +// on a subset of chains. +// +// One major concept to get about the Block is the concept of 'ordinal'. The +// ordinal is a number that is used to globally order every element of execution +// that happened throughout the processing of the block like +// [TransactionTracer], [Call], [Log], [BalanceChange], [StateChange], etc. +// Element that have a start and end interval, [Transaction] and [Call], will +// have two ordinals: `begin_ordinal` and `end_ordinal`. Element that are +// executed as "point in time" [Log], [BalanceChange], [StateChange], etc. will +// have only one ordinal named `ordinal`. If you take all of the message in the +// Block that have an 'ordinal' field in an array and you sort each element +// against the `ordinal` field, you will get the exact order of execution of +// each element in the block. +// +// All the 'ordinal' fields in a block are globally unique for the given block, +// it is **not** a chain-wide global ordering. Furthermore, caution must be take +// with reverted elements due to execution failure. For anything attached to a +// [Call] that has a `state_reverted` field set to `true`, the `ordinal` field +// is not reliable and should not be used to order the element against other +// elements in the block as those element might have 0 as the ordinal. Only +// successful calls have a reliable `ordinal` field. message Block { - int32 ver = 1; + // Hash is the block's hash. bytes hash = 2; + // Number is the block's height at which this block was mined. uint64 number = 3; + // Size is the size in bytes of the RLP encoding of the block according to Ethereum + // rules. uint64 size = 4; + // Header contain's the block's header information like its parent hash, the merkel root hash + // and all other information the form a block. BlockHeader header = 5; - // Uncles represents block produced with a valid solution but were not actually choosen + // Uncles represents block produced with a valid solution but were not actually chosen // as the canonical block for the given height so they are mostly "forked" blocks. // // If the Block has been produced using the Proof of Stake consensus algorithm, this // field will actually be always empty. repeated BlockHeader uncles = 6; + // TransactionTraces hold the execute trace of all the transactions that were executed + // in this block. In in there that you will find most of the Ethereum data model. + // + // They are ordered by the order of execution of the transaction in the block. repeated TransactionTrace transaction_traces = 10; + + // BalanceChanges here is the array of ETH transfer that happened at the block level + // outside of the normal transaction flow of a block. The best example of this is mining + // reward for the block mined, the transfer of ETH to the miner happens outside the normal + // transaction flow of the chain and is recorded as a `BalanceChange` here since we cannot + // attached it to any transaction. + // + // Only available in DetailLevel: EXTENDED repeated BalanceChange balance_changes = 11; - repeated CodeChange code_changes = 20; - reserved 40; // bool filtering_applied = 40 [deprecated = true]; - reserved 41; // string filtering_include_filter_expr = 41 [deprecated = true]; - reserved 42; // string filtering_exclude_filter_expr = 42 [deprecated = true]; -} + enum DetailLevel{ + DETAILLEVEL_EXTENDED = 0; + // DETAILLEVEL_TRACE = 1; // TBD + DETAILLEVEL_BASE = 2; + } -// HeaderOnlyBlock is used to optimally unpack the [Block] structure (note the -// corresponding message number for the `header` field) while consuming less -// memory, when only the `header` is desired. -// -// WARN: this is a client-side optimization pattern and should be moved in the -// consuming code. -message HeaderOnlyBlock { - BlockHeader header = 5; -} + // DetailLevel affects the data available in this block. + // + // ## DetailLevel_EXTENDED + // + // Describes the most complete block, with traces, balance changes, storage + // changes. It is extracted during the execution of the block. + // + // ## DetailLevel_BASE + // + // Describes a block that contains only the block header, transaction receipts + // and event logs: everything that can be extracted using the base JSON-RPC + // interface + // (https://ethereum.org/en/developers/docs/apis/json-rpc/#json-rpc-methods) + // Furthermore, the eth_getTransactionReceipt call has been avoided because it + // brings only minimal improvements at the cost of requiring an archive node + // or a full node with complete transaction index. + DetailLevel detail_level = 12; + + // CodeChanges here is the array of smart code change that happened that happened at the block level + // outside of the normal transaction flow of a block. Some Ethereum's fork like BSC and Polygon + // has some capabilities to upgrade internal smart contracts used usually to track the validator + // list. + // + // On hard fork, some procedure runs to upgrade the smart contract code to a new version. In those + // network, a `CodeChange` for each modified smart contract on upgrade would be present here. Note + // that this happen rarely, so the vast majority of block will have an empty list here. + // + // Only available in DetailLevel: EXTENDED + repeated CodeChange code_changes = 20; -// BlockWithRefs is a lightweight block, with traces and transactions -// purged from the `block` within, and only. It is used in transports -// to pass block data around. -message BlockWithRefs { - string id = 1; - Block block = 2; - TransactionRefs transaction_trace_refs = 3; - bool irreversible = 4; -} + // System calls are introduced in Cancun, along with blobs. They are executed outside of transactions but affect the state. + // + // Only available in DetailLevel: EXTENDED + repeated Call system_calls = 21; -message TransactionRefs { - repeated bytes hashes = 1; -} + // Withdrawals represents the list of validator balance withdrawals processed in this block. + // Introduced in the Shanghai hard fork (EIP-4895). + // + // This field has been added because Geth blocks include withdrawals after Shanghai fork, + // but our previous Firehose model didn't capture this data. Currently experimental - + // NOT ready for production use yet as we validate the tracing implementation. + // + // Only available when Shanghai fork is active on the chain. + repeated Withdrawal withdrawals = 22; -message UnclesHeaders { - repeated BlockHeader uncles = 1; -} + reserved 40; // bool filtering_applied = 40 [deprecated = true]; + reserved 41; // string filtering_include_filter_expr = 41 [deprecated = true]; + reserved 42; // string filtering_exclude_filter_expr = 42 [deprecated = true]; -message BlockRef { - bytes hash = 1; - uint64 number = 2; + // Ver represents that data model version of the block, it is used internally by Firehose on Ethereum + // as a validation that we are reading the correct version. + int32 ver = 1; } message BlockHeader { @@ -84,13 +164,10 @@ message BlockHeader { // consensus algorithm, this field will actually be constant and set to `0x00`. BigInt difficulty = 8; - // TotalDifficulty is the sum of all previous blocks difficulty including this block difficulty. + // TotalDifficulty used to be the sum of all previous blocks difficulty including this block difficulty. // - // If the Block containing this `BlockHeader` has been produced using the Proof of Stake - // consensus algorithm, this field will actually be constant and set to the terminal total difficulty - // that was required to transition to Proof of Stake algorithm, which varies per network. It is set to - // 58 750 000 000 000 000 000 000 on Ethereum Mainnet and to 10 790 000 on Ethereum Testnet Goerli. - BigInt total_difficulty = 17; + // It has been deprecated in geth v1.15.0 but was already removed from the JSON-RPC interface for a while + BigInt total_difficulty = 17 [deprecated = true]; uint64 number = 9; uint64 gas_limit = 10; @@ -134,19 +211,84 @@ message BlockHeader { // extra_data, // mix_hash, // nonce, - // base_fee_per_gas + // base_fee_per_gas (to be included only if London fork is active) + // withdrawals_root (to be included only if Shangai fork is active) + // blob_gas_used (to be included only if Cancun fork is active) + // excess_blob_gas (to be included only if Cancun fork is active) + // parent_beacon_root (to be included only if Cancun fork is active) + // requests_hash (to be included only if Prague fork is active) // ])) // bytes hash = 16; // Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. BigInt base_fee_per_gas = 18; + + // Withdrawals root hash according to EIP-4895 (e.g. Shangai Fork) rules, only set if Shangai is present/active on the chain. + // + // Only available in DetailLevel: EXTENDED + bytes withdrawals_root = 19; + + // TxDependency is list of transaction indexes that are dependent on each other in the block + // header. This is metadata only that was used by the internal Polygon parallel execution engine. + // + // This field was available in a few versions on Polygon Mainnet and Polygon Mumbai chains. It was actually + // removed and is not populated anymore. It's now embedded in the `extraData` field, refer to Polygon source + // code to determine how to extract it if you need it. + // + // Only available in DetailLevel: EXTENDED + Uint64NestedArray tx_dependency = 20; + + // BlobGasUsed was added by EIP-4844 and is ignored in legacy headers. + optional uint64 blob_gas_used = 22; + + // ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers. + optional uint64 excess_blob_gas = 23; + + // ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers. + bytes parent_beacon_root = 24; + + // RequestsHash was added by EIP-7685 and is ignored in legacy headers. + bytes requests_hash = 25; +} + +message Uint64NestedArray { + repeated Uint64Array val = 1; +} + +message Uint64Array { + repeated uint64 val = 1; } message BigInt { bytes bytes = 1; } +// TransactionTrace is full trace of execution of the transaction when the +// it actually executed on chain. +// +// It contains all the transaction details like `from`, `to`, `gas`, etc. +// as well as all the internal calls that were made during the transaction. +// +// The `calls` vector contains Call objects which have balance changes, events +// storage changes, etc. +// +// If ordering is important between elements, almost each message like `Log`, +// `Call`, `StorageChange`, etc. have an ordinal field that is represents "execution" +// order of the said element against all other elements in this block. +// +// Due to how the call tree works doing "naively", looping through all calls then +// through a Call's element like `logs` while not yielding the elements in the order +// they were executed on chain. A log in call could have been done before or after +// another in another call depending on the actual call tree. +// +// The `calls` are ordered by creation order and the call tree can be re-computing +// using fields found in `Call` object (parent/child relationship). +// +// Another important thing to note is that even if a transaction succeed, some calls +// within it could have been reverted internally, if this is important to you, you must +// check the field `state_reverted` on the `Call` to determine if it was fully committed +// to the chain or not. message TransactionTrace { // consensus bytes to = 1; @@ -192,19 +334,47 @@ message TransactionTrace { // All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. TRX_TYPE_LEGACY = 0; - // Field that specifies an access list of contract/storage_keys that is going to be used + // Transaction that specicy an access list of contract/storage_keys that is going to be used // in this transaction. // // Added in Berlin fork (EIP-2930). TRX_TYPE_ACCESS_LIST = 1; - // Transaction that specifies an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the + // Transaction that specifis an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the // max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are // executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. TRX_TYPE_DYNAMIC_FEE = 2; + + // Transaction which contain a large amount of data that cannot be accessed by EVM execution, but whose commitment + // can be accessed. The format is intended to be fully compatible with the format that will be used in full sharding. + // + // Transaction that defines an access list just like TRX_TYPE_ACCESS_LIST and enables dynamic fee just like + // TRX_TYPE_DYNAMIC_FEE but in addition defines the fields 'max_fee_per_data_gas' of type 'uint256' and the fields + // 'blob_versioned_hashes' which represents a list of hash outputs from 'kzg_to_versioned_hash'. + // + // Activated in Cancun fork (EIP-4844) + TRX_TYPE_BLOB = 3; + + // Transaction that sets code to an EOA (Externally Owned Accounts) + // + // Activated in Prague (EIP-7702) + TRX_TYPE_SET_CODE = 4; + + // Arbitrum-specific transactions + TRX_TYPE_ARBITRUM_DEPOSIT = 100; + TRX_TYPE_ARBITRUM_UNSIGNED = 101; + TRX_TYPE_ARBITRUM_CONTRACT = 102; + TRX_TYPE_ARBITRUM_RETRY = 104; + TRX_TYPE_ARBITRUM_SUBMIT_RETRYABLE = 105; + TRX_TYPE_ARBITRUM_INTERNAL = 106; + TRX_TYPE_ARBITRUM_LEGACY = 120; + + // OPTIMISM-specific transactions + TRX_TYPE_OPTIMISM_DEPOSIT = 126; + } - // AcccessList represents the storage access this transaction has agreed to do in which case those storage + // AccessList represents the storage access this transaction has agreed to do in which case those storage // access cost less gas unit per access. // // This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which @@ -215,6 +385,8 @@ message TransactionTrace { // // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only // if London fork is active on the chain. + // + // Only available in DetailLevel: EXTENDED BigInt max_fee_per_gas = 11; // MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's @@ -222,22 +394,106 @@ message TransactionTrace { // // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only // if London fork is active on the chain. + // + // Only available in DetailLevel: EXTENDED BigInt max_priority_fee_per_gas = 13; // meta uint32 index = 20; bytes hash = 21; bytes from = 22; + + // Only available in DetailLevel: EXTENDED + // Known Issues + // - Version 3: + // Field not populated. It will be empty. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. bytes return_data = 23; + + // Only available in DetailLevel: EXTENDED bytes public_key = 24; + + // The block's global ordinal when the transaction started executing, refer to + // [Block] documentation for further information about ordinals and total ordering. uint64 begin_ordinal = 25; + + // The block's global ordinal when the transaction finished executing, refer to + // [Block] documentation for further information about ordinals and total ordering. uint64 end_ordinal = 26; + // TransactionTraceStatus is the status of the transaction execution and will let you know if the transaction + // was successful or not. + // + // ## Explanation relevant only for blocks with `DetailLevel: EXTENDED` + // + // A successful transaction has been recorded to the blockchain's state for calls in it that were successful. + // This means it's possible only a subset of the calls were properly recorded, refer to [calls[].state_reverted] field + // to determine which calls were reverted. + // + // A quirks of the Ethereum protocol is that a transaction `FAILED` or `REVERTED` still affects the blockchain's + // state for **some** of the state changes. Indeed, in those cases, the transactions fees are still paid to the miner + // which means there is a balance change for the transaction's emitter (e.g. `from`) to pay the gas fees, an optional + // balance change for gas refunded to the transaction's emitter (e.g. `from`) and a balance change for the miner who + // received the transaction fees. There is also a nonce change for the transaction's emitter (e.g. `from`). + // + // This means that to properly record the state changes for a transaction, you need to conditionally procees the + // transaction's status. + // + // For a `SUCCEEDED` transaction, you iterate over the `calls` array and record the state changes for each call for + // which `state_reverted == false` (if a transaction succeeded, the call at #0 will always `state_reverted == false` + // because it aligns with the transaction). + // + // For a `FAILED` or `REVERTED` transaction, you iterate over the root call (e.g. at #0, will always exist) for + // balance changes you process those where `reason` is either `REASON_GAS_BUY`, `REASON_GAS_REFUND` or + // `REASON_REWARD_TRANSACTION_FEE` and for nonce change, still on the root call, you pick the nonce change which the + // smallest ordinal (if more than one). TransactionTraceStatus status = 30; + TransactionReceipt receipt = 31; + + // Only available in DetailLevel: EXTENDED repeated Call calls = 32; -} + // BlobGas is the amount of gas the transaction is going to pay for the blobs, this is a computed value + // equivalent to `self.blob_gas_fee_cap * len(self.blob_hashes)` and provided in the model for convenience. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional uint64 blob_gas = 33; + + // BlobGasFeeCap is the maximum fee per data gas the user is willing to pay for the data gas used. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional BigInt blob_gas_fee_cap = 34; + + // BlobHashes field represents a list of hash outputs from 'kzg_to_versioned_hash' which + // essentially is a version byte + the sha256 hash of the blob commitment (e.g. + // `BLOB_COMMITMENT_VERSION_KZG + sha256(commitment)[1:]`. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + repeated bytes blob_hashes = 35; + + // SetCodeAuthorizations represents the authorizations of a transaction to set code to an EOA (Externally Owned Accounts) + // as defined in EIP-7702. The list will contain all the authorizations as they were specified in the + // transaction itself regardless of their validity. If you need to determined if a given authorization was + // correctly applied on chain's state, refer to [SetCodeAuthorization.discarded] field that records + // if the authorization was discarded or not by the chain due to invalidity. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-7702 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_SET_CODE` which is possible only + // if Prague fork is active on the chain. + repeated SetCodeAuthorization set_code_authorizations = 36; +} // AccessTuple represents a list of storage keys for a given contract's address and is used // for AccessList construction. @@ -246,10 +502,64 @@ message AccessTuple { repeated bytes storage_keys = 2; } -// TransactionTraceWithBlockRef -message TransactionTraceWithBlockRef { - TransactionTrace trace = 1; - BlockRef block_ref = 2; +// SetCodeAuthorization represents the authorization of a transaction to set code of an EOA (Externally Owned Account) +// as defined in EIP-7702. +// +// The 'authority' field is the address that is authorizing the delegation mechanism. The 'authority' value is computed +// from the signature contained in the message using the computation +// `authority = ecrecover(keccak(MAGIC || rlp([chain_id, address, nonce])), y_parity, r, s)` +// where `MAGIC` is `0x5`, `||` is the bytes concatenation operator, `ecrecover` is the Ethereum signature recovery +// and `y_parity` is the recovery ID value denoted `v` in the message below. Checking the go-ethereum implementation +// at https://github.com/ethereum/go-ethereum/blob/v1.15.0/core/types/tx_setcode.go#L117 might prove easier to "read". +// +// We do extract the 'authority' value from the signature in the message and store it in the 'authority' field for +// convenience so you don't need to perform the computation yourself. +message SetCodeAuthorization { + // Discarded determines if this authorization was skipped due to being invalid. As EIP-7702 states, + // if the authorization is invalid (invalid signature, nonce mismatch, etc.) it must be simply + // discarded and the transaction is processed as if the authorization was not present in the + // authorization list. + // + // This boolean records if the authorization was discarded or not by the chain due to invalidity. + bool discarded = 1; + + // ChainID is the chain ID of the chain where the transaction was executed, used + // to recover the authority from the signature. + bytes chain_id = 2; + + // Address contains the address this account is delegating to. This address usually + // contain code that this account essentially "delegates" to. + // + // Note: This was missing when EIP-7702 was first activated on Holesky, Sepolia, BSC Chapel, + // BSC Mainnet and Arbitrum Sepolia but was ready for Ethereum Mainnet hard fork. We will backfill + // those missing values in the near future at which point we will remove this note. + bytes address = 8; + + // Nonce is the nonce of the account that is authorizing delegation mechanism, EIP-7702 rules + // states that nonce should be verified using this rule: + // + // - Verify the nonce of authority is equal to nonce. In case authority does not exist in the trie, + // verify that nonce is equal to 0. + // + // Read SetCodeAuthorization to know how to recover the `authority` value. + uint64 nonce = 3; + + // V is the recovery ID value for the signature Y point. While it's defined as a + // `uint32`, it's actually bounded by a `uint8` data type withing the Ethereum protocol. + uint32 v = 4; + + // R is the signature's X point on the elliptic curve (32 bytes). + bytes r = 5; + + // S is the signature's Y point on the elliptic curve (32 bytes). + bytes s = 6; + + // Authority is the address of the account that is authorizing delegation mechanism, it + // is computed from the signature contained in the message and stored for convenience. + // + // If the authority cannot be recovered from the signature, this field will be empty and + // the `discarded` field will be set to `true`. + optional bytes authority = 7; } enum TransactionTraceStatus { @@ -263,9 +573,7 @@ message TransactionReceipt { // State root is an intermediate state_root hash, computed in-between transactions to make // **sure** you could build a proof and point to state in the middle of a block. Geth client // uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles - // hardforks, see (read the EIPs first): - // - https://github.com/eoscanada/go-ethereum-private/blob/deep-mind/core/types/receipt.go#L147 - // - https://github.com/eoscanada/go-ethereum-private/blob/deep-mind/core/types/receipt.go#L50-L86 + // hard forks, see (read the EIPs first): // - https://github.com/ethereum/EIPs/blob/master/EIPS/eip-658.md // // Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are @@ -277,6 +585,23 @@ message TransactionReceipt { uint64 cumulative_gas_used = 2; bytes logs_bloom = 3; repeated Log logs = 4; + + // BlobGasUsed is the amount of blob gas that has been used within this transaction. At time + // of writing, this is equal to `self.blob_gas_fee_cap * len(self.blob_hashes)`. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional uint64 blob_gas_used = 5; + + // BlobGasPrice is the amount to pay per blob item in the transaction. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional BigInt blob_gas_price = 6; } message Log { @@ -285,8 +610,10 @@ message Log { bytes data = 3; // Index is the index of the log relative to the transaction. This index - // is always populated regardless of the state revertion of the the call + // is always populated regardless of the state reversion of the the call // that emitted this log. + // + // Only available in DetailLevel: EXTENDED uint32 index = 4; // BlockIndex represents the index of the log relative to the Block. @@ -306,6 +633,8 @@ message Log { // the `blockIndex` value will always be 0. uint32 blockIndex = 6; + // The block's global ordinal when the log was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 7; } @@ -316,32 +645,83 @@ message Call { CallType call_type = 4; bytes caller = 5; bytes address = 6; + + // AddressDelegatesTo contains the address from which the actual code to execute will be loaded + // as defined per EIP-7702 rules. If the Call's address value resolves to a code + // that delegates to another address, this field will be populated with the address + // that the call is delegated to. It will be empty in all other situations. + // + // Assumes that a 'SetCode' transaction set address `0xA` to delegates to address `0xB`, + // then when a call is made to `0xA`, the Call object would have: + // + // - caller = + // - address = 0xA + // - address_delegates_to = 0xB + // + // Again, it's important to emphasize that this field relates to EIP-7702, if the call is + // a DELEGATE or CALLCODE type, this field will not be populated and will remain empty. + // + // It will be populated only if EIP-7702 is active on the chain (Prague fork) and if the + // 'address' of the call was pointing to another address at time of execution. + optional bytes address_delegates_to = 34; + BigInt value = 7; uint64 gas_limit = 8; uint64 gas_consumed = 9; bytes return_data = 13; + + // Known Issues + // - Version 3: + // When call is `CREATE` or `CREATE2`, this field is not populated. A couple of suggestions: + // 1. You can get the contract's code in the `code_changes` field. + // 2. In the root `CREATE` call, you can directly use the `TransactionTrace`'s input field. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. bytes input = 14; + + // Indicates whether the call executed code. + // + // Known Issues + // - Version 3: + // This may be incorrectly set to `false` for accounts with code handling native value transfers, + // as well as for certain precompiles with no input. + // The value is initially set based on `call.type != CREATE && len(call.input) > 0` + // and later adjusted if the tracer detects an account without code. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. bool executed_code = 15; bool suicide = 16; /* hex representation of the hash -> preimage */ map keccak_preimages = 20; + + // Known Issues + // - Version 3: + // The data might be not be in order. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. repeated StorageChange storage_changes = 21; repeated BalanceChange balance_changes = 22; repeated NonceChange nonce_changes = 24; repeated Log logs = 25; repeated CodeChange code_changes = 26; - // Deprecated: repeated bytes created_accounts reserved 27; + // Known Issues + // - Version 3: + // Some gas changes are not correctly tracked: + // 1. Gas refunded due to data returned to the chain (occurs at the end of a transaction, before buyback). + // 2. Initial gas allocation (0 -> GasLimit) at the start of a call. + // 3. Final gas deduction (LeftOver -> 0) at the end of a call (if applicable). + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. repeated GasChange gas_changes = 28; // Deprecated: repeated GasEvent gas_events reserved 29; // In Ethereum, a call can be either: - // - Successfull, execution passes without any problem encountered + // - Successful, execution passes without any problem encountered // - Failed, execution failed, and remaining gas should be consumed // - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded // @@ -355,7 +735,7 @@ message Call { // see above for details about those flags. string failure_reason = 11; - // This field represents wheter or not the state changes performed + // This field represents whether or not the state changes performed // by this call were correctly recorded by the blockchain. // // On Ethereum, a transaction can record state changes even if some @@ -375,13 +755,44 @@ message Call { // ``` // // In the transaction above, while Call #2 and Call #3 would have the - // status `EXECUTED` + // status `EXECUTED`. + // + // If you check all calls and check only `state_reverted` flag, you might be missing + // some balance changes and nonce changes. This is because when a full transaction fails + // in ethereum (e.g. `calls.all(x.state_reverted == true)`), there is still the transaction + // fee that are recorded to the chain. + // + // Refer to [TransactionTrace#status] field for more details about the handling you must + // perform. bool state_reverted = 30; + // Known Issues + // - Version 3: + // 1. The block's global ordinal when the call started executing, refer to + // [Block] documentation for further information about ordinals and total ordering. + // 2. The transaction root call `begin_ordial` is always `0` (also in the GENESIS block), which can cause issues + // when sorting by this field. To ensure proper execution order, set it as follows: + // `trx.Calls[0].BeginOrdinal = trx.BeginOrdinal`. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. uint64 begin_ordinal = 31; + + // Known Issues + // - Version 3: + // 1. The block's global ordinal when the call finished executing, refer to + // [Block] documentation for further information about ordinals and total ordering. + // 2. The root call of the GENESIS block is always `0`. To fix it, you can set it as follows: + // `rx.Calls[0].EndOrdinal = max.Uint64`. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. uint64 end_ordinal = 32; - repeated AccountCreation account_creations = 33; + // Known Issues + // - Version 4: + // AccountCreations are NOT SUPPORTED anymore. DO NOT rely on them. + repeated AccountCreation account_creations = 33 [deprecated = true]; + + // The identifier 34 is taken by 'address_delegates_to' field above. reserved 50; // repeated ERC20BalanceChange erc20_balance_changes = 50 [deprecated = true]; reserved 51; // repeated ERC20TransferEvent erc20_transfer_events = 51 [deprecated = true]; @@ -403,20 +814,42 @@ message StorageChange { bytes old_value = 3; bytes new_value = 4; + // The block's global ordinal when the storage change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 5; } message BalanceChange { + // Address is the address of the account that has changed balance. bytes address = 1; + + // OldValue is the balance of the address before the change. This value + // can be **nil/null/None** if there was no previous balance for the address. + // It is safe in those case(s) to consider the balance as being 0. + // + // If you consume this from a Substreams, you can safely use: + // + // ```ignore + // let old_value = old_value.unwrap_or_default(); + // ``` BigInt old_value = 2; - BigInt new_value = 3; - Reason reason = 4; - // Obtain all balanche change reasons under deep mind repository: + // NewValue is the balance of the address after the change. This value + // can be **nil/null/None** if there was no previous balance for the address + // after the change. It is safe in those case(s) to consider the balance as being + // 0. // - // ```shell - // ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq + // If you consume this from a Substreams, you can safely use: + // + // ```ignore + // let new_value = new_value.unwrap_or_default(); // ``` + BigInt new_value = 3; + + // Reason is the reason why the balance has changed. This is useful to determine + // why the balance has changed and what is the context of the change. + Reason reason = 4; + enum Reason { REASON_UNKNOWN = 0; REASON_REWARD_MINE_UNCLE = 1; @@ -435,8 +868,20 @@ message BalanceChange { REASON_CALL_BALANCE_OVERRIDE = 12; // Used on chain(s) where some Ether burning happens REASON_BURN = 15; + REASON_WITHDRAWAL = 16; + + // Rewards for Blob processing on BNB chain added in Tycho hard-fork, refers + // to BNB documentation to check the timestamp at which it was activated. + REASON_REWARD_BLOB_FEE = 17; + + // This reason is used only on Optimism chain. + REASON_INCREASE_MINT = 18; + // This reason is used only on Optimism chain. + REASON_REVERT = 19; } + // The block's global ordinal when the balance change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 5; } @@ -444,11 +889,17 @@ message NonceChange { bytes address = 1; uint64 old_value = 2; uint64 new_value = 3; + + // The block's global ordinal when the nonce change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 4; } message AccountCreation { bytes account = 1; + + // The block's global ordinal when the account creation was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 2; } @@ -459,6 +910,8 @@ message CodeChange { bytes new_hash = 4; bytes new_code = 5; + // The block's global ordinal when the code change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 6; } @@ -473,36 +926,148 @@ message GasChange { uint64 new_value = 2; Reason reason = 3; - // Obtain all gas change reasons under deep mind repository: - // - // ```shell - // ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - // ``` enum Reason { REASON_UNKNOWN = 0; + // REASON_CALL is the amount of gas that will be charged for a 'CALL' opcode executed by the EVM REASON_CALL = 1; + // REASON_CALL_CODE is the amount of gas that will be charged for a 'CALLCODE' opcode executed by the EVM REASON_CALL_CODE = 2; + // REASON_CALL_DATA_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM REASON_CALL_DATA_COPY = 3; + // REASON_CODE_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM REASON_CODE_COPY = 4; + // REASON_CODE_STORAGE is the amount of gas that will be charged for code storage REASON_CODE_STORAGE = 5; + // REASON_CONTRACT_CREATION is the amount of gas that will be charged for a 'CREATE' opcode executed by the EVM and for the gas + // burned for a CREATE, today controlled by EIP150 rules REASON_CONTRACT_CREATION = 6; + // REASON_CONTRACT_CREATION2 is the amount of gas that will be charged for a 'CREATE2' opcode executed by the EVM and for the gas + // burned for a CREATE2, today controlled by EIP150 rules REASON_CONTRACT_CREATION2 = 7; + // REASON_DELEGATE_CALL is the amount of gas that will be charged for a 'DELEGATECALL' opcode executed by the EVM REASON_DELEGATE_CALL = 8; + // REASON_EVENT_LOG is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM REASON_EVENT_LOG = 9; + // REASON_EXT_CODE_COPY is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM REASON_EXT_CODE_COPY = 10; + // REASON_FAILED_EXECUTION is the burning of the remaining gas when the execution failed without a revert REASON_FAILED_EXECUTION = 11; + // REASON_INTRINSIC_GAS is the amount of gas that will be charged for the intrinsic cost of the transaction, there is + // always exactly one of those per transaction REASON_INTRINSIC_GAS = 12; + // GasChangePrecompiledContract is the amount of gas that will be charged for a precompiled contract execution REASON_PRECOMPILED_CONTRACT = 13; + // REASON_REFUND_AFTER_EXECUTION is the amount of gas that will be refunded to the caller after the execution of the call, + // if there is left over at the end of execution REASON_REFUND_AFTER_EXECUTION = 14; + // REASON_RETURN is the amount of gas that will be charged for a 'RETURN' opcode executed by the EVM REASON_RETURN = 15; + // REASON_RETURN_DATA_COPY is the amount of gas that will be charged for a 'RETURNDATACOPY' opcode executed by the EVM REASON_RETURN_DATA_COPY = 16; + // REASON_REVERT is the amount of gas that will be charged for a 'REVERT' opcode executed by the EVM REASON_REVERT = 17; + // REASON_SELF_DESTRUCT is the amount of gas that will be charged for a 'SELFDESTRUCT' opcode executed by the EVM REASON_SELF_DESTRUCT = 18; + // REASON_STATIC_CALL is the amount of gas that will be charged for a 'STATICALL' opcode executed by the EVM REASON_STATIC_CALL = 19; + // REASON_STATE_COLD_ACCESS is the amount of gas that will be charged for a cold storage access as controlled by EIP2929 rules + // // Added in Berlin fork (Geth 1.10+) REASON_STATE_COLD_ACCESS = 20; + + // REASON_TX_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call + // + // Added as new tracing reason in Geth, available only on some chains + REASON_TX_INITIAL_BALANCE = 21; + // REASON_TX_REFUNDS is the sum of all refunds which happened during the tx execution (e.g. storage slot being cleared) + // this generates an increase in gas. There is only one such gas change per transaction. + // + // Added as new tracing reason in Geth, available only on some chains + REASON_TX_REFUNDS = 22; + // REASON_TX_LEFT_OVER_RETURNED is the amount of gas left over at the end of transaction's execution that will be returned + // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. + // There is at most one of such gas change per transaction. + // + // Added as new tracing reason in Geth, available only on some chains + REASON_TX_LEFT_OVER_RETURNED = 23; + + // REASON_CALL_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call. There is only + // one such gas change per call. + // + // Added as new tracing reason in Geth, available only on some chains + REASON_CALL_INITIAL_BALANCE = 24; + // REASON_CALL_LEFT_OVER_RETURNED is the amount of gas left over that will be returned to the caller, this change will always + // be a negative change as we "drain" left over gas towards 0. If there was no gas left at the end of execution, no such even + // will be emitted. + REASON_CALL_LEFT_OVER_RETURNED = 25; + + // REASON_WITNESS_CONTRACT_INIT flags the event of adding to the witness during the contract creation initialization step. + REASON_WITNESS_CONTRACT_INIT = 26; + + // REASON_WITNESS_CONTRACT_CREATION flags the event of adding to the witness during the contract creation finalization step. + REASON_WITNESS_CONTRACT_CREATION = 27; + // REASON_WITNESS_CODE_CHUNK flags the event of adding one or more contract code chunks to the witness. + REASON_WITNESS_CODE_CHUNK = 28; + // REASON_WITNESS_CONTRACT_COLLISION_CHECK flags the event of adding to the witness when checking for contract address collision. + REASON_WITNESS_CONTRACT_COLLISION_CHECK = 29; + // REASON_TX_DATA_FLOOR is the amount of extra gas the transaction has to pay to reach the minimum gas requirement for the + // transaction data. This change will always be a negative change. + REASON_TX_DATA_FLOOR = 30; } + // The block's global ordinal when the gas change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. uint64 ordinal = 4; +} + +// HeaderOnlyBlock is used to optimally unpack the [Block] structure (note the +// corresponding message number for the `header` field) while consuming less +// memory, when only the `header` is desired. +// +// WARN: this is a client-side optimization pattern and should be moved in the +// consuming code. +message HeaderOnlyBlock { + BlockHeader header = 5; +} + +// BlockWithRefs is a lightweight block, with traces and transactions +// purged from the `block` within, and only. It is used in transports +// to pass block data around. +message BlockWithRefs { + string id = 1; + Block block = 2; + TransactionRefs transaction_trace_refs = 3; + bool irreversible = 4; +} + +message TransactionTraceWithBlockRef { + TransactionTrace trace = 1; + BlockRef block_ref = 2; +} + +message TransactionRefs { + repeated bytes hashes = 1; +} + +message BlockRef { + bytes hash = 1; + uint64 number = 2; +} + +// Withdrawal represents a validator withdrawal from the beacon chain to the EVM. +// Introduced in EIP-4895 (Shanghai hard fork). +message Withdrawal { + // Index is the monotonically increasing identifier of the withdrawal + uint64 index = 1; + + // ValidatorIndex is the index of the validator that is withdrawing + uint64 validator_index = 2; + + // Address is the Ethereum address receiving the withdrawn funds + bytes address = 3; + + // Amount is the value of the withdrawal in gwei (1 gwei = 1e9 wei) + uint64 amount = 4; } \ No newline at end of file diff --git a/chain/ethereum/src/adapter.rs b/chain/ethereum/src/adapter.rs index db607a40233..d37d5ba81f7 100644 --- a/chain/ethereum/src/adapter.rs +++ b/chain/ethereum/src/adapter.rs @@ -1,20 +1,26 @@ use anyhow::Error; -use ethabi::{Error as ABIError, Function, ParamType, Token}; -use futures::Future; +use async_trait::async_trait; +use graph::abi; use graph::blockchain::ChainIdentifier; +use graph::components::ethereum::AnyBlock; +use graph::components::subgraph::MappingError; +use graph::data::store::ethereum::call; +use graph::data_source::common::ContractCall; use graph::firehose::CallToFilter; use graph::firehose::CombinedFilter; use graph::firehose::LogFilter; +use graph::prelude::alloy::primitives::keccak256; +use graph::prelude::alloy::primitives::{Address, B256}; +use graph::prelude::alloy::rpc::types::Log; +use graph::prelude::alloy::transports::{RpcError, TransportErrorKind}; use itertools::Itertools; use prost::Message; use prost_types::Any; use std::cmp; use std::collections::{HashMap, HashSet}; use std::fmt; -use std::marker::Unpin; +use std::hash::Hash; use thiserror::Error; -use tiny_keccak::keccak256; -use web3::types::{Address, Log, H256}; use graph::prelude::*; use graph::{ @@ -23,40 +29,115 @@ use graph::{ petgraph::{self, graphmap::GraphMap}, }; +use graph::blockchain::BlockPtr; + const COMBINED_FILTER_TYPE_URL: &str = "type.googleapis.com/sf.ethereum.transform.v1.CombinedFilter"; use crate::capabilities::NodeCapabilities; use crate::data_source::{BlockHandlerFilter, DataSource}; -use crate::{Chain, Mapping, ENV_VARS}; +use crate::{Chain, ENV_VARS, Mapping}; -pub type EventSignature = H256; +pub type EventSignature = B256; pub type FunctionSelector = [u8; 4]; -#[derive(Clone, Debug)] -pub struct EthereumContractCall { - pub address: Address, - pub block_ptr: BlockPtr, - pub function: Function, - pub args: Vec, - pub gas: Option, +/// `EventSignatureWithTopics` is used to match events with +/// indexed arguments when they are defined in the subgraph +/// manifest. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EventSignatureWithTopics { + pub address: Option
, + pub signature: B256, + pub topic1: Option>, + pub topic2: Option>, + pub topic3: Option>, +} + +impl EventSignatureWithTopics { + pub fn new( + address: Option
, + signature: B256, + topic1: Option>, + topic2: Option>, + topic3: Option>, + ) -> Self { + EventSignatureWithTopics { + address, + signature, + topic1, + topic2, + topic3, + } + } + + /// Checks if an event matches the `EventSignatureWithTopics` + /// If self.address is None, it's considered a wildcard match. + /// Otherwise, it must match the provided address. + /// It must also match the topics if they are Some + pub fn matches(&self, address: Option<&Address>, sig: B256, topics: &[B256]) -> bool { + // If self.address is None, it's considered a wildcard match. Otherwise, it must match the provided address. + let address_matches = match self.address { + Some(ref self_addr) => address == Some(self_addr), + None => true, // self.address is None, so it matches any address. + }; + + address_matches + && self.signature == sig + && self + .topic1 + .as_ref() + .is_none_or(|t1| topics.get(1).is_some_and(|topic| t1.contains(topic))) + && self + .topic2 + .as_ref() + .is_none_or(|t2| topics.get(2).is_some_and(|topic| t2.contains(topic))) + && self + .topic3 + .as_ref() + .is_none_or(|t3| topics.get(3).is_some_and(|topic| t3.contains(topic))) + } +} + +#[derive(Error, Debug)] +pub enum EthereumRpcError { + #[error("call error: {0}")] + AlloyError(RpcError), + #[error("ethereum node took too long to perform call")] + Timeout, } #[derive(Error, Debug)] -pub enum EthereumContractCallError { - #[error("ABI error: {0}")] - ABIError(#[from] ABIError), - /// `Token` is not of expected `ParamType` - #[error("type mismatch, token {0:?} is not of kind {1:?}")] - TypeError(Token, ParamType), - #[error("error encoding input call data: {0}")] - EncodingError(ethabi::Error), +pub enum ContractCallError { + #[error("ABI error: {0:#}")] + ABIError(anyhow::Error), + #[error("type mismatch, decoded value {0:?} is not of kind {1:?}")] + TypeError(abi::DynSolValue, abi::DynSolType), + #[error("error encoding input call data: {0:#}")] + EncodingError(anyhow::Error), #[error("call error: {0}")] - Web3Error(web3::Error), - #[error("call reverted: {0}")] - Revert(String), + AlloyError(RpcError), #[error("ethereum node took too long to perform call")] Timeout, + #[error("internal error: {0}")] + Internal(String), +} + +impl From for MappingError { + fn from(e: ContractCallError) -> Self { + match e { + // Any error reported by the Ethereum node could be due to the block no longer being on + // the main chain. This is very unespecific but we don't want to risk failing a + // subgraph due to a transient error such as a reorg. + ContractCallError::AlloyError(e) => MappingError::PossibleReorg(anyhow::anyhow!( + "Ethereum node returned an error for an eth_call: {e}" + )), + // Also retry on timeouts. + ContractCallError::Timeout => MappingError::PossibleReorg(anyhow::anyhow!( + "Ethereum node did not respond in time to eth_call" + )), + e => MappingError::Unknown(anyhow::anyhow!("Error when making an eth_call: {e}")), + } + } } #[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] @@ -69,44 +150,104 @@ enum LogFilterNode { #[derive(Clone, Debug)] pub struct EthGetLogsFilter { pub contracts: Vec
, - pub event_signatures: Vec, + pub event_signatures: Vec, + pub topic1: Option>, + pub topic2: Option>, + pub topic3: Option>, } impl EthGetLogsFilter { + /// Convert to alloy Filter for the given block range + pub fn to_alloy_filter(&self, from: BlockNumber, to: BlockNumber) -> alloy::rpc::types::Filter { + let mut filter_builder = alloy::rpc::types::Filter::new() + .from_block(alloy::rpc::types::BlockNumberOrTag::Number(from as u64)) + .to_block(alloy::rpc::types::BlockNumberOrTag::Number(to as u64)) + .address(self.contracts.clone()) + .event_signature(self.event_signatures.clone()); + + if let Some(ref topic1) = self.topic1 { + filter_builder = filter_builder.topic1(topic1.clone()); + } + if let Some(ref topic2) = self.topic2 { + filter_builder = filter_builder.topic2(topic2.clone()); + } + if let Some(ref topic3) = self.topic3 { + filter_builder = filter_builder.topic3(topic3.clone()); + } + + filter_builder + } + fn from_contract(address: Address) -> Self { EthGetLogsFilter { contracts: vec![address], event_signatures: vec![], + topic1: None, + topic2: None, + topic3: None, } } - fn from_event(event: EventSignature) -> Self { + fn from_event(event: B256) -> Self { EthGetLogsFilter { contracts: vec![], event_signatures: vec![event], + topic1: None, + topic2: None, + topic3: None, + } + } + + fn from_event_with_topics(event: EventSignatureWithTopics) -> Self { + EthGetLogsFilter { + contracts: event.address.map_or(vec![], |a| vec![a]), + event_signatures: vec![event.signature], + topic1: event.topic1, + topic2: event.topic2, + topic3: event.topic3, } } } impl fmt::Display for EthGetLogsFilter { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.contracts.len() == 1 { - write!( - f, + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let base_msg = if self.contracts.len() == 1 { + format!( "contract {:?}, {} events", self.contracts[0], self.event_signatures.len() ) } else if self.event_signatures.len() == 1 { - write!( - f, + format!( "event {:?}, {} contracts", self.event_signatures[0], self.contracts.len() ) } else { - write!(f, "unreachable") - } + "unspecified filter".to_string() + }; + + // Helper to format topics as strings + let format_topics = |topics: &Option>| -> String { + topics.as_ref().map_or_else( + || "None".to_string(), + |ts| { + let signatures: Vec = ts.iter().map(|t| format!("{:?}", t)).collect(); + signatures.join(", ") + }, + ) + }; + + // Constructing topic strings + let topics_msg = format!( + ", topic1: [{}], topic2: [{}], topic3: [{}]", + format_topics(&self.topic1), + format_topics(&self.topic2), + format_topics(&self.topic3), + ); + + // Combine the base message with topic information + write!(f, "{}{}", base_msg, topics_msg) } } @@ -178,6 +319,11 @@ impl bc::TriggerFilter for TriggerFilter { trigger_every_block, } = self.block.clone(); + // If polling_intervals is empty this will return true, else it will be true only if all intervals are 0 + // ie: All triggers are initialization handlers. We do not need firehose to send all block headers for + // initialization handlers + let has_initilization_triggers_only = polling_intervals.iter().all(|(_, i)| *i == 0); + let log_filters: Vec = self.log.into(); let mut call_filters: Vec = self.call.into(); call_filters.extend(Into::>::into(self.block)); @@ -189,7 +335,9 @@ impl bc::TriggerFilter for TriggerFilter { let combined_filter = CombinedFilter { log_filters, call_filters, - send_all_block_headers: trigger_every_block || !polling_intervals.is_empty(), + // We need firehose to send all block headers when `trigger_every_block` is true and when + // We have polling triggers which are not from initiallization handlers + send_all_block_headers: trigger_every_block || !has_initilization_triggers_only, }; vec![Any { @@ -204,29 +352,131 @@ pub struct EthereumLogFilter { /// Log filters can be represented as a bipartite graph between contracts and events. An edge /// exists between a contract and an event if a data source for the contract has a trigger for /// the event. - /// Edges are of `bool` type and indicates when a trigger requires a transaction receipt. - contracts_and_events_graph: GraphMap, + /// Edge weights are booleans indicating whether the trigger requires a transaction receipt. + contracts_and_events_graph: MergeGraph, /// Event sigs with no associated address, matching on all addresses. - /// Maps to a boolean representing if a trigger requires a transaction receipt. - wildcard_events: HashMap, + /// Values are booleans indicating whether the trigger requires a transaction receipt. + wildcard_events: MergeMap, + /// Events with any of the topic filters set. + /// Values are booleans indicating whether the trigger requires a transaction receipt. + events_with_topic_filters: MergeMap, +} + +/// `HashMap` wrapper whose values are OR-merged on every write. +/// +/// The only mutator that writes values is [`MergeMap::or_insert`] — the inner +/// `HashMap` is private so callers cannot bypass the merge via +/// `HashMap::insert`. Used by `EthereumLogFilter` to track per-key receipt +/// requirements where any handler asking for a receipt at a given key forces +/// receipt fetching. +#[derive(Clone, Debug)] +struct MergeMap(HashMap); + +impl Default for MergeMap { + fn default() -> Self { + Self(HashMap::new()) + } +} + +impl MergeMap { + fn or_insert(&mut self, k: K, v: bool) { + self.0.entry(k).and_modify(|e| *e |= v).or_insert(v); + } + + fn get(&self, k: &K) -> Option<&bool> { + self.0.get(k) + } + + fn contains_key(&self, k: &K) -> bool { + self.0.contains_key(k) + } + + fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl IntoIterator for MergeMap { + type Item = (K, bool); + type IntoIter = std::collections::hash_map::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +/// `GraphMap` wrapper that OR-merges edge weights on +/// every write. +/// +/// The only mutator that writes edge weights is [`MergeGraph::or_add_edge`] — +/// the inner `GraphMap` is private so callers cannot bypass the merge via +/// `GraphMap::add_edge`. +#[derive(Clone, Debug)] +struct MergeGraph(GraphMap); + +impl Default for MergeGraph { + fn default() -> Self { + Self(GraphMap::new()) + } +} + +impl MergeGraph { + fn or_add_edge(&mut self, a: LogFilterNode, b: LogFilterNode, v: bool) { + // Short-circuit on `v == true`: a `true` weight always wins over any + // prior weight, so we can skip the edge_weight lookup entirely. + let merged = v || self.0.edge_weight(a, b).copied().unwrap_or(false); + self.0.add_edge(a, b, merged); + } + + fn edge_weight(&self, a: LogFilterNode, b: LogFilterNode) -> Option<&bool> { + self.0.edge_weight(a, b) + } + + fn contains_edge(&self, a: LogFilterNode, b: LogFilterNode) -> bool { + self.0.contains_edge(a, b) + } + + fn edge_count(&self) -> usize { + self.0.edge_count() + } + + fn nodes(&self) -> impl Iterator + '_ { + self.0.nodes() + } + + fn neighbors(&self, n: LogFilterNode) -> impl Iterator + '_ { + self.0.neighbors(n) + } + + fn remove_node(&mut self, n: LogFilterNode) -> bool { + self.0.remove_node(n) + } + + fn all_edges(&self) -> impl Iterator { + self.0.all_edges() + } } impl From for Vec { fn from(val: EthereumLogFilter) -> Self { - val.eth_get_logs_filters() + val.eth_get_logs_filters(ENV_VARS.get_logs_max_contracts) .map( |EthGetLogsFilter { contracts, event_signatures, + .. // TODO: Handle events with topic filters for firehose }| LogFilter { addresses: contracts .iter() - .map(|addr| addr.to_fixed_bytes().to_vec()) + .map(|addr| addr.to_vec()) .collect_vec(), event_signatures: event_signatures .iter() - .map(|sig| sig.to_fixed_bytes().to_vec()) + .map(|sig| sig.to_vec()) .collect_vec(), }, ) @@ -238,19 +488,22 @@ impl EthereumLogFilter { /// Check if this filter matches the specified `Log`. pub fn matches(&self, log: &Log) -> bool { // First topic should be event sig - match log.topics.first() { + match log.topics().first() { None => false, Some(sig) => { // The `Log` matches the filter either if the filter contains // a (contract address, event signature) pair that matches the // `Log`, or if the filter contains wildcard event that matches. - let contract = LogFilterNode::Contract(log.address); + let contract = LogFilterNode::Contract(log.address()); let event = LogFilterNode::Event(*sig); self.contracts_and_events_graph - .all_edges() - .any(|(s, t, _)| (s == contract && t == event) || (t == contract && s == event)) + .contains_edge(contract, event) || self.wildcard_events.contains_key(sig) + || self + .events_with_topic_filters + .iter() + .any(|(e, _)| e.matches(Some(&log.address()), *sig, log.topics())) } } } @@ -258,22 +511,43 @@ impl EthereumLogFilter { /// Similar to [`matches`], checks if a transaction receipt is required for this log filter. pub fn requires_transaction_receipt( &self, - event_signature: &H256, + event_signature: &B256, contract_address: Option<&Address>, + topics: &[B256], ) -> bool { - if let Some(true) = self.wildcard_events.get(event_signature) { - true - } else if let Some(address) = contract_address { - let contract = LogFilterNode::Contract(*address); - let event = LogFilterNode::Event(*event_signature); - self.contracts_and_events_graph - .all_edges() - .any(|(s, t, r)| { - *r && (s == contract && t == event) || (t == contract && s == event) - }) - } else { - false + // Check for wildcard events first. + if self.wildcard_events.get(event_signature) == Some(&true) { + return true; + } + + // Next, check events with topic filters. + if self + .events_with_topic_filters + .iter() + .any(|(event_with_topics, &requires_receipt)| { + requires_receipt + && event_with_topics.matches(contract_address, *event_signature, topics) + }) + { + return true; + } + + // Finally, check the contracts_and_events_graph if a contract address is specified. + if let Some(address) = contract_address { + let contract_node = LogFilterNode::Contract(*address); + let event_node = LogFilterNode::Event(*event_signature); + + if self + .contracts_and_events_graph + .edge_weight(contract_node, event_node) + == Some(&true) + { + return true; + } } + + // If none of the conditions above match, return false. + false } pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { @@ -282,16 +556,42 @@ impl EthereumLogFilter { for event_handler in ds.mapping.event_handlers.iter() { let event_sig = event_handler.topic0(); match ds.address { - Some(contract) => { - this.contracts_and_events_graph.add_edge( + Some(contract) if !event_handler.has_additional_topics() => { + this.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(contract), LogFilterNode::Event(event_sig), event_handler.receipt, ); } - None => { + Some(contract) => { + this.events_with_topic_filters.or_insert( + EventSignatureWithTopics::new( + Some(contract), + event_sig, + event_handler.topic1.clone(), + event_handler.topic2.clone(), + event_handler.topic3.clone(), + ), + event_handler.receipt, + ); + } + + None if (!event_handler.has_additional_topics()) => { this.wildcard_events - .insert(event_sig, event_handler.receipt); + .or_insert(event_sig, event_handler.receipt); + } + + None => { + this.events_with_topic_filters.or_insert( + EventSignatureWithTopics::new( + ds.address, + event_sig, + event_handler.topic1.clone(), + event_handler.topic2.clone(), + event_handler.topic3.clone(), + ), + event_handler.receipt, + ); } } } @@ -304,12 +604,18 @@ impl EthereumLogFilter { for event_handler in &mapping.event_handlers { let signature = event_handler.topic0(); this.wildcard_events - .insert(signature, event_handler.receipt); + .or_insert(signature, event_handler.receipt); } this } /// Extends this log filter with another one. + /// + /// The `receipt` flag stored at each filter key is OR-combined across the + /// two filters: a key whose receipt requirement is `true` in either filter + /// stays `true` in the merged result. Overwriting with the incoming value + /// would drop a prior `true` and silently downgrade receipt fetching for + /// any handler that declared `receipt: true` at that key. pub fn extend(&mut self, other: EthereumLogFilter) { if other.is_empty() { return; @@ -319,11 +625,17 @@ impl EthereumLogFilter { let EthereumLogFilter { contracts_and_events_graph, wildcard_events, + events_with_topic_filters, } = other; for (s, t, e) in contracts_and_events_graph.all_edges() { - self.contracts_and_events_graph.add_edge(s, t, *e); + self.contracts_and_events_graph.or_add_edge(s, t, *e); + } + for (k, v) in wildcard_events { + self.wildcard_events.or_insert(k, v); + } + for (k, v) in events_with_topic_filters { + self.events_with_topic_filters.or_insert(k, v); } - self.wildcard_events.extend(wildcard_events); } /// An empty filter is one that never matches. @@ -332,20 +644,35 @@ impl EthereumLogFilter { let EthereumLogFilter { contracts_and_events_graph, wildcard_events, + events_with_topic_filters, } = self; - contracts_and_events_graph.edge_count() == 0 && wildcard_events.is_empty() + contracts_and_events_graph.edge_count() == 0 + && wildcard_events.is_empty() + && events_with_topic_filters.is_empty() } /// Filters for `eth_getLogs` calls. The filters will not return false positives. This attempts /// to balance between having granular filters but too many calls and having few calls but too /// broad filters causing the Ethereum endpoint to timeout. - pub fn eth_get_logs_filters(self) -> impl Iterator { + pub fn eth_get_logs_filters( + self, + get_logs_max_contracts: usize, + ) -> impl Iterator { + let mut filters = Vec::new(); + // Start with the wildcard event filters. - let mut filters = self - .wildcard_events - .into_keys() - .map(EthGetLogsFilter::from_event) - .collect_vec(); + filters.extend( + self.wildcard_events + .into_iter() + .map(|(k, _)| EthGetLogsFilter::from_event(k)), + ); + + // Handle events with topic filters. + filters.extend( + self.events_with_topic_filters + .into_iter() + .map(|(k, _)| EthGetLogsFilter::from_event_with_topics(k)), + ); // The current algorithm is to repeatedly find the maximum cardinality vertex and turn all // of its edges into a filter. This is nice because it is neutral between filtering by @@ -380,7 +707,7 @@ impl EthereumLogFilter { for neighbor in g.neighbors(max_vertex) { match neighbor { LogFilterNode::Contract(address) => { - if filter.contracts.len() == ENV_VARS.get_logs_max_contracts { + if filter.contracts.len() == get_logs_max_contracts { // The batch size was reached, register the filter and start a new one. let event = filter.event_signatures[0]; push_filter(filter); @@ -419,21 +746,21 @@ pub struct EthereumCallFilter { pub wildcard_signatures: HashSet, } -impl Into> for EthereumCallFilter { - fn into(self) -> Vec { - if self.is_empty() { +impl From for Vec { + fn from(val: EthereumCallFilter) -> Self { + if val.is_empty() { return Vec::new(); } let EthereumCallFilter { contract_addresses_function_signatures, wildcard_signatures, - } = self; + } = val; let mut filters: Vec = contract_addresses_function_signatures .into_iter() .map(|(addr, (_, sigs))| CallToFilter { - addresses: vec![addr.to_fixed_bytes().to_vec()], + addresses: vec![addr.to_vec()], signatures: sigs.into_iter().map(|x| x.to_vec()).collect_vec(), }) .collect(); @@ -574,16 +901,13 @@ impl FromIterator<(BlockNumber, Address, FunctionSelector)> for EthereumCallFilt let mut lookup: HashMap)> = HashMap::new(); iter.into_iter() .for_each(|(start_block, address, function_signature)| { - lookup + let entry = lookup .entry(address) .or_insert((start_block, HashSet::default())); - lookup.get_mut(&address).map(|set| { - if set.0 > start_block { - set.0 = start_block - } - set.1.insert(function_signature); - set - }); + if entry.0 > start_block { + entry.0 = start_block; + } + entry.1.insert(function_signature); }); EthereumCallFilter { contract_addresses_function_signatures: lookup, @@ -615,15 +939,15 @@ pub struct EthereumBlockFilter { pub trigger_every_block: bool, } -impl Into> for EthereumBlockFilter { - fn into(self) -> Vec { - self.contract_addresses +impl From for Vec { + fn from(val: EthereumBlockFilter) -> Self { + val.contract_addresses .into_iter() .map(|(_, addr)| addr) .sorted() .dedup_by(|x, y| x == y) .map(|addr| CallToFilter { - addresses: vec![addr.to_fixed_bytes().to_vec()], + addresses: vec![addr.to_vec()], signatures: vec![], }) .collect_vec() @@ -652,9 +976,8 @@ impl EthereumBlockFilter { .block_handlers .clone() .into_iter() - .any(|block_handler| match block_handler.filter { - Some(BlockHandlerFilter::Call) => true, - _ => false, + .any(|block_handler| { + matches!(block_handler.filter, Some(BlockHandlerFilter::Call)) }); let has_block_handler_without_filter = data_source @@ -861,13 +1184,13 @@ impl SubgraphEthRpcMetrics { pub fn observe_request(&self, duration: f64, method: &str, provider: &str) { self.request_duration - .with_label_values(&[&self.deployment, method, provider]) + .with_label_values(&[self.deployment.as_str(), method, provider]) .set(duration); } pub fn add_error(&self, method: &str, provider: &str) { self.errors - .with_label_values(&[&self.deployment, method, provider]) + .with_label_values(&[self.deployment.as_str(), method, provider]) .inc(); } } @@ -885,23 +1208,8 @@ pub trait EthereumAdapter: Send + Sync + 'static { /// connected to. async fn net_identifiers(&self) -> Result; - /// Get the latest block, including full transactions. - fn latest_block( - &self, - logger: &Logger, - ) -> Box + Send + Unpin>; - /// Get the latest block, with only the header and transaction hashes. - fn latest_block_header( - &self, - logger: &Logger, - ) -> Box, Error = bc::IngestorError> + Send>; - - fn load_block( - &self, - logger: &Logger, - block_hash: H256, - ) -> Box + Send>; + async fn latest_block_ptr(&self, logger: &Logger) -> Result; /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. /// May use the `chain_store` as a cache. @@ -909,74 +1217,93 @@ pub trait EthereumAdapter: Send + Sync + 'static { &self, logger: Logger, chain_store: Arc, - block_hashes: HashSet, - ) -> Box, Error = Error> + Send>; + block_hashes: HashSet, + ) -> Result>, Error>; /// Find a block by its hash. - fn block_by_hash( + async fn block_by_hash( &self, logger: &Logger, - block_hash: H256, - ) -> Box, Error = Error> + Send>; + block_hash: B256, + ) -> Result, Error>; - fn block_by_number( + async fn block_by_number( &self, logger: &Logger, block_number: BlockNumber, - ) -> Box, Error = Error> + Send>; + ) -> Result, Error>; /// Load full information for the specified `block` (in particular, transaction receipts). - fn load_full_block( + async fn load_full_block( &self, logger: &Logger, - block: LightEthereumBlock, - ) -> Pin> + Send>>; + block: AnyBlock, + ) -> Result; - /// Load block pointer for the specified `block number`. - fn block_pointer_from_number( + /// Finds the hash and number of the lowest non-null block with height greater than or equal to + /// the given number. + /// + /// Note that the same caveats on reorgs apply as for `block_hash_by_block_number`, and must + /// also be considered for the resolved block, in case it is higher than the requested number. + async fn next_existing_ptr_to_number( &self, logger: &Logger, block_number: BlockNumber, - ) -> Box + Send>; + ) -> Result; - /// Find a block by its number, according to the Ethereum node. - /// - /// Careful: don't use this function without considering race conditions. - /// Chain reorgs could happen at any time, and could affect the answer received. - /// Generally, it is only safe to use this function with blocks that have received enough - /// confirmations to guarantee no further reorgs, **and** where the Ethereum node is aware of - /// those confirmations. - /// If the Ethereum node is far behind in processing blocks, even old blocks can be subject to - /// reorgs. - fn block_hash_by_block_number( + /// Call the function of a smart contract. A return of `None` indicates + /// that the call reverted. The returned `CallSource` indicates where + /// the result came from for accounting purposes + async fn contract_call( &self, logger: &Logger, - block_number: BlockNumber, - ) -> Box, Error = Error> + Send>; + call: &ContractCall, + cache: Arc, + ) -> Result<(Option>, call::Source), ContractCallError>; - /// Call the function of a smart contract. - fn contract_call( + /// Make multiple contract calls in a single batch. The returned `Vec` + /// has results in the same order as the calls in `calls` on input. The + /// calls must all be for the same block + async fn contract_calls( &self, logger: &Logger, - call: EthereumContractCall, + calls: &[&ContractCall], cache: Arc, - ) -> Box, Error = EthereumContractCallError> + Send>; + ) -> Result>, call::Source)>, ContractCallError>; + + async fn get_balance( + &self, + logger: &Logger, + address: Address, + block_ptr: BlockPtr, + ) -> Result; + + // Returns the compiled bytecode of a smart contract + async fn get_code( + &self, + logger: &Logger, + address: Address, + block_ptr: BlockPtr, + ) -> Result; + + /// Returns a boolean indicating whether the adapter can reach + /// the RPC provider it is configured to use. + /// This is used to determine if a provider should be considered healthy. + async fn is_reachable(&self) -> bool; } #[cfg(test)] mod tests { - use crate::adapter::{FunctionSelector, COMBINED_FILTER_TYPE_URL}; + use crate::adapter::{COMBINED_FILTER_TYPE_URL, FunctionSelector}; use super::{EthereumBlockFilter, LogFilterNode}; use super::{EthereumCallFilter, EthereumLogFilter, TriggerFilter}; + use base64::prelude::*; use graph::blockchain::TriggerFilter as _; use graph::firehose::{CallToFilter, CombinedFilter, LogFilter, MultiLogFilter}; - use graph::petgraph::graphmap::GraphMap; - use graph::prelude::ethabi::ethereum_types::H256; - use graph::prelude::web3::types::Address; - use graph::prelude::web3::types::Bytes; use graph::prelude::EthereumCall; + use graph::prelude::alloy::primitives::{Address, B256, Bytes, U256}; use hex::ToHex; use itertools::Itertools; use prost::Message; @@ -1002,7 +1329,7 @@ mod tests { .map(|addr| { format!( "0x{}", - H256::from_str(addr) + B256::from_str(addr) .expect("unable to parse addr") .encode_hex::() ) @@ -1013,16 +1340,11 @@ mod tests { let sigs = event_sigs .iter() - .map(|addr| { - H256::from_str(addr) - .expect("unable to parse addr") - .to_fixed_bytes() - .to_vec() - }) + .map(|addr| B256::from_str(addr).expect("unable to parse addr").to_vec()) .collect_vec(); let filter = LogFilter { - addresses: vec![address.to_fixed_bytes().to_vec()], + addresses: vec![address.to_vec()], event_signatures: sigs, }; // This base64 was provided by Streamingfast as a binding example of the expected encoded for the @@ -1033,7 +1355,7 @@ mod tests { log_filters: vec![filter], }; - let output = base64::encode(filter.encode_to_vec()); + let output = BASE64_STANDARD.encode(filter.encode_to_vec()); assert_eq!(expected_base64, output); } @@ -1049,10 +1371,11 @@ mod tests { assert_eq!(sig, actual_sig); let filter = LogFilter { - addresses: vec![Address::from_str(hex_addr) - .expect("failed to parse address") - .to_fixed_bytes() - .to_vec()], + addresses: vec![ + Address::from_str(hex_addr) + .expect("failed to parse address") + .to_vec(), + ], event_signatures: vec![fs.to_vec()], }; @@ -1060,19 +1383,15 @@ mod tests { // addresses and signatures above. let expected_base64 = "ChTu0rd1bilakwDlPdBJrrB1GJm64xIEqQWcuw=="; - let output = base64::encode(filter.encode_to_vec()); + let output = BASE64_STANDARD.encode(filter.encode_to_vec()); assert_eq!(expected_base64, output); } #[test] fn ethereum_trigger_filter_to_firehose() { - let address = Address::from_low_u64_be; - let sig = H256::from_low_u64_le; + let sig = |value: u64| B256::from(U256::from(value)); let mut filter = TriggerFilter { - log: EthereumLogFilter { - contracts_and_events_graph: GraphMap::new(), - wildcard_events: HashMap::new(), - }, + log: EthereumLogFilter::default(), call: EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), @@ -1096,42 +1415,42 @@ mod tests { let expected_call_filters = vec![ CallToFilter { - addresses: vec![address(0).to_fixed_bytes().to_vec()], + addresses: vec![address(0).to_vec()], signatures: vec![[0u8; 4].to_vec()], }, CallToFilter { - addresses: vec![address(1).to_fixed_bytes().to_vec()], + addresses: vec![address(1).to_vec()], signatures: vec![[1u8; 4].to_vec()], }, CallToFilter { - addresses: vec![address(2).to_fixed_bytes().to_vec()], + addresses: vec![address(2).to_vec()], signatures: vec![], }, CallToFilter { - addresses: vec![address(1000).to_fixed_bytes().to_vec()], + addresses: vec![address(1000).to_vec()], signatures: vec![], }, CallToFilter { - addresses: vec![address(2000).to_fixed_bytes().to_vec()], + addresses: vec![address(2000).to_vec()], signatures: vec![], }, CallToFilter { - addresses: vec![address(3000).to_fixed_bytes().to_vec()], + addresses: vec![address(3000).to_vec()], signatures: vec![], }, ]; - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(100)), false, ); - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(101)), false, ); - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(20)), LogFilterNode::Event(sig(100)), false, @@ -1139,15 +1458,12 @@ mod tests { let expected_log_filters = vec![ LogFilter { - addresses: vec![address(10).to_fixed_bytes().to_vec()], - event_signatures: vec![sig(101).to_fixed_bytes().to_vec()], + addresses: vec![address(10).to_vec()], + event_signatures: vec![sig(101).to_vec()], }, LogFilter { - addresses: vec![ - address(10).to_fixed_bytes().to_vec(), - address(20).to_fixed_bytes().to_vec(), - ], - event_signatures: vec![sig(100).to_fixed_bytes().to_vec()], + addresses: vec![address(10).to_vec(), address(20).to_vec()], + event_signatures: vec![sig(100).to_vec()], }, ]; @@ -1186,18 +1502,15 @@ mod tests { filter.event_signatures.sort(); } assert_eq!(expected_log_filters, actual_log_filters); - assert_eq!(true, actual_send_all_block_headers); + assert!(actual_send_all_block_headers); } #[test] fn ethereum_trigger_filter_to_firehose_every_block_plus_logfilter() { - let address = Address::from_low_u64_be; - let sig = H256::from_low_u64_le; + let address = |value: u64| Address::left_padding_from(&value.to_le_bytes()); + let sig = |value: u64| B256::left_padding_from(&value.to_le_bytes()); let mut filter = TriggerFilter { - log: EthereumLogFilter { - contracts_and_events_graph: GraphMap::new(), - wildcard_events: HashMap::new(), - }, + log: EthereumLogFilter::default(), call: EthereumCallFilter { contract_addresses_function_signatures: HashMap::new(), wildcard_signatures: HashSet::new(), @@ -1209,15 +1522,15 @@ mod tests { }, }; - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(101)), false, ); let expected_log_filters = vec![LogFilter { - addresses: vec![address(10).to_fixed_bytes().to_vec()], - event_signatures: vec![sig(101).to_fixed_bytes().to_vec()], + addresses: vec![address(10).to_vec()], + event_signatures: vec![sig(101).to_vec()], }]; let firehose_filter = filter.clone().to_firehose_filter(); @@ -1252,7 +1565,7 @@ mod tests { } assert_eq!(expected_log_filters, actual_log_filters); - assert_eq!(true, actual_send_all_block_headers); + assert!(actual_send_all_block_headers); } #[test] @@ -1279,76 +1592,63 @@ mod tests { wildcard_signatures: HashSet::from_iter(vec![[11u8; 4]]), }; - assert_eq!( - false, - filter.matches(&call(address(2), vec![])), + assert!( + !filter.matches(&call(address(2), vec![])), "call with empty bytes are always ignore, whatever the condition" ); - assert_eq!( - false, - filter.matches(&call(address(4), vec![1; 36])), + assert!( + !filter.matches(&call(address(4), vec![1; 36])), "call with incorrect address should be ignored" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(1), vec![1; 36])), "call with correct address & signature should match" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(1), vec![1; 32])), "call with correct address & signature, but with incorrect input size should match" ); - assert_eq!( - false, - filter.matches(&call(address(1), vec![4u8; 36])), + assert!( + !filter.matches(&call(address(1), vec![4u8; 36])), "call with correct address but incorrect signature for a specific contract filter (i.e. matches some signatures) should be ignored" ); - assert_eq!( - false, - filter.matches(&call(address(0), vec![11u8; 36])), + assert!( + !filter.matches(&call(address(0), vec![11u8; 36])), "this signature should not match filter1, this avoid false passes if someone changes the code" ); - assert_eq!( - false, - filter2.matches(&call(address(1), vec![10u8; 36])), + assert!( + !filter2.matches(&call(address(1), vec![10u8; 36])), "this signature should not match filter2 because the address is not the expected one" ); - assert_eq!( - true, + assert!( filter2.matches(&call(address(0), vec![10u8; 36])), "this signature should match filter2 on the non wildcard clause" ); - assert_eq!( - true, + assert!( filter2.matches(&call(address(0), vec![11u8; 36])), "this signature should match filter2 on the wildcard clause" ); // extend filter1 and test the filter 2 stuff again filter.extend(filter2); - assert_eq!( - true, + assert!( filter.matches(&call(address(0), vec![11u8; 36])), "this signature should not match filter1, this avoid false passes if someone changes the code" ); - assert_eq!( - false, - filter.matches(&call(address(1), vec![10u8; 36])), + assert!( + !filter.matches(&call(address(1), vec![10u8; 36])), "this signature should not match filter2 because the address is not the expected one" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(0), vec![10u8; 36])), "this signature should match filter2 on the non wildcard clause" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(0), vec![11u8; 36])), "this signature should match filter2 on the wildcard clause" ); @@ -1448,12 +1748,12 @@ mod tests { base.extend(extension); - assert_eq!(true, base.trigger_every_block); + assert!(base.trigger_every_block); } #[test] - fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses_and_polling_intervals( - ) { + fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses_and_polling_intervals() + { let mut base = EthereumBlockFilter { polling_intervals: HashSet::from_iter(vec![(10, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(2))]), @@ -1468,7 +1768,7 @@ mod tests { base.extend(extension); - assert_eq!(true, base.trigger_every_block); + assert!(base.trigger_every_block); assert_eq!( HashSet::from_iter(vec![(10, address(2))]), base.contract_addresses, @@ -1492,7 +1792,7 @@ mod tests { base.extend(extension); - assert_eq!(true, base.trigger_every_block); + assert!(base.trigger_every_block); assert_eq!( HashSet::from_iter(vec![(10, address(2)), (10, address(1))]), base.contract_addresses, @@ -1507,51 +1807,36 @@ mod tests { fn extending_ethereum_call_filter() { let mut base = EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ - ( - Address::from_low_u64_be(0), - (0, HashSet::from_iter(vec![[0u8; 4]])), - ), - ( - Address::from_low_u64_be(1), - (1, HashSet::from_iter(vec![[1u8; 4]])), - ), + (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), + (address(1), (1, HashSet::from_iter(vec![[1u8; 4]]))), ]), wildcard_signatures: HashSet::new(), }; let extension = EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ - ( - Address::from_low_u64_be(0), - (2, HashSet::from_iter(vec![[2u8; 4]])), - ), - ( - Address::from_low_u64_be(3), - (3, HashSet::from_iter(vec![[3u8; 4]])), - ), + (address(0), (2, HashSet::from_iter(vec![[2u8; 4]]))), + (address(3), (3, HashSet::from_iter(vec![[3u8; 4]]))), ]), wildcard_signatures: HashSet::new(), }; base.extend(extension); assert_eq!( - base.contract_addresses_function_signatures - .get(&Address::from_low_u64_be(0)), + base.contract_addresses_function_signatures.get(&address(0)), Some(&(0, HashSet::from_iter(vec![[0u8; 4], [2u8; 4]]))) ); assert_eq!( - base.contract_addresses_function_signatures - .get(&Address::from_low_u64_be(3)), + base.contract_addresses_function_signatures.get(&address(3)), Some(&(3, HashSet::from_iter(vec![[3u8; 4]]))) ); assert_eq!( - base.contract_addresses_function_signatures - .get(&Address::from_low_u64_be(1)), + base.contract_addresses_function_signatures.get(&address(1)), Some(&(1, HashSet::from_iter(vec![[1u8; 4]]))) ); } - fn address(id: u64) -> Address { - Address::from_low_u64_be(id) + fn address(value: u64) -> Address { + Address::left_padding_from(&value.to_be_bytes()) } fn bytes(value: Vec) -> Bytes { @@ -1567,16 +1852,16 @@ fn complete_log_filter() { // Test a few combinations of complete graphs. for i in [1, 2] { - let events: BTreeSet<_> = (0..i).map(H256::from_low_u64_le).collect(); + let events: BTreeSet<_> = (0..i).map(|n| B256::from([n as u8; 32])).collect(); for j in [1, 1000, 2000, 3000] { - let contracts: BTreeSet<_> = (0..j).map(Address::from_low_u64_le).collect(); + let contracts: BTreeSet<_> = (0..j).map(|n| Address::from([n as u8; 20])).collect(); // Construct the complete bipartite graph with i events and j contracts. - let mut contracts_and_events_graph = GraphMap::new(); + let mut filter = EthereumLogFilter::default(); for &contract in &contracts { for &event in &events { - contracts_and_events_graph.add_edge( + filter.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(contract), LogFilterNode::Event(event), false, @@ -1585,12 +1870,9 @@ fn complete_log_filter() { } // Run `eth_get_logs_filters`, which is what we want to test. - let logs_filters: Vec<_> = EthereumLogFilter { - contracts_and_events_graph, - wildcard_events: HashMap::new(), - } - .eth_get_logs_filters() - .collect(); + let logs_filters: Vec<_> = filter + .eth_get_logs_filters(ENV_VARS.get_logs_max_contracts) + .collect(); // Assert that a contract or event is filtered on iff it was present in the graph. assert_eq!( @@ -1618,27 +1900,46 @@ fn complete_log_filter() { } } +#[test] +fn test_call_filter_first_signature_not_lost() { + use crate::adapter::{EthereumCallFilter, FunctionSelector}; + use alloy::primitives::Address; + + let addr = Address::left_padding_from(&1u64.to_be_bytes()); + let sig1: FunctionSelector = [0xaa, 0xbb, 0xcc, 0xdd]; + let sig2: FunctionSelector = [0x11, 0x22, 0x33, 0x44]; + + let filter: EthereumCallFilter = vec![(100i32, addr, sig1), (100i32, addr, sig2)] + .into_iter() + .collect(); + + let (_, sigs) = filter + .contract_addresses_function_signatures + .get(&addr) + .unwrap(); + assert_eq!(sigs.len(), 2); + assert!(sigs.contains(&sig1)); + assert!(sigs.contains(&sig2)); +} + #[test] fn log_filter_require_transacion_receipt_method() { + let address = |value: u64| Address::left_padding_from(&value.to_be_bytes()); + let b256 = |value: u64| B256::left_padding_from(&value.to_be_bytes()); + // test data - let event_signature_a = H256::zero(); - let event_signature_b = H256::from_low_u64_be(1); - let event_signature_c = H256::from_low_u64_be(2); - let contract_a = Address::from_low_u64_be(3); - let contract_b = Address::from_low_u64_be(4); - let contract_c = Address::from_low_u64_be(5); - - let wildcard_event_with_receipt = H256::from_low_u64_be(6); - let wildcard_event_without_receipt = H256::from_low_u64_be(7); - let wildcard_events = [ - (wildcard_event_with_receipt, true), - (wildcard_event_without_receipt, false), - ] - .into_iter() - .collect(); - - let alien_event_signature = H256::from_low_u64_be(8); // those will not be inserted in the graph - let alien_contract_address = Address::from_low_u64_be(9); + let event_signature_a = b256(0); + let event_signature_b = b256(1); + let event_signature_c = b256(2); + let contract_a = address(3); + let contract_b = address(4); + let contract_c = address(5); + + let wildcard_event_with_receipt = b256(6); + let wildcard_event_without_receipt = b256(7); + + let alien_event_signature = b256(8); // those will not be inserted in the graph + let alien_contract_address = address(9); // test graph nodes let event_a_node = LogFilterNode::Event(event_signature_a); @@ -1663,51 +1964,376 @@ fn log_filter_require_transacion_receipt_method() { // event_b -- contract_a [ receipt=false ] // } // ``` - let mut contracts_and_events_graph = GraphMap::new(); - - let event_a_id = contracts_and_events_graph.add_node(event_a_node); - let event_b_id = contracts_and_events_graph.add_node(event_b_node); - let event_c_id = contracts_and_events_graph.add_node(event_c_node); - let contract_a_id = contracts_and_events_graph.add_node(contract_a_node); - let contract_b_id = contracts_and_events_graph.add_node(contract_b_node); - let contract_c_id = contracts_and_events_graph.add_node(contract_c_node); - contracts_and_events_graph.add_edge(event_a_id, contract_a_id, true); - contracts_and_events_graph.add_edge(event_b_id, contract_b_id, true); - contracts_and_events_graph.add_edge(event_a_id, contract_b_id, false); - contracts_and_events_graph.add_edge(event_b_id, contract_a_id, false); - contracts_and_events_graph.add_edge(event_c_id, contract_c_id, true); - - let filter = EthereumLogFilter { - contracts_and_events_graph, - wildcard_events, - }; + let mut filter = EthereumLogFilter::default(); + filter + .contracts_and_events_graph + .or_add_edge(event_a_node, contract_a_node, true); + filter + .contracts_and_events_graph + .or_add_edge(event_b_node, contract_b_node, true); + filter + .contracts_and_events_graph + .or_add_edge(event_a_node, contract_b_node, false); + filter + .contracts_and_events_graph + .or_add_edge(event_b_node, contract_a_node, false); + filter + .contracts_and_events_graph + .or_add_edge(event_c_node, contract_c_node, true); + filter + .wildcard_events + .or_insert(wildcard_event_with_receipt, true); + filter + .wildcard_events + .or_insert(wildcard_event_without_receipt, false); + + let empty_vec: Vec = vec![]; // connected contracts and events graph - assert!(filter.requires_transaction_receipt(&event_signature_a, Some(&contract_a))); - assert!(filter.requires_transaction_receipt(&event_signature_b, Some(&contract_b))); - assert!(filter.requires_transaction_receipt(&event_signature_c, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&contract_b))); - assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&contract_a))); + assert!(filter.requires_transaction_receipt(&event_signature_a, Some(&contract_a), &empty_vec)); + assert!(filter.requires_transaction_receipt(&event_signature_b, Some(&contract_b), &empty_vec)); + assert!(filter.requires_transaction_receipt(&event_signature_c, Some(&contract_c), &empty_vec)); + assert!(!filter.requires_transaction_receipt( + &event_signature_a, + Some(&contract_b), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_b, + Some(&contract_a), + &empty_vec + )); // Event C and Contract C are not connected to the other events and contracts - assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&contract_a))); - assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&contract_b))); + assert!(!filter.requires_transaction_receipt( + &event_signature_a, + Some(&contract_c), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_b, + Some(&contract_c), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_c, + Some(&contract_a), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_c, + Some(&contract_b), + &empty_vec + )); // Wildcard events - assert!(filter.requires_transaction_receipt(&wildcard_event_with_receipt, None)); - assert!(!filter.requires_transaction_receipt(&wildcard_event_without_receipt, None)); + assert!(filter.requires_transaction_receipt(&wildcard_event_with_receipt, None, &empty_vec)); + assert!(!filter.requires_transaction_receipt( + &wildcard_event_without_receipt, + None, + &empty_vec + )); // Alien events and contracts always return false - assert!( - !filter.requires_transaction_receipt(&alien_event_signature, Some(&alien_contract_address)) + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&alien_contract_address), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt(&alien_event_signature, None, &empty_vec),); + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&contract_a), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&contract_b), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&contract_c), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_a, + Some(&alien_contract_address), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_b, + Some(&alien_contract_address), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_c, + Some(&alien_contract_address), + &empty_vec + )); +} + +// Tests that `EthereumLogFilter` OR-merges per-handler `receipt` flags across +// every insertion site (`from_data_sources`, `from_mapping`, `extend`). + +#[cfg(test)] +fn receipt_merge_test_addr(n: u64) -> Address { + Address::left_padding_from(&n.to_be_bytes()) +} + +#[cfg(test)] +fn receipt_merge_test_sig(n: u64) -> B256 { + B256::left_padding_from(&n.to_be_bytes()) +} + +#[cfg(test)] +fn receipt_merge_test_mock_abi() -> std::sync::Arc { + std::sync::Arc::new(graph::data_source::common::MappingABI { + name: "mock_abi".to_string(), + contract: abi::JsonAbi::new(), + }) +} + +#[cfg(test)] +fn receipt_merge_test_event_handler( + sig: B256, + topic1: Option>, + topic2: Option>, + topic3: Option>, + receipt: bool, +) -> crate::data_source::MappingEventHandler { + crate::data_source::MappingEventHandler { + event: "Event()".to_string(), + topic0: Some(sig), + topic1, + topic2, + topic3, + handler: "handleEvent".to_string(), + receipt, + calls: graph::data_source::common::CallDecls::default(), + } +} + +#[cfg(test)] +fn receipt_merge_test_mapping( + handlers: Vec, +) -> crate::Mapping { + crate::Mapping { + kind: "ethereum/events".to_string(), + api_version: semver::Version::new(0, 0, 7), + language: "wasm/assemblyscript".to_string(), + entities: vec![], + abis: vec![receipt_merge_test_mock_abi()], + block_handlers: vec![], + call_handlers: vec![], + event_handlers: handlers, + runtime: std::sync::Arc::new(vec![]), + link: graph::prelude::Link { + link: "test".to_string(), + }, + } +} + +#[cfg(test)] +fn receipt_merge_test_data_source( + address: Option
, + handlers: Vec, +) -> crate::data_source::DataSource { + crate::data_source::DataSource { + kind: "ethereum/contract".to_string(), + network: Some("test".to_string()), + name: "Test".to_string(), + manifest_idx: 0, + address, + start_block: 0, + end_block: None, + mapping: receipt_merge_test_mapping(handlers), + context: std::sync::Arc::new(None), + creation_block: None, + contract_abi: receipt_merge_test_mock_abi(), + } +} + +/// Run two data sources with `receipt: true` and `receipt: false` at the +/// same effective filter key through `from_data_sources` and assert the +/// merged filter still requires a transaction receipt. Runs both +/// declaration orders so order-independence is verified per variant. +#[cfg(test)] +fn assert_from_data_sources_or_merges( + label: &str, + address: Option
, + topic1: Option>, + log_topics: &[B256], +) { + let event_sig = receipt_merge_test_sig(100); + let ds_yes = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + true, + )], ); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, None)); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_a))); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_b))); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&alien_contract_address))); - assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&alien_contract_address))); - assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&alien_contract_address))); + let ds_no = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + false, + )], + ); + + for (order, dss) in [("yes,no", [&ds_yes, &ds_no]), ("no,yes", [&ds_no, &ds_yes])] { + let filter = EthereumLogFilter::from_data_sources(dss); + assert!( + filter.requires_transaction_receipt(&event_sig, address.as_ref(), log_topics), + "{label} ({order}): receipt:true must survive a later receipt:false", + ); + } +} + +/// Build two filters via `from_data_sources`, each with one handler at the +/// same effective key but with opposite receipt flags, then merge them via +/// `extend` and assert the merged filter still requires a transaction +/// receipt. Runs both extend directions so order-independence is verified +/// per variant. +#[cfg(test)] +fn assert_extend_or_merges( + label: &str, + address: Option
, + topic1: Option>, + log_topics: &[B256], +) { + let event_sig = receipt_merge_test_sig(105); + let ds_yes = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + true, + )], + ); + let ds_no = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + false, + )], + ); + + for (order, base, ext) in [ + ("yes.extend(no)", &ds_yes, &ds_no), + ("no.extend(yes)", &ds_no, &ds_yes), + ] { + let mut filter = EthereumLogFilter::from_data_sources([base]); + filter.extend(EthereumLogFilter::from_data_sources([ext])); + assert!( + filter.requires_transaction_receipt(&event_sig, address.as_ref(), log_topics), + "{label} ({order}): extend must OR-merge", + ); + } +} + +#[test] +fn from_data_sources_or_merges_at_every_insertion_site() { + let contract = receipt_merge_test_addr(1); + let event_sig = receipt_merge_test_sig(100); + let topic = receipt_merge_test_sig(200); + let with_topic = vec![event_sig, topic]; + + // Each case maps to one arm of the `match ds.address` in `from_data_sources`. + assert_from_data_sources_or_merges("graph edge", Some(contract), None, &[]); + assert_from_data_sources_or_merges( + "addressed topics", + Some(contract), + Some(vec![topic]), + &with_topic, + ); + assert_from_data_sources_or_merges("wildcard", None, None, &[]); + assert_from_data_sources_or_merges("wildcard topics", None, Some(vec![topic]), &with_topic); +} + +#[test] +fn from_mapping_or_merges_via_extend() { + // Two templates handling the same event signature with different receipt + // flags merged via `from_mapping` + `extend`. + let event_sig = receipt_merge_test_sig(104); + + let mapping_yes = receipt_merge_test_mapping(vec![receipt_merge_test_event_handler( + event_sig, None, None, None, true, + )]); + let mapping_no = receipt_merge_test_mapping(vec![receipt_merge_test_event_handler( + event_sig, None, None, None, false, + )]); + + let mut filter = EthereumLogFilter::from_mapping(&mapping_yes); + filter.extend(EthereumLogFilter::from_mapping(&mapping_no)); + + assert!(filter.requires_transaction_receipt(&event_sig, None, &[])); +} + +#[test] +fn extend_or_merges_at_every_collection() { + let contract = receipt_merge_test_addr(5); + let event_sig = receipt_merge_test_sig(105); + let topic = receipt_merge_test_sig(203); + let with_topic = vec![event_sig, topic]; + + assert_extend_or_merges("graph edge", Some(contract), None, &[]); + assert_extend_or_merges( + "addressed topics", + Some(contract), + Some(vec![topic]), + &with_topic, + ); + assert_extend_or_merges("wildcard", None, None, &[]); + assert_extend_or_merges("wildcard topics", None, Some(vec![topic]), &with_topic); +} + +#[test] +fn requires_transaction_receipt_has_or_semantics_across_handlers() { + // `requires_transaction_receipt` must return true iff at least one handler + // at the key declared `receipt: true`, regardless of declaration order. + let event_sig = receipt_merge_test_sig(108); + + for flags in [ + vec![false], + vec![true], + vec![true, false], + vec![false, true], + vec![false, false, true], + vec![true, false, false], + vec![false, true, false], + vec![true, true, false], + vec![false, false, false], + ] { + let dss: Vec<_> = flags + .iter() + .map(|&r| { + receipt_merge_test_data_source( + None, + vec![receipt_merge_test_event_handler( + event_sig, None, None, None, r, + )], + ) + }) + .collect(); + let filter = EthereumLogFilter::from_data_sources(dss.iter()); + + let any_requires_receipt = flags.iter().any(|&r| r); + assert_eq!( + filter.requires_transaction_receipt(&event_sig, None, &[]), + any_requires_receipt, + "flag sequence {flags:?}", + ); + } } diff --git a/chain/ethereum/src/buffered_call_cache.rs b/chain/ethereum/src/buffered_call_cache.rs new file mode 100644 index 00000000000..29aa84e90ab --- /dev/null +++ b/chain/ethereum/src/buffered_call_cache.rs @@ -0,0 +1,146 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use graph::{ + cheap_clone::CheapClone, + components::store::EthereumCallCache, + data::store::ethereum::call, + prelude::{BlockPtr, CachedEthereumCall}, + slog::{Logger, error}, +}; + +/// A wrapper around an Ethereum call cache that buffers call results in +/// memory for the duration of a block. If `get_call` or `set_call` are +/// called with a different block pointer than the one used in the previous +/// call, the buffer is cleared. +pub struct BufferedCallCache { + call_cache: Arc, + buffer: Arc>>, + block: Arc>>, +} + +impl BufferedCallCache { + pub fn new(call_cache: Arc) -> Self { + Self { + call_cache, + buffer: Arc::new(Mutex::new(HashMap::new())), + block: Arc::new(Mutex::new(None)), + } + } + + fn check_block(&self, block: &BlockPtr) { + let mut self_block = self.block.lock().unwrap(); + if self_block.as_ref() != Some(block) { + *self_block = Some(block.clone()); + self.buffer.lock().unwrap().clear(); + } + } + + fn get(&self, call: &call::Request) -> Option { + let buffer = self.buffer.lock().unwrap(); + buffer.get(call).map(|retval| { + call.cheap_clone() + .response(retval.clone(), call::Source::Memory) + }) + } +} + +#[async_trait] +impl EthereumCallCache for BufferedCallCache { + async fn get_call( + &self, + call: &call::Request, + block: BlockPtr, + ) -> Result, graph::prelude::Error> { + self.check_block(&block); + + if let Some(value) = self.get(call) { + return Ok(Some(value)); + } + + let result = self.call_cache.get_call(call, block).await?; + + let mut buffer = self.buffer.lock().unwrap(); + if let Some(call::Response { + retval, + req: _, + source: _, + }) = &result + { + buffer.insert(call.cheap_clone(), retval.clone()); + } + Ok(result) + } + + async fn get_calls( + &self, + reqs: &[call::Request], + block: BlockPtr, + ) -> Result<(Vec, Vec), graph::prelude::Error> { + self.check_block(&block); + + let mut missing = Vec::new(); + let mut resps = Vec::new(); + + for call in reqs { + match self.get(call) { + Some(resp) => resps.push(resp), + None => missing.push(call.cheap_clone()), + } + } + + let (stored, calls) = self.call_cache.get_calls(&missing, block).await?; + + { + let mut buffer = self.buffer.lock().unwrap(); + for resp in &stored { + buffer.insert(resp.req.cheap_clone(), resp.retval.clone()); + } + } + + resps.extend(stored); + Ok((resps, calls)) + } + + async fn get_calls_in_block( + &self, + block: BlockPtr, + ) -> Result, graph::prelude::Error> { + self.call_cache.get_calls_in_block(block).await + } + + async fn set_call( + self: Arc, + logger: &Logger, + call: call::Request, + block: BlockPtr, + return_value: call::Retval, + ) -> Result<(), graph::prelude::Error> { + self.check_block(&block); + + // Enter the call into the in-memory cache immediately so that + // handlers will find it, but add it to the underlying cache in the + // background so we do not have to wait for that as it will be a + // cache backed by the database + { + let mut buffer = self.buffer.lock().unwrap(); + buffer.insert(call.cheap_clone(), return_value.clone()); + } + + let cache = self.call_cache.cheap_clone(); + let logger = logger.cheap_clone(); + if let Err(e) = cache + .set_call(&logger, call.cheap_clone(), block, return_value) + .await + { + error!(logger, "BufferedCallCache: call cache set error"; + "contract_address" => format!("{:?}", call.address), + "error" => e.to_string()) + } + + Ok(()) + } +} diff --git a/chain/ethereum/src/call_helper.rs b/chain/ethereum/src/call_helper.rs new file mode 100644 index 00000000000..8a1216b58c5 --- /dev/null +++ b/chain/ethereum/src/call_helper.rs @@ -0,0 +1,156 @@ +use crate::{ContractCallError, ENV_VARS}; +use graph::{ + abi, + data::store::ethereum::call, + prelude::{ + Logger, + alloy::transports::{RpcError, TransportErrorKind}, + serde_json, + }, + slog::info, +}; + +// ------------------------------------------------------------------ +// Constants and helper utilities used across eth_call handling +// ------------------------------------------------------------------ + +// Try to check if the call was reverted. The JSON-RPC response for reverts is +// not standardized, so we have ad-hoc checks for each Ethereum client. + +// 0xfe is the "designated bad instruction" of the EVM, and Solidity uses it for +// asserts. +const PARITY_BAD_INSTRUCTION_FE: &str = "Bad instruction fe"; + +// 0xfd is REVERT, but on some contracts, and only on older blocks, +// this happens. Makes sense to consider it a revert as well. +const PARITY_BAD_INSTRUCTION_FD: &str = "Bad instruction fd"; + +const PARITY_BAD_JUMP_PREFIX: &str = "Bad jump"; +const PARITY_STACK_LIMIT_PREFIX: &str = "Out of stack"; + +// See f0af4ab0-6b7c-4b68-9141-5b79346a5f61. +const PARITY_OUT_OF_GAS: &str = "Out of gas"; + +// Also covers Nethermind reverts +const PARITY_VM_EXECUTION_ERROR: i64 = -32015; +const PARITY_REVERT_PREFIX: &str = "revert"; + +const XDAI_REVERT: &str = "revert"; + +// Deterministic RPC execution errors. We might need to expand this as +// subgraphs come across other errors. See +// https://github.com/ethereum/go-ethereum/blob/cd57d5cd38ef692de8fbedaa56598b4e9fbfbabc/core/vm/errors.go +const RPC_EXECUTION_ERRORS: &[&str] = &[ + // The "revert" substring covers a few known error messages, including: + // Hardhat: "error: transaction reverted", + // Ganache and Moonbeam: "vm exception while processing transaction: revert", + // Geth: "execution reverted" + // And others. + "revert", + "invalid jump destination", + "invalid opcode", + // Ethereum says 1024 is the stack sizes limit, so this is deterministic. + "stack limit reached 1024", + // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61 for why the gas limit is considered deterministic. + "out of gas", + "stack underflow", + "vm execution error", + "invalidjump", + "notactivated", + "invalidfeopcode", + // Reth surfaces EVM halts via `EvmHalt(HaltReason)`, formatted with the + // reason's `Debug` repr (`"EVM error: {0:?}"`). revm's `HaltReason` + // variants are CamelCase with no spaces, so e.g. a stack underflow arrives + // as "EVM error: StackUnderflow", which the space-separated "stack + // underflow" above does not match. "invalidjump"/"invalidfeopcode" already + // cover the matching variants; these add the rest. Reth's OutOfGas is + // handled before EvmHalt and rendered as "out of gas: ...", so it is + // already covered above. See https://github.com/streamingfast/eth-go/pull/10. + "stackunderflow", + "stackoverflow", + "opcodenotfound", +]; + +/// Helper that checks if a RPC error message corresponds to a revert. +fn is_rpc_revert_message(message: &str) -> bool { + let env_rpc_call_errors = ENV_VARS.rpc_eth_call_errors.iter(); + let mut execution_errors = RPC_EXECUTION_ERRORS + .iter() + .copied() + .chain(env_rpc_call_errors.map(|s| s.as_str())); + execution_errors.any(|e| message.to_lowercase().contains(e)) +} + +/// Decode a Solidity revert(reason) payload, returning the reason string when possible. +fn as_solidity_revert_reason(bytes: &[u8]) -> Option { + let selector = &graph::prelude::alloy::primitives::keccak256(b"Error(string)")[..4]; + if bytes.len() >= 4 && &bytes[..4] == selector { + abi::DynSolType::String + .abi_decode(&bytes[4..]) + .ok() + .and_then(|val| val.clone().as_str().map(ToOwned::to_owned)) + } else { + None + } +} + +/// Interpret the error returned by `eth_call`, distinguishing genuine failures from +/// EVM reverts. Returns `Ok(Null)` for reverts or a proper error otherwise. +pub fn interpret_eth_call_error( + logger: &Logger, + err: RpcError, +) -> Result { + fn reverted(logger: &Logger, reason: &str) -> Result { + info!(logger, "Contract call reverted"; "reason" => reason); + Ok(call::Retval::Null) + } + + if let RpcError::ErrorResp(rpc_error) = &err + && is_rpc_revert_message(&rpc_error.message) + { + return reverted(logger, &rpc_error.message); + } + + if let RpcError::ErrorResp(rpc_error) = &err { + let code = rpc_error.code; + let data: Option = rpc_error + .data + .as_ref() + .and_then(|d| serde_json::from_str(d.get()).ok()); + + if code == PARITY_VM_EXECUTION_ERROR + && let Some(data) = data + && is_parity_revert(&data) + { + return reverted(logger, &parity_revert_reason(&data)); + } + } + + Err(ContractCallError::AlloyError(err)) +} + +fn is_parity_revert(data: &str) -> bool { + data.to_lowercase().starts_with(PARITY_REVERT_PREFIX) + || data.starts_with(PARITY_BAD_JUMP_PREFIX) + || data.starts_with(PARITY_STACK_LIMIT_PREFIX) + || data == PARITY_BAD_INSTRUCTION_FE + || data == PARITY_BAD_INSTRUCTION_FD + || data == PARITY_OUT_OF_GAS + || data == XDAI_REVERT +} + +/// Checks if the given data corresponds to a Parity / Nethermind style EVM +/// revert and, if so, tries to extract a human-readable revert reason. Returns `Some` +/// with the reason when the error is identified as a revert, otherwise `None`. +fn parity_revert_reason(data: &str) -> String { + if data == PARITY_BAD_INSTRUCTION_FE { + return PARITY_BAD_INSTRUCTION_FE.to_owned(); + } + + // Otherwise try to decode a Solidity revert reason payload. + let payload = data.trim_start_matches(PARITY_REVERT_PREFIX); + hex::decode(payload) + .ok() + .and_then(|decoded| as_solidity_revert_reason(&decoded)) + .unwrap_or_else(|| "no reason".to_owned()) +} diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 32697c15dd2..405c3123bdc 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -1,38 +1,44 @@ -use anyhow::{anyhow, bail, Result}; use anyhow::{Context, Error}; +use anyhow::{Result, anyhow, bail}; +use async_trait::async_trait; use graph::blockchain::client::ChainClient; use graph::blockchain::firehose_block_ingestor::{FirehoseBlockIngestor, Transforms}; -use graph::blockchain::{BlockIngestor, BlockTime, BlockchainKind, TriggersAdapterSelector}; -use graph::components::store::DeploymentCursorTracker; +use graph::blockchain::{ + BlockIngestor, BlockTime, BlockchainKind, ChainIdentifier, ExtendedBlockPtr, + TriggerFilterWrapper, TriggersAdapterSelector, +}; +use graph::components::network_provider::ChainName; +use graph::components::store::{DeploymentCursorTracker, SourceableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::firehose::{FirehoseEndpoint, ForkStep}; +use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints, ForkStep}; +use graph::futures03::TryStreamExt; use graph::prelude::{ BlockHash, ComponentLoggerConfig, ElasticComponentLoggerConfig, EthereumBlock, - EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, + EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, StoreError, + retry, }; -use graph::schema::InputSchema; -use graph::substreams::Clock; +use graph::slog::{debug, error, trace, warn}; use graph::{ blockchain::{ + Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, block_stream::{ BlockRefetcher, BlockStreamEvent, BlockWithTriggers, FirehoseError, FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, }, firehose_block_stream::FirehoseBlockStream, - polling_block_stream::PollingBlockStream, - Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, - RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, }, cheap_clone::CheapClone, components::store::DeploymentLocator, firehose, prelude::{ - async_trait, o, serde_json as json, BlockNumber, ChainStore, EthereumBlockWithCalls, - Future01CompatExt, Logger, LoggerFactory, NodeId, + BlockNumber, ChainStore, EthereumBlockWithCalls, Logger, LoggerFactory, o, + serde_json as json, }, }; use prost::Message; -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; +use std::future::Future; use std::iter::FromIterator; use std::sync::Arc; use std::time::Duration; @@ -42,9 +48,11 @@ use crate::data_source::DataSourceTemplate; use crate::data_source::UnresolvedDataSourceTemplate; use crate::ingestor::PollingBlockIngestor; use crate::network::EthereumNetworkAdapters; -use crate::EthereumAdapter; -use crate::NodeCapabilities; +use crate::polling_block_stream::PollingBlockStream; +use crate::runtime::runtime_adapter::eth_call_gas; +use crate::{BufferedCallCache, NodeCapabilities}; use crate::{ + ENV_VARS, SubgraphEthRpcMetrics, TriggerFilter, adapter::EthereumAdapter as _, codec, data_source::{DataSource, UnresolvedDataSource}, @@ -52,15 +60,53 @@ use crate::{ blocks_with_triggers, get_calls, parse_block_triggers, parse_call_triggers, parse_log_triggers, }, - SubgraphEthRpcMetrics, TriggerFilter, ENV_VARS, }; +use crate::{EthereumAdapter, RuntimeAdapter}; use graph::blockchain::block_stream::{ - BlockStream, BlockStreamBuilder, BlockStreamMapper, FirehoseCursor, + BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, + TriggersAdapterWrapper, }; - /// Celo Mainnet: 42220, Testnet Alfajores: 44787, Testnet Baklava: 62320 const CELO_CHAIN_IDS: [u64; 3] = [42220, 44787, 62320]; +/// Resolved per-chain settings. Populated at chain initialisation from the config file (with +/// ENV_VAR fallbacks) and stored on [`Chain`] and [`crate::EthereumAdapter`]. +#[derive(Clone, Debug)] +pub struct ChainSettings { + pub polling_interval: Duration, + pub json_rpc_timeout: Duration, + pub request_retries: usize, + pub max_block_range_size: BlockNumber, + pub block_batch_size: usize, + pub block_ptr_batch_size: usize, + pub max_event_only_range: BlockNumber, + pub target_triggers_per_block_range: u64, + pub get_logs_max_contracts: usize, + pub block_ingestor_max_concurrent_json_rpc_calls: usize, + pub genesis_block_number: u64, +} + +impl ChainSettings { + /// Constructs a [`ChainSettings`] from environment variable defaults. + /// Used in tests and for firehose-only chains that have no RPC config. + pub fn from_env_defaults() -> Self { + ChainSettings { + polling_interval: graph::env::ENV_VARS.ingestor_polling_interval, + json_rpc_timeout: ENV_VARS.json_rpc_timeout, + request_retries: ENV_VARS.request_retries, + max_block_range_size: ENV_VARS.max_block_range_size, + block_batch_size: ENV_VARS.block_batch_size, + block_ptr_batch_size: ENV_VARS.block_ptr_batch_size, + max_event_only_range: ENV_VARS.max_event_only_range, + target_triggers_per_block_range: ENV_VARS.target_triggers_per_block_range, + get_logs_max_contracts: ENV_VARS.get_logs_max_contracts, + block_ingestor_max_concurrent_json_rpc_calls: ENV_VARS + .block_ingestor_max_concurrent_json_rpc_calls, + genesis_block_number: ENV_VARS.genesis_block_number, + } + } +} + pub struct EthereumStreamBuilder {} #[async_trait] @@ -104,16 +150,26 @@ impl BlockStreamBuilder for EthereumStreamBuilder { ))) } - async fn build_substreams( + async fn build_subgraph_block_stream( &self, - _chain: &Chain, - _schema: InputSchema, - _deployment: DeploymentLocator, - _block_cursor: FirehoseCursor, - _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, + chain: &Chain, + deployment: DeploymentLocator, + start_blocks: Vec, + source_subgraph_stores: Vec>, + subgraph_current_block: Option, + filter: Arc>, + unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - unimplemented!() + self.build_polling( + chain, + deployment, + start_blocks, + source_subgraph_stores, + subgraph_current_block, + filter, + unified_api_version, + ) + .await } async fn build_polling( @@ -121,60 +177,75 @@ impl BlockStreamBuilder for EthereumStreamBuilder { chain: &Chain, deployment: DeploymentLocator, start_blocks: Vec, + source_subgraph_stores: Vec>, subgraph_current_block: Option, - filter: Arc<::TriggerFilter>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - let requirements = filter.node_capabilities(); - let adapter = chain - .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) - .unwrap_or_else(|_| { - panic!( - "no adapter for network {} with capabilities {}", - chain.name, requirements - ) - }); + let requirements = filter.chain_filter.node_capabilities(); + let is_using_subgraph_composition = !source_subgraph_stores.is_empty(); + let adapter = TriggersAdapterWrapper::new( + chain + .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) + .unwrap_or_else(|_| { + panic!( + "no adapter for network {} with capabilities {}", + chain.name, requirements + ) + }), + source_subgraph_stores, + ); let logger = chain .logger_factory .subgraph_logger(&deployment) .new(o!("component" => "BlockStream")); - let chain_store = chain.chain_store(); let chain_head_update_stream = chain .chain_head_update_listener - .subscribe(chain.name.clone(), logger.clone()); + .subscribe(chain.name.to_string(), logger.clone()); // Special case: Detect Celo and set the threshold to 0, so that eth_getLogs is always used. // This is ok because Celo blocks are always final. And we _need_ to do this because // some events appear only in eth_getLogs but not in transaction receipts. // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. - let chain_id = match chain.chain_client().as_ref() { + let reorg_threshold = match chain.chain_client().as_ref() { ChainClient::Rpc(adapter) => { - adapter + let chain_id = adapter .cheapest() + .await .ok_or(anyhow!("unable to get eth adapter for chan_id call"))? .chain_id() - .await? + .await?; + + if CELO_CHAIN_IDS.contains(&chain_id) { + 0 + } else { + chain.reorg_threshold + } } - _ => panic!("expected rpc when using polling blockstream"), + _ if is_using_subgraph_composition => chain.reorg_threshold, + _ => panic!( + "expected rpc when using polling blockstream : {}", + is_using_subgraph_composition + ), }; - let reorg_threshold = match CELO_CHAIN_IDS.contains(&chain_id) { - false => chain.reorg_threshold, - true => 0, + + let max_block_range_size = if is_using_subgraph_composition { + chain.settings.max_block_range_size * 10 + } else { + chain.settings.max_block_range_size }; Ok(Box::new(PollingBlockStream::new( - chain_store, chain_head_update_stream, - adapter, - chain.node_id.clone(), + Arc::new(adapter), deployment.hash, filter, start_blocks, reorg_threshold, logger, - ENV_VARS.max_block_range_size, - ENV_VARS.target_triggers_per_block_range, + max_block_range_size, + chain.settings.target_triggers_per_block_range, unified_api_version, subgraph_current_block, ))) @@ -195,7 +266,7 @@ impl BlockRefetcher for EthereumBlockRefetcher { logger: &Logger, cursor: FirehoseCursor, ) -> Result { - let endpoint = chain.chain_client().firehose_endpoint()?; + let endpoint: Arc = chain.chain_client().firehose_endpoint().await?; let block = endpoint.get_block::(cursor, logger).await?; let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; Ok(BlockFinality::NonFinal(ethereum_block)) @@ -207,6 +278,7 @@ pub struct EthereumAdapterSelector { client: Arc>, registry: Arc, chain_store: Arc, + eth_adapters: Arc, } impl EthereumAdapterSelector { @@ -215,12 +287,14 @@ impl EthereumAdapterSelector { client: Arc>, registry: Arc, chain_store: Arc, + eth_adapters: Arc, ) -> Self { Self { logger_factory, client, registry, chain_store, + eth_adapters, } } } @@ -246,27 +320,57 @@ impl TriggersAdapterSelector for EthereumAdapterSelector { chain_store: self.chain_store.cheap_clone(), unified_api_version, capabilities: *capabilities, + eth_adapters: self.eth_adapters.cheap_clone(), }; Ok(Arc::new(adapter)) } } +/// We need this so that the runner tests can use a `NoopRuntimeAdapter` +/// instead of the `RuntimeAdapter` from this crate to avoid needing +/// ethereum adapters +pub trait RuntimeAdapterBuilder: Send + Sync + 'static { + fn build( + &self, + eth_adapters: Arc, + call_cache: Arc, + chain_identifier: Arc, + ) -> Arc>; +} + +pub struct EthereumRuntimeAdapterBuilder {} + +impl RuntimeAdapterBuilder for EthereumRuntimeAdapterBuilder { + fn build( + &self, + eth_adapters: Arc, + call_cache: Arc, + chain_identifier: Arc, + ) -> Arc> { + Arc::new(RuntimeAdapter { + eth_adapters, + call_cache, + chain_identifier, + }) + } +} + pub struct Chain { logger_factory: LoggerFactory, - name: String, - node_id: NodeId, + pub name: ChainName, registry: Arc, client: Arc>, chain_store: Arc, call_cache: Arc, chain_head_update_listener: Arc, reorg_threshold: BlockNumber, - polling_ingestor_interval: Duration, pub is_ingestible: bool, block_stream_builder: Arc>, block_refetcher: Arc>, adapter_selector: Arc>, - runtime_adapter: Arc>, + runtime_adapter_builder: Arc, + eth_adapters: Arc, + pub settings: Arc, } impl std::fmt::Debug for Chain { @@ -275,12 +379,43 @@ impl std::fmt::Debug for Chain { } } +/// Walk back from a block pointer by following parent pointers. +/// This is the core logic used as a fallback when the cache doesn't have ancestor block. +/// +async fn walk_back_ancestor( + start_ptr: BlockPtr, + offset: BlockNumber, + root: Option, + mut parent_getter: F, +) -> Result, E> +where + F: FnMut(BlockPtr) -> Fut, + Fut: std::future::Future, E>>, +{ + let mut current_ptr = start_ptr; + + for _ in 0..offset { + match parent_getter(current_ptr.clone()).await? { + Some(parent) => { + if let Some(root_hash) = &root + && parent.hash == *root_hash + { + break; + } + current_ptr = parent; + } + None => return Ok(None), + } + } + + Ok(Some(current_ptr)) +} + impl Chain { /// Creates a new Ethereum [`Chain`]. pub fn new( logger_factory: LoggerFactory, - name: String, - node_id: NodeId, + name: ChainName, registry: Arc, chain_store: Arc, call_cache: Arc, @@ -289,15 +424,15 @@ impl Chain { block_stream_builder: Arc>, block_refetcher: Arc>, adapter_selector: Arc>, - runtime_adapter: Arc>, + runtime_adapter_builder: Arc, + eth_adapters: Arc, reorg_threshold: BlockNumber, - polling_ingestor_interval: Duration, is_ingestible: bool, + settings: Arc, ) -> Self { Chain { logger_factory, name, - node_id, registry, client, chain_store, @@ -306,10 +441,11 @@ impl Chain { block_stream_builder, block_refetcher, adapter_selector, - runtime_adapter, + runtime_adapter_builder, + eth_adapters, reorg_threshold, is_ingestible, - polling_ingestor_interval, + settings, } } @@ -318,15 +454,22 @@ impl Chain { self.call_cache.clone() } + pub async fn block_number( + &self, + hash: &BlockHash, + ) -> Result, Option)>, StoreError> { + self.chain_store.block_number(hash).await + } + // TODO: This is only used to build the block stream which could prolly // be moved to the chain itself and return a block stream future that the // caller can spawn. - pub fn cheapest_adapter(&self) -> Arc { + pub async fn cheapest_adapter(&self) -> Arc { let adapters = match self.client.as_ref() { ChainClient::Firehose(_) => panic!("no adapter with firehose"), ChainClient::Rpc(adapter) => adapter, }; - adapters.cheapest().unwrap() + adapters.cheapest().await.unwrap() } } @@ -354,6 +497,8 @@ impl Blockchain for Chain { type NodeCapabilities = crate::capabilities::NodeCapabilities; + type DecoderHook = crate::data_source::DecoderHook; + fn triggers_adapter( &self, loc: &DeploymentLocator, @@ -369,10 +514,27 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + source_subgraph_stores: Vec>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let current_ptr = store.block_ptr(); + + if !filter.subgraph_filter.is_empty() { + return self + .block_stream_builder + .build_subgraph_block_stream( + self, + deployment, + start_blocks, + source_subgraph_stores, + current_ptr, + filter, + unified_api_version, + ) + .await; + } + match self.chain_client().as_ref() { ChainClient::Rpc(_) => { self.block_stream_builder @@ -380,6 +542,7 @@ impl Blockchain for Chain { self, deployment, start_blocks, + source_subgraph_stores, current_ptr, filter, unified_api_version, @@ -394,7 +557,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, current_ptr, - filter, + filter.chain_filter.clone(), unified_api_version, ) .await @@ -402,8 +565,8 @@ impl Blockchain for Chain { } } - fn chain_store(&self) -> Arc { - self.chain_store.clone() + async fn chain_head_ptr(&self) -> Result, Error> { + self.chain_store.cheap_clone().chain_head_ptr().await } async fn block_pointer_from_number( @@ -413,20 +576,34 @@ impl Blockchain for Chain { ) -> Result { match self.client.as_ref() { ChainClient::Firehose(endpoints) => endpoints - .endpoint()? + .endpoint() + .await? .block_ptr_for_number::(logger, number) .await .map_err(IngestorError::Unknown), ChainClient::Rpc(adapters) => { + let cached = self + .chain_store + .cheap_clone() + .block_ptrs_by_numbers(vec![number]) + .await + .unwrap_or_default(); + if let Some(ptrs) = cached.get(&number) + && ptrs.len() == 1 + { + return Ok(BlockPtr::new(ptrs[0].hash.clone(), ptrs[0].number)); + } + let adapter = adapters .cheapest() + .await .with_context(|| format!("no adapter for chain {}", self.name))? .clone(); adapter - .block_pointer_from_number(logger, number) - .compat() + .next_existing_ptr_to_number(logger, number) .await + .map_err(From::from) } } } @@ -443,19 +620,37 @@ impl Blockchain for Chain { self.block_refetcher.get_block(self, logger, cursor).await } - fn runtime_adapter(&self) -> Arc> { - self.runtime_adapter.clone() + async fn runtime( + &self, + ) -> anyhow::Result<(Arc>, Self::DecoderHook)> { + let call_cache = Arc::new(BufferedCallCache::new(self.call_cache.cheap_clone())); + let chain_ident = self.chain_store.chain_identifier().await?; + + let builder = self.runtime_adapter_builder.build( + self.eth_adapters.cheap_clone(), + call_cache.cheap_clone(), + Arc::new(chain_ident.clone()), + ); + let eth_call_gas = eth_call_gas(&chain_ident); + + let decoder_hook = crate::data_source::DecoderHook::new( + self.eth_adapters.cheap_clone(), + call_cache, + eth_call_gas, + ); + + Ok((builder, decoder_hook)) } fn chain_client(&self) -> Arc> { self.client.clone() } - fn block_ingestor(&self) -> anyhow::Result> { + async fn block_ingestor(&self) -> anyhow::Result> { let ingestor: Box = match self.chain_client().as_ref() { ChainClient::Firehose(_) => { let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), + self.chain_store.cheap_clone().as_head_store(), self.chain_client(), self.logger_factory .component_logger("EthereumFirehoseBlockIngestor", None), @@ -465,10 +660,7 @@ impl Blockchain for Chain { Box::new(ingestor) } - ChainClient::Rpc(rpc) => { - let eth_adapter = rpc - .cheapest() - .ok_or_else(|| anyhow!("unable to get adapter for ethereum block ingestor"))?; + ChainClient::Rpc(_) => { let logger = self .logger_factory .component_logger( @@ -479,7 +671,7 @@ impl Blockchain for Chain { }), }), ) - .new(o!("provider" => eth_adapter.provider().to_string())); + .new(o!()); if !self.is_ingestible { bail!( @@ -493,10 +685,10 @@ impl Blockchain for Chain { // present in the DB. Box::new(PollingBlockIngestor::new( logger, - graph::env::ENV_VARS.reorg_threshold, - eth_adapter, - self.chain_store().cheap_clone(), - self.polling_ingestor_interval, + graph::env::ENV_VARS.reorg_threshold(), + self.chain_client(), + self.chain_store.cheap_clone(), + self.settings.polling_interval, self.name.clone(), )?) } @@ -516,6 +708,8 @@ pub enum BlockFinality { // If a block may still be reorged, we need to work with more local data. NonFinal(EthereumBlockWithCalls), + + Ptr(Arc), } impl Default for BlockFinality { @@ -529,15 +723,7 @@ impl BlockFinality { match self { BlockFinality::Final(block) => block, BlockFinality::NonFinal(block) => &block.ethereum_block.block, - } - } -} - -impl<'a> From<&'a BlockFinality> for BlockPtr { - fn from(block: &'a BlockFinality) -> BlockPtr { - match block { - BlockFinality::Final(b) => BlockPtr::from(&**b), - BlockFinality::NonFinal(b) => BlockPtr::from(&b.ethereum_block), + BlockFinality::Ptr(_) => unreachable!("light_block called on HeaderOnly"), } } } @@ -547,6 +733,7 @@ impl Block for BlockFinality { match self { BlockFinality::Final(block) => block.block_ptr(), BlockFinality::NonFinal(block) => block.ethereum_block.block.block_ptr(), + BlockFinality::Ptr(block) => BlockPtr::new(block.hash.clone(), block.number), } } @@ -554,6 +741,9 @@ impl Block for BlockFinality { match self { BlockFinality::Final(block) => block.parent_ptr(), BlockFinality::NonFinal(block) => block.ethereum_block.block.parent_ptr(), + BlockFinality::Ptr(block) => { + Some(BlockPtr::new(block.parent_hash.clone(), block.number - 1)) + } } } @@ -586,16 +776,22 @@ impl Block for BlockFinality { json::to_value(eth_block) } BlockFinality::NonFinal(block) => json::to_value(&block.ethereum_block), + BlockFinality::Ptr(_) => Ok(json::Value::Null), } } fn timestamp(&self) -> BlockTime { - let ts = match self { - BlockFinality::Final(block) => block.timestamp, - BlockFinality::NonFinal(block) => block.ethereum_block.block.timestamp, - }; - let ts = i64::try_from(ts.as_u64()).unwrap(); - BlockTime::since_epoch(ts, 0) + match self { + BlockFinality::Final(block) => { + let ts = i64::try_from(block.timestamp_u64()).unwrap(); + BlockTime::since_epoch(ts, 0) + } + BlockFinality::NonFinal(block) => { + let ts = i64::try_from(block.ethereum_block.block.timestamp_u64()).unwrap(); + BlockTime::since_epoch(ts, 0) + } + BlockFinality::Ptr(block) => block.timestamp, + } } } @@ -608,6 +804,104 @@ pub struct TriggersAdapter { chain_client: Arc>, capabilities: NodeCapabilities, unified_api_version: UnifiedMappingApiVersion, + eth_adapters: Arc, +} + +/// Fetches blocks from the cache based on block numbers, excluding duplicates +/// (i.e., multiple blocks for the same number), and identifying missing blocks that +/// need to be fetched via RPC/Firehose. Returns a tuple of the found blocks and the missing block numbers. +async fn fetch_unique_blocks_from_cache( + logger: &Logger, + chain_store: Arc, + block_numbers: BTreeSet, +) -> (Vec>, Vec) { + // Load blocks from the cache + let blocks_map = chain_store + .cheap_clone() + .block_ptrs_by_numbers(block_numbers.iter().copied().collect::>()) + .await + .map_err(|e| { + error!(logger, "Error accessing block cache {}", e); + e + }) + .unwrap_or_default(); + + // Collect blocks and filter out ones with multiple entries + let blocks: Vec> = blocks_map + .into_values() + .filter_map(|values| { + if values.len() == 1 { + Some(Arc::new(values[0].clone())) + } else { + None + } + }) + .collect(); + + // Identify missing blocks + let missing_blocks: Vec = block_numbers + .into_iter() + .filter(|&number| !blocks.iter().any(|block| block.block_number() == number)) + .collect(); + + if !missing_blocks.is_empty() { + debug!( + logger, + "Loading {} block(s) not in the block cache", + missing_blocks.len() + ); + trace!(logger, "Missing blocks {:?}", missing_blocks.len()); + } + + (blocks, missing_blocks) +} + +// This is used to load blocks from the RPC. +async fn load_blocks_with_rpc( + logger: &Logger, + adapter: Arc, + chain_store: Arc, + block_numbers: BTreeSet, +) -> Result> { + let logger_clone = logger.clone(); + load_blocks( + logger, + chain_store, + block_numbers, + |missing_numbers| async move { + adapter + .load_block_ptrs_by_numbers_rpc(logger_clone, missing_numbers) + .try_collect() + .await + }, + ) + .await +} + +/// Fetches blocks by their numbers, first attempting to load from cache. +/// Missing blocks are retrieved from an external source, with all blocks sorted and converted to `BlockFinality` format. +async fn load_blocks( + logger: &Logger, + chain_store: Arc, + block_numbers: BTreeSet, + fetch_missing: F, +) -> Result> +where + F: FnOnce(Vec) -> Fut, + Fut: Future>>>, +{ + // Fetch cached blocks and identify missing ones + let (mut cached_blocks, missing_block_numbers) = + fetch_unique_blocks_from_cache(logger, chain_store, block_numbers).await; + + // Fetch missing blocks if any + if !missing_block_numbers.is_empty() { + let missing_blocks = fetch_missing(missing_block_numbers).await?; + cached_blocks.extend(missing_blocks); + cached_blocks.sort_by_key(|block| block.number); + } + + Ok(cached_blocks.into_iter().map(BlockFinality::Ptr).collect()) } #[async_trait] @@ -617,9 +911,12 @@ impl TriggersAdapterTrait for TriggersAdapter { from: BlockNumber, to: BlockNumber, filter: &TriggerFilter, - ) -> Result>, Error> { + ) -> Result<(Vec>, BlockNumber), Error> { blocks_with_triggers( - self.chain_client.rpc()?.cheapest_with(&self.capabilities)?, + self.chain_client + .rpc()? + .cheapest_with(&self.capabilities) + .await?, self.logger.clone(), self.chain_store.clone(), self.ethrpc_metrics.clone(), @@ -631,6 +928,100 @@ impl TriggersAdapterTrait for TriggersAdapter { .await } + async fn load_block_ptrs_by_numbers( + &self, + logger: Logger, + block_numbers: BTreeSet, + ) -> Result> { + match &*self.chain_client { + ChainClient::Firehose(endpoints) => { + // If the force_rpc_for_block_ptrs flag is set, we will use the RPC to load the blocks + // even if the firehose is available. If no adapter is available, we will log an error. + // And then fallback to the firehose. + if ENV_VARS.force_rpc_for_block_ptrs { + trace!( + logger, + "Loading blocks from RPC (force_rpc_for_block_ptrs is set)"; + "block_numbers" => format!("{:?}", block_numbers) + ); + match self.eth_adapters.cheapest_with(&self.capabilities).await { + Ok(adapter) => { + match load_blocks_with_rpc( + &logger, + adapter, + self.chain_store.clone(), + block_numbers.clone(), + ) + .await + { + Ok(blocks) => return Ok(blocks), + Err(e) => { + warn!(logger, "Error loading blocks from RPC: {}", e); + } + } + } + Err(e) => { + warn!(logger, "Error getting cheapest adapter: {}", e); + } + } + } + + trace!( + logger, + "Loading blocks from firehose"; + "block_numbers" => format!("{:?}", block_numbers) + ); + + let endpoint = endpoints.endpoint().await?; + let chain_store = self.chain_store.clone(); + let logger_clone = logger.clone(); + + load_blocks( + &logger, + chain_store, + block_numbers, + |missing_numbers| async move { + let blocks = endpoint + .load_blocks_by_numbers::( + missing_numbers.iter().map(|&n| n as u64).collect(), + &logger_clone, + ) + .await? + .into_iter() + .map(|block| { + Arc::new(ExtendedBlockPtr { + hash: block.hash(), + number: block.number(), + parent_hash: block.parent_hash().unwrap_or_default(), + timestamp: block.timestamp(), + }) + }) + .collect::>(); + Ok(blocks) + }, + ) + .await + } + + ChainClient::Rpc(eth_adapters) => { + trace!( + logger, + "Loading blocks from RPC"; + "block_numbers" => format!("{:?}", block_numbers) + ); + + let adapter = eth_adapters.cheapest_with(&self.capabilities).await?; + load_blocks_with_rpc(&logger, adapter, self.chain_store.clone(), block_numbers) + .await + } + } + } + + async fn chain_head_ptr(&self) -> Result, Error> { + let chain_store = self.chain_store.clone(); + chain_store.chain_head_ptr().await + } + async fn triggers_in_block( &self, logger: &Logger, @@ -649,9 +1040,13 @@ impl TriggersAdapterTrait for TriggersAdapter { match &block { BlockFinality::Final(_) => { - let adapter = self.chain_client.rpc()?.cheapest_with(&self.capabilities)?; + let adapter = self + .chain_client + .rpc()? + .cheapest_with(&self.capabilities) + .await?; let block_number = block.number() as BlockNumber; - let blocks = blocks_with_triggers( + let (blocks, _) = blocks_with_triggers( adapter, logger.clone(), self.chain_store.clone(), @@ -675,69 +1070,232 @@ impl TriggersAdapterTrait for TriggersAdapter { triggers.append(&mut parse_block_triggers(&filter.block, full_block)); Ok(BlockWithTriggers::new(block, triggers, logger)) } + BlockFinality::Ptr(_) => unreachable!("triggers_in_block called on HeaderOnly"), } } async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { - self.chain_client - .rpc()? - .cheapest() - .ok_or(anyhow!("unable to get adapter for is_on_main_chain"))? - .is_on_main_chain(&self.logger, ptr.clone()) - .await + // It is tempting to use the block cache here; but that can go wrong + // when graph-node gets shut down and some of its nonfinal blocks + // then are reorged; when graph-node gets started again when those + // block numbers have become final, it might consider a reorged + // block as canonical; allowing the use of the block cache here + // would require us to also track the finality of blocks. + match &*self.chain_client { + ChainClient::Firehose(endpoints) => { + let endpoint = endpoints.endpoint().await?; + let block = endpoint + .get_block_by_number_with_retry::(ptr.number as u64, &self.logger) + .await + .context(format!( + "Failed to fetch block {} from firehose", + ptr.number + ))?; + Ok(block.hash() == ptr.hash) + } + ChainClient::Rpc(adapter) => { + let adapter = adapter + .cheapest() + .await + .ok_or_else(|| anyhow!("unable to get adapter for is_on_main_chain"))?; + + adapter.is_on_main_chain(&self.logger, ptr).await + } + } } + // Find an ancestor block at the specified offset from the given block pointer. + // Primarily used for reorg detection to verify if the indexed position remains + // on the main chain. + // + // Parameters: + // - ptr: Starting block pointer from which to walk backwards (typically the chain head) + // - offset: Number of blocks to traverse backwards (0 returns ptr, 1 returns parent, etc.) + // - root: Optional block hash that serves as a boundary for traversal. This is ESSENTIAL + // for chains with skipped blocks (e.g., Filecoin EVM) where block numbers are not + // consecutive. When provided, traversal stops upon reaching the child of root, + // ensuring correct ancestor relationships even with gaps in block numbers. + // + // The function attempts to use the database cache first for performance, + // with RPC fallback implemented to handle cases where the cache is unavailable. async fn ancestor_block( &self, ptr: BlockPtr, offset: BlockNumber, + root: Option, ) -> Result, Error> { - let block: Option = self + let ptr_for_log = ptr.clone(); + let cached = self .chain_store .cheap_clone() - .ancestor_block(ptr, offset) - .await? - .map(json::from_value) - .transpose()?; - Ok(block.map(|block| { - BlockFinality::NonFinal(EthereumBlockWithCalls { - ethereum_block: block, - calls: None, - }) - })) + .ancestor_block(ptr.clone(), offset, root.clone()) + .await?; + + // Use full blocks (with receipts) directly from cache. + // Light blocks (no receipts) need to be fetched from Firehose/RPC. + let block_ptr = match cached { + Some((cached_block, ptr)) => match cached_block.into_full_block() { + Some(block) => { + return Ok(Some(BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block: block, + calls: None, + }))); + } + None => { + trace!( + self.logger, + "Cached block #{} {} is light (no receipts). Falling back to Firehose/RPC.", + ptr.number, + ptr.hash_hex(), + ); + ptr + } + }, + None => { + // Cache miss - fall back to walking the chain via parent_ptr() calls. + // This provides resilience when the block cache is empty (e.g., after truncation). + debug!( + self.logger, + "ancestor_block cache miss for {} at offset {}, walking back via parent_ptr", + ptr_for_log.hash_hex(), + offset + ); + + match walk_back_ancestor( + ptr.clone(), + offset, + root.clone(), + |block_ptr| async move { self.parent_ptr(&block_ptr).await }, + ) + .await? + { + Some(ptr) => ptr, + None => return Ok(None), + } + } + }; + + // Fetch the actual block data for the identified block pointer. + // This path is taken for both cache misses and deserialization failures. + match self.chain_client.as_ref() { + ChainClient::Firehose(endpoints) => { + let block = self + .fetch_block_with_firehose(endpoints, &block_ptr) + .await?; + let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; + Ok(Some(BlockFinality::NonFinal(ethereum_block))) + } + ChainClient::Rpc(adapters) => { + match self.fetch_full_block_with_rpc(adapters, &block_ptr).await? { + Some(ethereum_block) => { + Ok(Some(BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block, + calls: None, + }))) + } + None => Ok(None), + } + } + } } async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - use futures::stream::Stream; use graph::prelude::LightEthereumBlockExt; let block = match self.chain_client.as_ref() { - ChainClient::Firehose(_) => Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - }), - ChainClient::Rpc(adapters) => { - let blocks = adapters - .cheapest_with(&self.capabilities)? - .load_blocks( - self.logger.cheap_clone(), - self.chain_store.cheap_clone(), - HashSet::from_iter(Some(block.hash_as_h256())), - ) - .await - .collect() - .compat() - .await?; - assert_eq!(blocks.len(), 1); + ChainClient::Firehose(endpoints) => { + let chain_store = self.chain_store.cheap_clone(); + // First try to get the parent pointer from the store header columns + if let Ok(Some(parent)) = chain_store.block_parent_ptr(&block.hash).await { + return Ok(Some(parent)); + } - blocks[0].parent_ptr() + // If not in store, fetch from Firehose + self.fetch_block_with_firehose(endpoints, block) + .await? + .parent_ptr() } + ChainClient::Rpc(adapters) => self + .fetch_light_block_with_rpc(adapters, block) + .await? + .expect("block must exist for parent_ptr") + .parent_ptr(), }; Ok(block) } } +impl TriggersAdapter { + async fn fetch_block_with_firehose( + &self, + endpoints: &FirehoseEndpoints, + block_ptr: &BlockPtr, + ) -> Result { + let endpoint = endpoints.endpoint().await?; + let logger = self.logger.clone(); + let retry_log_message = format!("fetch_block_with_firehose {}", block_ptr); + let block_ptr = block_ptr.clone(); + + let block = retry(retry_log_message, &logger) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let endpoint = endpoint.cheap_clone(); + let logger = logger.cheap_clone(); + let block_ptr = block_ptr.clone(); + async move { + endpoint + .get_block_by_ptr::(&block_ptr, &logger) + .await + .context(format!("Failed to fetch block {} from firehose", block_ptr)) + } + }) + .await?; + + Ok(block) + } + + async fn fetch_light_block_with_rpc( + &self, + adapters: &EthereumNetworkAdapters, + block_ptr: &BlockPtr, + ) -> Result>, Error> { + let blocks = adapters + .cheapest_with(&self.capabilities) + .await? + .load_blocks( + self.logger.cheap_clone(), + self.chain_store.cheap_clone(), + HashSet::from_iter(Some(block_ptr.hash.as_b256())), + ) + .await?; + + Ok(blocks.into_iter().next()) + } + + async fn fetch_full_block_with_rpc( + &self, + adapters: &EthereumNetworkAdapters, + block_ptr: &BlockPtr, + ) -> Result, Error> { + // Use the cache-aware light block fetch first; it checks recent_blocks_cache + // and the DB before falling back to eth_getBlockByHash. + let light_block = self.fetch_light_block_with_rpc(adapters, block_ptr).await?; + match light_block { + Some(light_block) => { + let adapter = adapters.cheapest_with(&self.capabilities).await?; + let ethereum_block = adapter + .load_full_block(&self.logger, light_block.inner().clone()) + .await + .map_err(|e| anyhow!("Failed to load full block: {}", e))?; + Ok(Some(ethereum_block)) + } + None => Ok(None), + } + } +} + pub struct FirehoseMapper { adapter: Arc>, filter: Arc, @@ -745,10 +1303,15 @@ pub struct FirehoseMapper { #[async_trait] impl BlockStreamMapper for FirehoseMapper { - fn decode_block(&self, output: Option<&[u8]>) -> Result, Error> { + fn decode_block( + &self, + output: Option<&[u8]>, + ) -> Result, BlockStreamError> { let block = match output { Some(block) => codec::Block::decode(block)?, - None => anyhow::bail!("ethereum mapper is expected to always have a block"), + None => Err(anyhow::anyhow!( + "ethereum mapper is expected to always have a block" + ))?, }; // See comment(437a9f17-67cc-478f-80a3-804fe554b227) ethereum_block.calls is always Some even if calls @@ -762,20 +1325,11 @@ impl BlockStreamMapper for FirehoseMapper { &self, logger: &Logger, block: BlockFinality, - ) -> Result, Error> { + ) -> Result, BlockStreamError> { self.adapter .triggers_in_block(logger, block, &self.filter) .await - } - - async fn handle_substreams_block( - &self, - _logger: &Logger, - _clock: Clock, - _cursor: FirehoseCursor, - _block: Vec, - ) -> Result, Error> { - unimplemented!() + .map_err(BlockStreamError::from) } } @@ -790,7 +1344,7 @@ impl FirehoseMapperTrait for FirehoseMapper { logger: &Logger, response: &firehose::Response, ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + let step = ForkStep::try_from(response.step).unwrap_or_else(|_| { panic!( "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", response.step @@ -835,7 +1389,9 @@ impl FirehoseMapperTrait for FirehoseMapper { } StepFinal => { - unreachable!("irreversible step is not handled and should not be requested in the Firehose request") + unreachable!( + "irreversible step is not handled and should not be requested in the Firehose request" + ) } StepUnset => { @@ -873,3 +1429,251 @@ impl FirehoseMapperTrait for FirehoseMapper { .await } } + +#[cfg(test)] +mod tests { + use graph::blockchain::mock::MockChainStore; + use graph::slog; + + use super::*; + use std::sync::Arc; + + // Helper function to create test blocks + fn create_test_block(number: BlockNumber, hash: &str) -> ExtendedBlockPtr { + let hash = BlockHash(hash.as_bytes().to_vec().into_boxed_slice()); + let ptr = BlockPtr::new(hash.clone(), number); + ExtendedBlockPtr { + hash, + number, + parent_hash: BlockHash(vec![0; 32].into_boxed_slice()), + timestamp: BlockTime::for_test(&ptr), + } + } + + #[graph::test] + async fn test_fetch_unique_blocks_single_block() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add a single block + let block = create_test_block(1, "block1"); + chain_store.blocks.insert(1, vec![block.clone()]); + + let block_numbers: BTreeSet<_> = vec![1].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].number, 1); + assert!(missing.is_empty()); + } + + #[graph::test] + async fn test_fetch_unique_blocks_duplicate_blocks() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add multiple blocks for the same number + let block1 = create_test_block(1, "block1a"); + let block2 = create_test_block(1, "block1b"); + chain_store + .blocks + .insert(1, vec![block1.clone(), block2.clone()]); + + let block_numbers: BTreeSet<_> = vec![1].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + // Should filter out the duplicate block + assert!(blocks.is_empty()); + assert_eq!(missing, vec![1]); + assert_eq!(missing[0], 1); + } + + #[graph::test] + async fn test_fetch_unique_blocks_missing_blocks() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add block number 1 but not 2 + let block = create_test_block(1, "block1"); + chain_store.blocks.insert(1, vec![block.clone()]); + + let block_numbers: BTreeSet<_> = vec![1, 2].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].number, 1); + assert_eq!(missing, vec![2]); + } + + #[graph::test] + async fn test_fetch_unique_blocks_multiple_valid_blocks() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add multiple valid blocks + let block1 = create_test_block(1, "block1"); + let block2 = create_test_block(2, "block2"); + chain_store.blocks.insert(1, vec![block1.clone()]); + chain_store.blocks.insert(2, vec![block2.clone()]); + + let block_numbers: BTreeSet<_> = vec![1, 2].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 2); + assert!(blocks.iter().any(|b| b.number == 1)); + assert!(blocks.iter().any(|b| b.number == 2)); + assert!(missing.is_empty()); + } + + #[graph::test] + async fn test_fetch_unique_blocks_mixed_scenario() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add a mix of scenarios: + // - Block 1: Single valid block + // - Block 2: Multiple blocks (duplicate) + // - Block 3: Missing + let block1 = create_test_block(1, "block1"); + let block2a = create_test_block(2, "block2a"); + let block2b = create_test_block(2, "block2b"); + + chain_store.blocks.insert(1, vec![block1.clone()]); + chain_store + .blocks + .insert(2, vec![block2a.clone(), block2b.clone()]); + + let block_numbers: BTreeSet<_> = vec![1, 2, 3].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].number, 1); + assert_eq!(missing.len(), 2); + assert!(missing.contains(&2)); + assert!(missing.contains(&3)); + } + + #[tokio::test] + async fn test_walk_back_ancestor() { + use std::collections::HashMap; + + let block_100_hash = BlockHash("block100".as_bytes().to_vec().into_boxed_slice()); + let block_101_hash = BlockHash("block101".as_bytes().to_vec().into_boxed_slice()); + let block_102_hash = BlockHash("block102".as_bytes().to_vec().into_boxed_slice()); + let block_103_hash = BlockHash("block103".as_bytes().to_vec().into_boxed_slice()); + let block_104_hash = BlockHash("block104".as_bytes().to_vec().into_boxed_slice()); + let block_105_hash = BlockHash("block105".as_bytes().to_vec().into_boxed_slice()); + + let block_105 = BlockPtr::new(block_105_hash.clone(), 105); + let block_104 = BlockPtr::new(block_104_hash.clone(), 104); + let block_103 = BlockPtr::new(block_103_hash.clone(), 103); + let block_102 = BlockPtr::new(block_102_hash.clone(), 102); + let block_101 = BlockPtr::new(block_101_hash.clone(), 101); + let block_100 = BlockPtr::new(block_100_hash.clone(), 100); + + let mut parent_map = HashMap::new(); + parent_map.insert(block_105_hash.clone(), block_104.clone()); + parent_map.insert(block_104_hash.clone(), block_103.clone()); + parent_map.insert(block_103_hash.clone(), block_102.clone()); + parent_map.insert(block_102_hash.clone(), block_101.clone()); + parent_map.insert(block_101_hash.clone(), block_100.clone()); + + let result = super::walk_back_ancestor(block_105.clone(), 2, None, |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }) + .await + .unwrap(); + assert_eq!(result, Some(block_103.clone())); + + let result = super::walk_back_ancestor( + block_105.clone(), + 10, + Some(block_102_hash.clone()), + |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }, + ) + .await + .unwrap(); + assert_eq!( + result, + Some(block_103.clone()), + "Should stop at child of root" + ); + } + + #[tokio::test] + async fn test_walk_back_ancestor_skipped_blocks_with_root() { + use std::collections::HashMap; + + let block_100_hash = BlockHash("block100".as_bytes().to_vec().into_boxed_slice()); + let block_101_hash = BlockHash("block101".as_bytes().to_vec().into_boxed_slice()); + let block_102_hash = BlockHash("block102".as_bytes().to_vec().into_boxed_slice()); + let block_110_hash = BlockHash("block110".as_bytes().to_vec().into_boxed_slice()); + let block_111_hash = BlockHash("block111".as_bytes().to_vec().into_boxed_slice()); + let block_112_hash = BlockHash("block112".as_bytes().to_vec().into_boxed_slice()); + let block_120_hash = BlockHash("block120".as_bytes().to_vec().into_boxed_slice()); + + let block_120 = BlockPtr::new(block_120_hash.clone(), 120); + let block_112 = BlockPtr::new(block_112_hash.clone(), 112); + let block_111 = BlockPtr::new(block_111_hash.clone(), 111); + let block_110 = BlockPtr::new(block_110_hash.clone(), 110); + let block_102 = BlockPtr::new(block_102_hash.clone(), 102); + let block_101 = BlockPtr::new(block_101_hash.clone(), 101); + let block_100 = BlockPtr::new(block_100_hash.clone(), 100); + + let mut parent_map = HashMap::new(); + parent_map.insert(block_120_hash.clone(), block_112.clone()); + parent_map.insert(block_112_hash.clone(), block_111.clone()); + parent_map.insert(block_111_hash.clone(), block_110.clone()); + parent_map.insert(block_110_hash.clone(), block_102.clone()); + parent_map.insert(block_102_hash.clone(), block_101.clone()); + parent_map.insert(block_101_hash.clone(), block_100.clone()); + + let result = super::walk_back_ancestor( + block_120.clone(), + 10, + Some(block_110_hash.clone()), + |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }, + ) + .await + .unwrap(); + assert_eq!( + result, + Some(block_111.clone()), + "root=110: should stop at 111 (child of root)" + ); + + let result = super::walk_back_ancestor( + block_120.clone(), + 10, + Some(block_101_hash.clone()), + |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }, + ) + .await + .unwrap(); + assert_eq!( + result, + Some(block_102.clone()), + "root=101: should stop at 102 (child of root, across skip)" + ); + } +} diff --git a/chain/ethereum/src/codec.rs b/chain/ethereum/src/codec.rs index 347dd615406..b40bcd11910 100644 --- a/chain/ethereum/src/codec.rs +++ b/chain/ethereum/src/codec.rs @@ -1,4 +1,5 @@ #[rustfmt::skip] +#[allow(clippy::doc_lazy_continuation, clippy::doc_overindented_list_items)] #[path = "protobuf/sf.ethereum.r#type.v2.rs"] mod pbcodec; @@ -7,11 +8,19 @@ use graph::{ blockchain::{ self, Block as BlockchainBlock, BlockPtr, BlockTime, ChainStoreBlock, ChainStoreData, }, + components::ethereum::{ + AnyBlock, AnyHeader, AnyRpcHeader, AnyTransactionReceiptBare, AnyTxEnvelope, + }, prelude::{ - web3, - web3::types::{Bytes, H160, H2048, H256, H64, U256, U64}, BlockNumber, Error, EthereumBlock, EthereumBlockWithCalls, EthereumCall, LightEthereumBlock, + alloy::{ + self, + consensus::{ReceiptWithBloom, TxEnvelope, TxType}, + network::AnyReceiptEnvelope, + primitives::{Address, B256, Bloom, Bytes, LogData, U256, aliases::B2048}, + rpc::types::{self as alloy_rpc_types, AccessList, AccessListItem, Transaction}, + }, }, }; use std::sync::Arc; @@ -34,13 +43,13 @@ where } } -impl TryDecodeProto<[u8; 256], H2048> for &[u8] {} -impl TryDecodeProto<[u8; 32], H256> for &[u8] {} -impl TryDecodeProto<[u8; 20], H160> for &[u8] {} +impl TryDecodeProto<[u8; 32], B256> for &[u8] {} +impl TryDecodeProto<[u8; 256], B2048> for &[u8] {} +impl TryDecodeProto<[u8; 20], Address> for &[u8] {} -impl From<&BigInt> for web3::types::U256 { +impl From<&BigInt> for U256 { fn from(val: &BigInt) -> Self { - web3::types::U256::from_big_endian(&val.bytes) + U256::from_be_slice(&val.bytes) } } @@ -68,9 +77,9 @@ impl<'a> TryInto for CallAt<'a> { .value .as_ref() .map_or_else(|| U256::from(0), |v| v.into()), - gas_used: U256::from(self.call.gas_consumed), - input: Bytes(self.call.input.clone()), - output: Bytes(self.call.return_data.clone()), + gas_used: self.call.gas_consumed, + input: Bytes::from(self.call.input.clone()), + output: Bytes::from(self.call.return_data.clone()), block_hash: self.block.hash.try_decode_proto("call block hash")?, block_number: self.block.number as i32, transaction_hash: Some(self.trace.hash.try_decode_proto("call transaction hash")?), @@ -79,41 +88,6 @@ impl<'a> TryInto for CallAt<'a> { } } -impl TryInto for Call { - type Error = Error; - - fn try_into(self) -> Result { - Ok(web3::types::Call { - from: self.caller.try_decode_proto("call from address")?, - to: self.address.try_decode_proto("call to address")?, - value: self - .value - .as_ref() - .map_or_else(|| U256::from(0), |v| v.into()), - gas: U256::from(self.gas_limit), - input: Bytes::from(self.input.clone()), - call_type: CallType::from_i32(self.call_type) - .ok_or_else(|| format_err!("invalid call type: {}", self.call_type,))? - .into(), - }) - } -} - -impl From for web3::types::CallType { - fn from(val: CallType) -> Self { - match val { - CallType::Unspecified => web3::types::CallType::None, - CallType::Call => web3::types::CallType::Call, - CallType::Callcode => web3::types::CallType::CallCode, - CallType::Delegate => web3::types::CallType::DelegateCall, - CallType::Static => web3::types::CallType::StaticCall, - - // FIXME (SF): Really not sure what this should map to, we are using None for now, need to revisit - CallType::Create => web3::types::CallType::None, - } - } -} - pub struct LogAt<'a> { log: &'a Log, block: &'a Block, @@ -126,51 +100,38 @@ impl<'a> LogAt<'a> { } } -impl<'a> TryInto for LogAt<'a> { +impl<'a> TryInto for LogAt<'a> { type Error = Error; - fn try_into(self) -> Result { - Ok(web3::types::Log { - address: self.log.address.try_decode_proto("log address")?, - topics: self - .log - .topics - .iter() - .map(|t| t.try_decode_proto("topic")) - .collect::, Error>>()?, - data: Bytes::from(self.log.data.clone()), + fn try_into(self) -> Result { + let topics = self + .log + .topics + .iter() + .map(|t| t.try_decode_proto("topic")) + .collect::, Error>>()?; + + Ok(alloy::rpc::types::Log { + inner: alloy::primitives::Log { + address: self.log.address.try_decode_proto("log address")?, + data: LogData::new(topics, self.log.data.clone().into()) + .ok_or_else(|| format_err!("invalid log data"))?, + }, block_hash: Some(self.block.hash.try_decode_proto("log block hash")?), - block_number: Some(U64::from(self.block.number)), + block_number: Some(self.block.number), transaction_hash: Some(self.trace.hash.try_decode_proto("log transaction hash")?), - transaction_index: Some(U64::from(self.trace.index as u64)), - log_index: Some(U256::from(self.log.block_index)), - transaction_log_index: Some(U256::from(self.log.index)), - log_type: None, - removed: None, + transaction_index: Some(self.trace.index as u64), + log_index: Some(self.log.block_index as u64), + removed: false, + block_timestamp: self + .block + .header + .as_ref() + .and_then(|h| h.timestamp.as_ref().map(|t| t.seconds as u64)), }) } } -impl From for web3::types::U64 { - fn from(val: TransactionTraceStatus) -> Self { - let status: Option = val.into(); - status.unwrap_or_else(|| web3::types::U64::from(0)) - } -} - -impl Into> for TransactionTraceStatus { - fn into(self) -> Option { - match self { - Self::Unknown => { - panic!("Got a transaction trace with status UNKNOWN, datasource is broken") - } - Self::Succeeded => Some(web3::types::U64::from(1)), - Self::Failed => Some(web3::types::U64::from(0)), - Self::Reverted => Some(web3::types::U64::from(0)), - } - } -} - pub struct TransactionTraceAt<'a> { trace: &'a TransactionTrace, block: &'a Block, @@ -182,46 +143,323 @@ impl<'a> TransactionTraceAt<'a> { } } -impl<'a> TryInto for TransactionTraceAt<'a> { +impl<'a> TryInto> for TransactionTraceAt<'a> { type Error = Error; - fn try_into(self) -> Result { - Ok(web3::types::Transaction { - hash: self.trace.hash.try_decode_proto("transaction hash")?, - nonce: U256::from(self.trace.nonce), - block_hash: Some(self.block.hash.try_decode_proto("transaction block hash")?), - block_number: Some(U64::from(self.block.number)), - transaction_index: Some(U64::from(self.trace.index as u64)), - from: Some( - self.trace - .from - .try_decode_proto("transaction from address")?, - ), - to: match self.trace.calls.len() { - 0 => Some(self.trace.to.try_decode_proto("transaction to address")?), - _ => { - match CallType::from_i32(self.trace.calls[0].call_type).ok_or_else(|| { - format_err!("invalid call type: {}", self.trace.calls[0].call_type,) - })? { - CallType::Create => { - None // we don't want the 'to' address on a transaction that creates the contract, to align with RPC behavior - } - _ => Some(self.trace.to.try_decode_proto("transaction to")?), - } - } + fn try_into(self) -> Result, Self::Error> { + use alloy::{ + consensus::transaction::Recovered, + consensus::{ + Signed, TxEip1559, TxEip2930, TxEip4844, TxEip4844Variant, TxEip7702, TxLegacy, }, - value: self.trace.value.as_ref().map_or(U256::zero(), |x| x.into()), - gas_price: self.trace.gas_price.as_ref().map(|x| x.into()), - gas: U256::from(self.trace.gas_limit), - input: Bytes::from(self.trace.input.clone()), - v: None, - r: None, - s: None, - raw: None, - access_list: None, - max_fee_per_gas: None, - max_priority_fee_per_gas: None, - transaction_type: None, + network::{AnyTxEnvelope, AnyTxType, UnknownTxEnvelope, UnknownTypedTransaction}, + primitives::{Bytes, TxKind, U256}, + rpc::types::Transaction as AlloyTransaction, + serde::OtherFields, + }; + use std::collections::BTreeMap; + + // Extract data from trace and block + let block_hash = self.block.hash.try_decode_proto("transaction block hash")?; + let block_number = self.block.number; + let block_timestamp = self + .block + .header + .as_ref() + .and_then(|h| h.timestamp.as_ref().map(|t| t.seconds as u64)); + let transaction_index = Some(self.trace.index as u64); + let from_address = self + .trace + .from + .try_decode_proto("transaction from address")?; + let to = get_to_address(self.trace)?; + let value = self.trace.value.as_ref().map_or(U256::ZERO, |x| x.into()); + let gas_price = self.trace.gas_price.as_ref().map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }); + let gas_limit = self.trace.gas_limit; + let input = Bytes::from(self.trace.input.clone()); + + let tx_type_u64 = u64::try_from(self.trace.r#type).map_err(|_| { + format_err!( + "Invalid transaction type value {} in transaction trace. Transaction type must be a valid u64.", + self.trace.r#type + ) + })?; + + // Try to convert to known Ethereum transaction type + let tx_type_result = TxType::try_from(tx_type_u64); + + // If this is an unknown transaction type, create an UnknownTxEnvelope + if tx_type_result.is_err() { + let mut fields_map = BTreeMap::new(); + + fields_map.insert( + "nonce".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", self.trace.nonce)), + ); + fields_map.insert( + "from".to_string(), + jsonrpc_core::serde_json::json!(format!("{:?}", from_address)), + ); + if let Some(to_addr) = to { + fields_map.insert( + "to".to_string(), + jsonrpc_core::serde_json::json!(format!("{:?}", to_addr)), + ); + } + fields_map.insert( + "value".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", value)), + ); + fields_map.insert( + "gas".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", gas_limit)), + ); + fields_map.insert( + "gasPrice".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", gas_price)), + ); + fields_map.insert( + "input".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{}", hex::encode(&input))), + ); + + let fields = OtherFields::new(fields_map); + let unknown_tx = UnknownTypedTransaction { + ty: AnyTxType(tx_type_u64 as u8), + fields, + memo: Default::default(), + }; + + let tx_hash = self.trace.hash.try_decode_proto("transaction hash")?; + let unknown_envelope = UnknownTxEnvelope { + hash: tx_hash, + inner: unknown_tx, + }; + + let any_envelope = AnyTxEnvelope::Unknown(unknown_envelope); + let recovered = Recovered::new_unchecked(any_envelope, from_address); + + return Ok(AlloyTransaction { + inner: recovered, + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp, + transaction_index, + effective_gas_price: if gas_price > 0 { Some(gas_price) } else { None }, + }); + } + + let tx_type = tx_type_result.unwrap(); + let nonce = self.trace.nonce; + + // Extract EIP-1559 fee fields from trace + let max_fee_per_gas_u128 = self.trace.max_fee_per_gas.as_ref().map_or(gas_price, |x| { + let val: U256 = x.into(); + val.to::() + }); + + let max_priority_fee_per_gas_u128 = + self.trace + .max_priority_fee_per_gas + .as_ref() + .map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }); + + // Extract access list from trace + let access_list: AccessList = self + .trace + .access_list + .iter() + .map(|access_tuple| -> Result<_, Error> { + let address = access_tuple + .address + .try_decode_proto("access tuple address")?; + let storage_keys = access_tuple + .storage_keys + .iter() + .map(|key| key.try_decode_proto("storage key")) + .collect::, _>>()?; + Ok(AccessListItem { + address, + storage_keys, + }) + }) + .collect::, Error>>()? + .into(); + + // Extract actual signature components from trace + let signature = extract_signature_from_trace(self.trace, tx_type)?; + + let to_kind = match to { + Some(addr) => TxKind::Call(addr), + None => TxKind::Create, + }; + + let envelope = match tx_type { + TxType::Legacy => { + let tx = TxLegacy { + chain_id: None, + nonce, + gas_price, + gas_limit, + to: to_kind, + value, + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Legacy(signed_tx) + } + TxType::Eip2930 => { + let tx = TxEip2930 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_price, + gas_limit, + to: to_kind, + value, + access_list: access_list.clone(), // Use actual access list from trace + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip2930(signed_tx) + } + TxType::Eip1559 => { + let tx = TxEip1559 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_limit, + max_fee_per_gas: max_fee_per_gas_u128, + max_priority_fee_per_gas: max_priority_fee_per_gas_u128, + to: to_kind, + value, + access_list: access_list.clone(), // Use actual access list from trace + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip1559(signed_tx) + } + TxType::Eip4844 => { + let to_address = to.ok_or_else(|| { + format_err!("EIP-4844 transactions cannot be contract creation transactions. The 'to' field must contain a valid address.") + })?; + + let blob_versioned_hashes: Vec = self + .trace + .blob_hashes + .iter() + .map(|hash| hash.try_decode_proto("blob hash")) + .collect::, _>>()?; + + let max_fee_per_blob_gas_u128 = + self.trace.blob_gas_fee_cap.as_ref().map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }); + + let tx_eip4844 = TxEip4844 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_limit, + max_fee_per_gas: max_fee_per_gas_u128, + max_priority_fee_per_gas: max_priority_fee_per_gas_u128, + to: to_address, + value, + access_list: access_list.clone(), // Use actual access list from trace + blob_versioned_hashes, + max_fee_per_blob_gas: max_fee_per_blob_gas_u128, + input: input.clone(), + }; + let tx = TxEip4844Variant::TxEip4844(tx_eip4844); + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip4844(signed_tx) + } + TxType::Eip7702 => { + let to_address = to.ok_or_else(|| { + format_err!("EIP-7702 transactions cannot be contract creation transactions. The 'to' field must contain a valid address.") + })?; + + // Convert set_code_authorizations to alloy authorization list + let authorization_list: Vec = self + .trace + .set_code_authorizations + .iter() + .map(|auth| -> Result<_, Error> { + let inner = alloy::eips::eip7702::Authorization { + chain_id: U256::from_be_slice(&auth.chain_id), + address: auth.address.try_decode_proto("authorization address")?, + nonce: auth.nonce, + }; + + let r = U256::from_be_slice(&auth.r); + let s = U256::from_be_slice(&auth.s); + let y_parity = auth.v as u8; + + Ok(alloy::eips::eip7702::SignedAuthorization::new_unchecked( + inner, y_parity, r, s, + )) + }) + .collect::, Error>>()?; + + let tx = TxEip7702 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_limit, + max_fee_per_gas: max_fee_per_gas_u128, + max_priority_fee_per_gas: max_priority_fee_per_gas_u128, + to: to_address, + value, + access_list: access_list.clone(), // Use actual access list from trace + authorization_list, + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip7702(signed_tx) + } + }; + + let any_envelope = AnyTxEnvelope::Ethereum(envelope); + let recovered = Recovered::new_unchecked(any_envelope, from_address); + + Ok(AlloyTransaction { + inner: recovered, + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp, + transaction_index, + effective_gas_price: if gas_price > 0 { Some(gas_price) } else { None }, // gas_price already contains effective gas price per protobuf spec }) } } @@ -234,143 +472,131 @@ impl TryInto for &Block { } } +impl TryInto for &Block { + type Error = Error; + + fn try_into(self) -> Result { + let header = self.header(); + + let block_hash = self.hash.try_decode_proto("block hash")?; + let consensus_header = alloy::consensus::Header { + number: header.number, + beneficiary: header.coinbase.try_decode_proto("author / coinbase")?, + parent_hash: header.parent_hash.try_decode_proto("parent hash")?, + ommers_hash: header.uncle_hash.try_decode_proto("uncle hash")?, + state_root: header.state_root.try_decode_proto("state root")?, + transactions_root: header + .transactions_root + .try_decode_proto("transactions root")?, + receipts_root: header.receipt_root.try_decode_proto("receipt root")?, + gas_used: header.gas_used, + gas_limit: header.gas_limit, + base_fee_per_gas: header.base_fee_per_gas.as_ref().map(|v| { + let val: U256 = v.into(); + val.to::() + }), + extra_data: Bytes::from(header.extra_data.clone()), + logs_bloom: if header.logs_bloom.is_empty() { + Bloom::ZERO + } else { + Bloom::try_from(header.logs_bloom.as_slice())? + }, + timestamp: header.timestamp.as_ref().map_or(0, |v| v.seconds as u64), + difficulty: header + .difficulty + .as_ref() + .map_or_else(|| U256::ZERO, |v| v.into()), + + mix_hash: header.mix_hash.try_decode_proto("mix hash")?, + nonce: header.nonce.into(), + + withdrawals_root: if header.withdrawals_root.is_empty() { + None + } else { + Some( + header + .withdrawals_root + .try_decode_proto("withdrawals root")?, + ) + }, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, + parent_beacon_block_root: if header.parent_beacon_root.is_empty() { + None + } else { + Some( + header + .parent_beacon_root + .try_decode_proto("parent beacon root")?, + ) + }, + requests_hash: if header.requests_hash.is_empty() { + None + } else { + Some(header.requests_hash.try_decode_proto("requests hash")?) + }, + block_access_list_hash: None, + slot_number: None, + }; + + let rpc_header = alloy::rpc::types::Header { + hash: block_hash, + inner: consensus_header, + total_difficulty: { + #[allow(deprecated)] + let total_difficulty = &header.total_difficulty; + total_difficulty.as_ref().map(|v| v.into()) + }, + size: Some(U256::from(self.size)), + }; + + let transactions = self + .transaction_traces + .iter() + .map(|t| TransactionTraceAt::new(t, self).try_into()) + .collect::>, Error>>()?; + + let uncles = self + .uncles + .iter() + .map(|u| u.hash.try_decode_proto("uncle hash")) + .collect::, _>>()?; + + use alloy::rpc::types::Block; + + let any_header: AnyRpcHeader = rpc_header.map(AnyHeader::from); + + Ok(Block { + header: any_header, + transactions: alloy::rpc::types::BlockTransactions::Full(transactions), + uncles, + withdrawals: None, + }) + } +} + impl TryInto for &Block { type Error = Error; fn try_into(self) -> Result { - let header = self - .header - .as_ref() - .expect("block header should always be present from gRPC Firehose"); + let alloy_block: AnyBlock = self.try_into()?; + + let transaction_receipts = self + .transaction_traces + .iter() + .filter_map(|t| transaction_trace_to_alloy_txn_reciept(t, self).transpose()) + .collect::, Error>>()? + .into_iter() + // Transaction receipts will be shared along the code, so we put them into an + // Arc here to avoid excessive cloning. + .map(Arc::new) + .collect(); + #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(LightEthereumBlock { - hash: Some(self.hash.try_decode_proto("block hash")?), - number: Some(U64::from(self.number)), - author: header.coinbase.try_decode_proto("author / coinbase")?, - parent_hash: header.parent_hash.try_decode_proto("parent hash")?, - uncles_hash: header.uncle_hash.try_decode_proto("uncle hash")?, - state_root: header.state_root.try_decode_proto("state root")?, - transactions_root: header - .transactions_root - .try_decode_proto("transactions root")?, - receipts_root: header.receipt_root.try_decode_proto("receipt root")?, - gas_used: U256::from(header.gas_used), - gas_limit: U256::from(header.gas_limit), - base_fee_per_gas: Some( - header - .base_fee_per_gas - .as_ref() - .map_or_else(U256::default, |v| v.into()), - ), - extra_data: Bytes::from(header.extra_data.clone()), - logs_bloom: match &header.logs_bloom.len() { - 0 => None, - _ => Some(header.logs_bloom.try_decode_proto("logs bloom")?), - }, - timestamp: header - .timestamp - .as_ref() - .map_or_else(U256::default, |v| U256::from(v.seconds)), - difficulty: header - .difficulty - .as_ref() - .map_or_else(U256::default, |v| v.into()), - total_difficulty: Some( - header - .total_difficulty - .as_ref() - .map_or_else(U256::default, |v| v.into()), - ), - // FIXME (SF): Firehose does not have seal fields, are they really used? Might be required for POA chains only also, I've seen that stuff on xDai (is this important?) - seal_fields: vec![], - uncles: self - .uncles - .iter() - .map(|u| u.hash.try_decode_proto("uncle hash")) - .collect::, _>>()?, - transactions: self - .transaction_traces - .iter() - .map(|t| TransactionTraceAt::new(t, self).try_into()) - .collect::, Error>>()?, - size: Some(U256::from(self.size)), - mix_hash: Some(header.mix_hash.try_decode_proto("mix hash")?), - nonce: Some(H64::from_low_u64_be(header.nonce)), - }), - transaction_receipts: self - .transaction_traces - .iter() - .filter_map(|t| { - t.receipt.as_ref().map(|r| { - Ok(web3::types::TransactionReceipt { - transaction_hash: t.hash.try_decode_proto("transaction hash")?, - transaction_index: U64::from(t.index), - block_hash: Some( - self.hash.try_decode_proto("transaction block hash")?, - ), - block_number: Some(U64::from(self.number)), - cumulative_gas_used: U256::from(r.cumulative_gas_used), - // FIXME (SF): What is the rule here about gas_used being None, when it's 0? - gas_used: Some(U256::from(t.gas_used)), - contract_address: { - match t.calls.len() { - 0 => None, - _ => { - match CallType::from_i32(t.calls[0].call_type) - .ok_or_else(|| { - format_err!( - "invalid call type: {}", - t.calls[0].call_type, - ) - })? { - CallType::Create => { - Some(t.calls[0].address.try_decode_proto( - "transaction contract address", - )?) - } - _ => None, - } - } - } - }, - logs: r - .logs - .iter() - .map(|l| LogAt::new(l, self, t).try_into()) - .collect::, Error>>()?, - status: TransactionTraceStatus::from_i32(t.status) - .ok_or_else(|| { - format_err!( - "invalid transaction trace status: {}", - t.status - ) - })? - .into(), - root: match r.state_root.len() { - 0 => None, // FIXME (SF): should this instead map to [0;32]? - // FIXME (SF): if len < 32, what do we do? - _ => Some( - r.state_root.try_decode_proto("transaction state root")?, - ), - }, - logs_bloom: r - .logs_bloom - .try_decode_proto("transaction logs bloom")?, - from: t.from.try_decode_proto("transaction from")?, - to: Some(t.to.try_decode_proto("transaction to")?), - transaction_type: None, - effective_gas_price: None, - }) - }) - }) - .collect::, Error>>()? - .into_iter() - // Transaction receipts will be shared along the code, so we put them into an - // Arc here to avoid excessive cloning. - .map(Arc::new) - .collect(), + block: Arc::new(LightEthereumBlock::new(alloy_block)), + transaction_receipts, }, // Comment (437a9f17-67cc-478f-80a3-804fe554b227): This Some() will avoid calls in the triggers_in_block // TODO: Refactor in a way that this is no longer needed. @@ -393,12 +619,118 @@ impl TryInto for &Block { } } +fn transaction_trace_to_alloy_txn_reciept( + t: &TransactionTrace, + block: &Block, +) -> Result, Error> { + use alloy::consensus::{Eip658Value, Receipt}; + let r = t.receipt.as_ref(); + + if r.is_none() { + return Ok(None); + } + + let r = r.unwrap(); + + let contract_address = match t.calls.len() { + 0 => None, + _ => { + match CallType::try_from(t.calls[0].call_type).map_err(|_| { + graph::anyhow::anyhow!("invalid call type: {}", t.calls[0].call_type) + })? { + CallType::Create => Some( + t.calls[0] + .address + .try_decode_proto("transaction contract address")?, + ), + _ => None, + } + } + }; + + let state_root = match &r.state_root { + b if b.is_empty() => None, + _ => Some(r.state_root.try_decode_proto("transaction state root")?), + }; + + let status = match TransactionTraceStatus::try_from(t.status) + .map_err(|_| format_err!("invalid transaction trace status: {}", t.status))? + { + TransactionTraceStatus::Unknown => { + return Err(format_err!( + "Transaction trace has UNKNOWN status; datasource is broken" + )); + } + TransactionTraceStatus::Succeeded => true, + TransactionTraceStatus::Failed | TransactionTraceStatus::Reverted => false, + }; + + // [EIP-658]: https://eips.ethereum.org/EIPS/eip-658 + // Before EIP-658, the state root field was used to indicate the status of the transaction. + // After EIP-658, the status field is used to indicate the status of the transaction. + let status = match state_root { + Some(root) => Eip658Value::PostState(root), + None => Eip658Value::Eip658(status), + }; + + let logs: Vec = r + .logs + .iter() + .map(|l| LogAt::new(l, block, t).try_into()) + .collect::, Error>>()?; + + let core_receipt = Receipt { + status, + cumulative_gas_used: r.cumulative_gas_used, + logs, + }; + + let logs_bloom = Bloom::try_from(r.logs_bloom.as_slice())?; + + let receipt_with_bloom = ReceiptWithBloom::new(core_receipt, logs_bloom); + + let tx_type_u64 = u64::try_from(t.r#type).map_err(|_| { + format_err!( + "Invalid transaction type value {} in transaction receipt. Transaction type must be a valid u64.", + t.r#type + ) + })?; + + let any_envelope = AnyReceiptEnvelope { + inner: receipt_with_bloom, + r#type: tx_type_u64 as u8, + }; + + let receipt = alloy_rpc_types::TransactionReceipt { + transaction_hash: t.hash.try_decode_proto("transaction hash")?, + transaction_index: Some(t.index as u64), + block_hash: Some(block.hash.try_decode_proto("transaction block hash")?), + block_number: Some(block.number), + gas_used: t.gas_used, + contract_address, + from: t.from.try_decode_proto("transaction from")?, + to: get_to_address(t)?, + effective_gas_price: t.gas_price.as_ref().map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }), // gas_price already contains effective gas price per protobuf spec + blob_gas_used: r.blob_gas_used, + blob_gas_price: r.blob_gas_price.as_ref().map(|x| { + let val: U256 = x.into(); + val.to::() + }), + inner: any_envelope, + }; + + Ok(Some(receipt)) +} + impl BlockHeader { pub fn parent_ptr(&self) -> Option { match self.parent_hash.len() { 0 => None, _ => Some(BlockPtr::from(( - H256::from_slice(self.parent_hash.as_ref()), + B256::from_slice(self.parent_hash.as_ref()), self.number - 1, ))), } @@ -407,13 +739,13 @@ impl BlockHeader { impl<'a> From<&'a BlockHeader> for BlockPtr { fn from(b: &'a BlockHeader) -> BlockPtr { - BlockPtr::from((H256::from_slice(b.hash.as_ref()), b.number)) + BlockPtr::from((B256::from_slice(b.hash.as_ref()), b.number)) } } impl<'a> From<&'a Block> for BlockPtr { fn from(b: &'a Block) -> BlockPtr { - BlockPtr::from((H256::from_slice(b.hash.as_ref()), b.number)) + BlockPtr::from((B256::from_slice(b.hash.as_ref()), b.number)) } } @@ -516,6 +848,33 @@ impl BlockchainBlock for HeaderOnlyBlock { } } +fn extract_signature_from_trace( + _trace: &TransactionTrace, + _tx_type: TxType, +) -> Result { + use alloy::primitives::{Signature as PrimitiveSignature, U256}; + + // Create a dummy signature with r = 0, s = 0 and even y-parity (false) + let dummy = PrimitiveSignature::new(U256::ZERO, U256::ZERO, false); + + Ok(dummy) +} + +fn get_to_address(trace: &TransactionTrace) -> Result, Error> { + // Try to detect contract creation transactions, which have no 'to' address + let is_contract_creation = trace.to.is_empty() + || trace + .calls + .first() + .is_some_and(|call| CallType::try_from(call.call_type) == Ok(CallType::Create)); + + if is_contract_creation { + Ok(None) + } else { + Ok(Some(trace.to.try_decode_proto("transaction to address")?)) + } +} + #[cfg(test)] mod test { use graph::{blockchain::Block as _, prelude::chrono::Utc}; @@ -528,14 +887,19 @@ mod test { #[test] fn ensure_block_serialization() { let now = Utc::now().timestamp(); - let mut block = Block::default(); - let mut header = BlockHeader::default(); - header.timestamp = Some(Timestamp { - seconds: now, - nanos: 0, - }); - block.header = Some(header); + let header = BlockHeader { + timestamp: Some(Timestamp { + seconds: now, + nanos: 0, + }), + ..Default::default() + }; + + let block = Block { + header: Some(header.clone()), + ..Default::default() + }; let str_block = block.data().unwrap().to_string(); @@ -545,4 +909,116 @@ mod test { format!(r#"{{"block":{{"data":null,"timestamp":"{}"}}}}"#, now) ); } + + #[test] + fn test_unknown_transaction_type_conversion() { + use super::TransactionTraceAt; + use crate::codec::TransactionTrace; + use graph::prelude::alloy::network::AnyTxEnvelope; + use graph::prelude::alloy::primitives::B256; + + let header = BlockHeader { + number: 123456, + timestamp: Some(Timestamp { + seconds: 1234567890, + nanos: 0, + }), + ..Default::default() + }; + let block = Block { + header: Some(header), + number: 123456, + hash: vec![0u8; 32], + ..Default::default() + }; + + let trace = TransactionTrace { + r#type: 126, // 0x7e Optimism deposit transaction + hash: vec![1u8; 32], + from: vec![2u8; 20], + to: vec![3u8; 20], + nonce: 42, + gas_limit: 21000, + index: 0, + ..Default::default() + }; + + let trace_at = TransactionTraceAt::new(&trace, &block); + let result: Result< + graph::prelude::alloy::rpc::types::Transaction, + graph::prelude::Error, + > = trace_at.try_into(); + + assert!( + result.is_ok(), + "Should successfully convert unknown transaction type" + ); + + let tx = result.unwrap(); + + match tx.inner.inner() { + AnyTxEnvelope::Unknown(unknown_envelope) => { + assert_eq!(unknown_envelope.inner.ty.0, 126); + assert_eq!(unknown_envelope.hash, B256::from_slice(&trace.hash)); + assert!( + !unknown_envelope.inner.fields.is_empty(), + "OtherFields should contain transaction data" + ); + } + _ => panic!("Expected AnyTxEnvelope::Unknown, got Ethereum variant"), + } + + assert_eq!(tx.block_number, Some(123456)); + assert_eq!(tx.transaction_index, Some(0)); + assert_eq!(tx.block_hash, Some(B256::from_slice(&block.hash))); + } + + #[test] + fn test_unknown_receipt_type_conversion() { + use super::transaction_trace_to_alloy_txn_reciept; + use crate::codec::TransactionTrace; + + let header = BlockHeader { + number: 123456, + ..Default::default() + }; + let block = Block { + header: Some(header), + hash: vec![0u8; 32], + ..Default::default() + }; + + let receipt = super::TransactionReceipt { + cumulative_gas_used: 21000, + logs_bloom: vec![0u8; 256], + ..Default::default() + }; + let trace = TransactionTrace { + r#type: 126, // 0x7e Optimism deposit transaction + hash: vec![1u8; 32], + from: vec![2u8; 20], + to: vec![3u8; 20], + index: 0, + gas_used: 21000, + status: 1, + receipt: Some(receipt), + ..Default::default() + }; + + let result = transaction_trace_to_alloy_txn_reciept(&trace, &block); + + assert!( + result.is_ok(), + "Should successfully convert receipt with unknown transaction type" + ); + + let receipt_opt = result.unwrap(); + assert!(receipt_opt.is_some(), "Receipt should be present"); + + let receipt = receipt_opt.unwrap(); + + assert_eq!(receipt.inner.r#type, 126); + assert_eq!(receipt.gas_used, 21000); + assert_eq!(receipt.transaction_index, Some(0)); + } } diff --git a/chain/ethereum/src/data_source.rs b/chain/ethereum/src/data_source.rs index 47abc96649d..709884496e8 100644 --- a/chain/ethereum/src/data_source.rs +++ b/chain/ethereum/src/data_source.rs @@ -1,37 +1,61 @@ -use anyhow::{anyhow, Error}; -use anyhow::{ensure, Context}; -use graph::blockchain::TriggerWithHandler; -use graph::components::store::StoredDynamicDataSource; -use graph::data_source::CausalityRegion; -use graph::prelude::ethabi::ethereum_types::H160; -use graph::prelude::ethabi::StateMutability; -use graph::prelude::futures03::future::try_join; -use graph::prelude::futures03::stream::FuturesOrdered; -use graph::prelude::{Link, SubgraphManifestValidationError}; -use graph::slog::{o, trace}; +use anyhow::{Context, ensure}; +use anyhow::{Error, anyhow}; +use async_trait::async_trait; +use graph::abi; +use graph::abi::EventExt; +use graph::abi::FunctionExt; +use graph::blockchain::{BlockPtr, TriggerWithHandler}; +use graph::components::ethereum::AnyTransaction; +use graph::components::link_resolver::LinkResolverContext; +use graph::components::metrics::subgraph::SubgraphInstanceMetrics; +use graph::components::store::{EthereumCallCache, StoredDynamicDataSource}; +use graph::components::subgraph::{HostMetrics, InstanceDSTemplateInfo, MappingError}; +use graph::components::trigger_processor::RunnableTriggers; +use graph::data::subgraph::DeploymentHash; +use graph::data_source::common::{ + AbiJson, CallDecls, DeclaredCall, FindMappingABI, MappingABI, UnresolvedCallDecls, + UnresolvedMappingABI, +}; +use graph::data_source::{CausalityRegion, MappingTrigger as MappingTriggerType}; +use graph::env::ENV_VARS; +use graph::futures03::TryStreamExt; +use graph::futures03::future::try_join; +use graph::futures03::stream::FuturesOrdered; +use graph::prelude::alloy::primitives::keccak256; +use graph::prelude::alloy::{ + consensus::{TxEnvelope, TxLegacy}, + network::TransactionResponse, + primitives::{Address, B256, U256}, + rpc::types::Log, +}; +use graph::prelude::{Link, SubgraphManifestValidationError, alloy}; +use graph::slog::{debug, error, o, trace}; +use itertools::Itertools; +use serde::de::Error as ErrorD; +use serde::{Deserialize, Deserializer}; use std::collections::HashSet; use std::num::NonZeroU32; use std::str::FromStr; use std::sync::Arc; -use tiny_keccak::{keccak256, Keccak}; +use std::time::{Duration, Instant}; use graph::{ blockchain::{self, Blockchain}, prelude::{ - async_trait, - ethabi::{Address, Contract, Event, Function, LogParam, ParamType, RawLog}, - serde_json, warn, - web3::types::{Log, Transaction, H256}, - BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, EthereumCall, - LightEthereumBlock, LightEthereumBlockExt, LinkResolver, Logger, TryStreamExt, + BlockNumber, CheapClone, EthereumCall, LightEthereumBlock, LightEthereumBlockExt, + LinkResolver, Logger, serde_json, warn, }, }; use graph::data::subgraph::{ - calls_host_fn, DataSourceContext, Source, MIN_SPEC_VERSION, SPEC_VERSION_0_0_8, + DataSourceContext, MIN_SPEC_VERSION, SPEC_VERSION_0_0_8, SPEC_VERSION_1_2_0, Source, + calls_host_fn, }; +use crate::NodeCapabilities; +use crate::adapter::EthereumAdapter as _; use crate::chain::Chain; +use crate::network::EthereumNetworkAdapters; use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger, MappingTrigger}; // The recommended kind is `ethereum`, `ethereum/contract` is accepted for backwards compatibility. @@ -59,20 +83,26 @@ pub struct DataSource { } impl blockchain::DataSource for DataSource { - fn from_template_info(info: DataSourceTemplateInfo) -> Result { - let DataSourceTemplateInfo { - template, + fn from_template_info( + info: InstanceDSTemplateInfo, + ds_template: &graph::data_source::DataSourceTemplate, + ) -> Result { + // Note: There clearly is duplication between the data in `ds_template and the `template` + // field here. Both represent a template definition, would be good to unify them. + let InstanceDSTemplateInfo { + template: _, params, context, creation_block, } = info; - let template = template.into_onchain().ok_or(anyhow!( + + let template = ds_template.as_onchain().ok_or(anyhow!( "Cannot create onchain data source from offchain template" ))?; // Obtain the address from the parameters let string = params - .get(0) + .first() .with_context(|| { format!( "Failed to create data source from template `{}`: address parameter is missing", @@ -94,14 +124,14 @@ impl blockchain::DataSource for DataSource { .with_context(|| format!("template `{}`", template.name))?; Ok(DataSource { - kind: template.kind, - network: template.network, - name: template.name, + kind: template.kind.clone(), + network: template.network.clone(), + name: template.name.clone(), manifest_idx: template.manifest_idx, address: Some(address), start_block: creation_block, end_block: None, - mapping: template.mapping, + mapping: template.mapping.clone(), context: Arc::new(context), creation_block: Some(creation_block), contract_abi, @@ -109,7 +139,14 @@ impl blockchain::DataSource for DataSource { } fn address(&self) -> Option<&[u8]> { - self.address.as_ref().map(|x| x.as_bytes()) + self.address.as_ref().map(|x| x.as_slice()) + } + + fn has_declared_calls(&self) -> bool { + self.mapping + .event_handlers + .iter() + .any(|handler| !handler.calls.decls.is_empty()) } fn handler_kinds(&self) -> HashSet<&str> { @@ -206,7 +243,7 @@ impl blockchain::DataSource for DataSource { } fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - let param = self.address.map(|addr| addr.0.into()); + let param = self.address.map(|addr| addr.as_slice().into()); StoredDynamicDataSource { manifest_idx: self.manifest_idx, param, @@ -245,7 +282,7 @@ impl blockchain::DataSource for DataSource { let contract_abi = template.mapping.find_abi(&template.source.abi)?; - let address = param.map(|x| H160::from_slice(&x)); + let address = param.map(|x| Address::from_slice(&x)); Ok(DataSource { kind: template.kind.to_string(), network: template.network.as_ref().map(|s| s.to_string()), @@ -261,7 +298,7 @@ impl blockchain::DataSource for DataSource { }) } - fn validate(&self) -> Vec { + fn validate(&self, spec_version: &semver::Version) -> Vec { let mut errors = vec![]; if !ETHEREUM_KINDS.contains(&self.kind.as_str()) { @@ -340,6 +377,33 @@ impl blockchain::DataSource for DataSource { } } + if spec_version < &SPEC_VERSION_1_2_0 { + for handler in &self.mapping.event_handlers { + if !handler.calls.decls.is_empty() { + errors.push(anyhow!( + "handler {}: declaring eth calls on handlers is only supported for specVersion >= 1.2.0", handler.event + )); + break; + } + } + } + + for handler in &self.mapping.event_handlers { + for call in handler.calls.decls.as_ref() { + match self.mapping.find_abi(&call.expr.abi) { + // TODO: Handle overloaded functions by passing a signature + Ok(abi) => match abi.function(&call.expr.abi, &call.expr.func, None) { + Ok(_) => {} + Err(e) => { + errors.push(e); + } + }, + Err(e) => { + errors.push(e); + } + } + } + } errors } @@ -348,17 +412,24 @@ impl blockchain::DataSource for DataSource { } fn min_spec_version(&self) -> semver::Version { - self.mapping - .block_handlers - .iter() - .fold(MIN_SPEC_VERSION, |mut min, handler| { - min = match handler.filter { - Some(BlockHandlerFilter::Polling { every: _ }) => SPEC_VERSION_0_0_8, - Some(BlockHandlerFilter::Once) => SPEC_VERSION_0_0_8, - _ => min, - }; - min - }) + let mut min_version = MIN_SPEC_VERSION; + + for handler in &self.mapping.block_handlers { + match handler.filter { + Some(BlockHandlerFilter::Polling { every: _ }) | Some(BlockHandlerFilter::Once) => { + min_version = std::cmp::max(min_version, SPEC_VERSION_0_0_8); + } + _ => {} + } + } + + for handler in &self.mapping.event_handlers { + if handler.has_additional_topics() { + min_version = std::cmp::max(min_version, SPEC_VERSION_1_2_0); + } + } + + min_version } fn runtime(&self) -> Option>> { @@ -366,6 +437,43 @@ impl blockchain::DataSource for DataSource { } } +/// Generic function that creates a mock legacy Transaction from ANY log +fn create_dummy_transaction( + block_number: u64, + block_hash: B256, + transaction_index: Option, + transaction_hash: Option, +) -> Result { + use graph::components::ethereum::AnyTxEnvelope; + use graph::prelude::alloy::{ + consensus::Signed, consensus::transaction::Recovered, primitives::Signature, + rpc::types::Transaction, + }; + + let tx = TxLegacy::default(); + + // Create a dummy signature + let signature = Signature::new(U256::ZERO, U256::ZERO, false); + + let tx_hash = transaction_hash.ok_or(anyhow!("Log has no transaction hash"))?; + let signed_tx = Signed::new_unchecked(tx, signature, tx_hash); + let eth_envelope = TxEnvelope::Legacy(signed_tx); + + // Wrap in AnyTxEnvelope + let any_envelope = AnyTxEnvelope::Ethereum(eth_envelope); + + let recovered = Recovered::new_unchecked(any_envelope, Address::ZERO); + + Ok(Transaction { + inner: recovered, + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp: None, + transaction_index, + effective_gas_price: None, + }) +} + impl DataSource { fn from_manifest( kind: String, @@ -397,23 +505,16 @@ impl DataSource { }) } - fn handlers_for_log(&self, log: &Log) -> Vec { - // Get signature from the log - let topic0 = match log.topics.get(0) { - Some(topic0) => topic0, - // Events without a topic should just be be ignored - None => return vec![], - }; - + fn handlers_for_log(&self, log: &alloy::rpc::types::Log) -> Vec { self.mapping .event_handlers .iter() - .filter(|handler| *topic0 == handler.topic0()) + .filter(|handler| handler.matches(log)) .cloned() .collect::>() } - fn handler_for_call(&self, call: &EthereumCall) -> Result, Error> { + fn handler_for_call(&self, call: &EthereumCall) -> Result, Error> { // First four bytes of the input for the call are the first four // bytes of hash of the function signature ensure!( @@ -423,80 +524,73 @@ impl DataSource { let target_method_id = &call.input.0[..4]; - Ok(self - .mapping - .call_handlers - .iter() - .find(move |handler| { - let fhash = keccak256(handler.function.as_bytes()); - let actual_method_id = [fhash[0], fhash[1], fhash[2], fhash[3]]; - target_method_id == actual_method_id - }) - .cloned()) + Ok(self.mapping.call_handlers.iter().find(move |handler| { + let fhash = keccak256(handler.function.as_bytes()); + let actual_method_id = [fhash[0], fhash[1], fhash[2], fhash[3]]; + target_method_id == actual_method_id + })) } fn handler_for_block( &self, trigger_type: &EthereumBlockTriggerType, block: BlockNumber, - ) -> Option { + ) -> Option<&MappingBlockHandler> { match trigger_type { // Start matches only initialization handlers with a `once` filter - EthereumBlockTriggerType::Start => self - .mapping - .block_handlers - .iter() - .find(move |handler| match handler.filter { - Some(BlockHandlerFilter::Once) => block == self.start_block, - _ => false, - }) - .cloned(), + EthereumBlockTriggerType::Start => { + self.mapping + .block_handlers + .iter() + .find(move |handler| match handler.filter { + Some(BlockHandlerFilter::Once) => block == self.start_block, + _ => false, + }) + } // End matches all handlers without a filter or with a `polling` filter - EthereumBlockTriggerType::End => self - .mapping - .block_handlers - .iter() - .find(move |handler| match handler.filter { - Some(BlockHandlerFilter::Polling { every }) => { - let start_block = self.start_block; - let should_trigger = (block - start_block) % every.get() as i32 == 0; - should_trigger - } - None => true, - _ => false, - }) - .cloned(), + EthereumBlockTriggerType::End => { + self.mapping + .block_handlers + .iter() + .find(move |handler| match handler.filter { + Some(BlockHandlerFilter::Polling { every }) => { + let start_block = self.start_block; + (block - start_block) % every.get() as i32 == 0 + } + None => true, + _ => false, + }) + } EthereumBlockTriggerType::WithCallTo(_address) => self .mapping .block_handlers .iter() - .find(move |handler| handler.filter == Some(BlockHandlerFilter::Call)) - .cloned(), + .find(move |handler| handler.filter == Some(BlockHandlerFilter::Call)), } } - /// Returns the contract event with the given signature, if it exists. A an event from the ABI + /// Returns the contract event with the given signature, if it exists. An event from the ABI /// will be matched if: /// 1. An event signature is equal to `signature`. /// 2. There are no equal matches, but there is exactly one event that equals `signature` if all /// `indexed` modifiers are removed from the parameters. - fn contract_event_with_signature(&self, signature: &str) -> Option<&Event> { + fn contract_event_with_signature(&self, signature: &str) -> Option<&abi::Event> { // Returns an `Event(uint256,address)` signature for an event, without `indexed` hints. - fn ambiguous_event_signature(event: &Event) -> String { + fn ambiguous_event_signature(event: &abi::Event) -> String { format!( "{}({})", event.name, event .inputs .iter() - .map(|input| event_param_type_signature(&input.kind)) + .map(|input| input.selector_type().into_owned()) .collect::>() .join(",") ) } // Returns an `Event(indexed uint256,address)` type signature for an event. - fn event_signature(event: &Event) -> String { + fn event_signature(event: &abi::Event) -> String { format!( "{}({})", event.name, @@ -506,40 +600,13 @@ impl DataSource { .map(|input| format!( "{}{}", if input.indexed { "indexed " } else { "" }, - event_param_type_signature(&input.kind) + input.selector_type() )) .collect::>() .join(",") ) } - // Returns the signature of an event parameter type (e.g. `uint256`). - fn event_param_type_signature(kind: &ParamType) -> String { - use ParamType::*; - - match kind { - Address => "address".into(), - Bytes => "bytes".into(), - Int(size) => format!("int{}", size), - Uint(size) => format!("uint{}", size), - Bool => "bool".into(), - String => "string".into(), - Array(inner) => format!("{}[]", event_param_type_signature(inner)), - FixedBytes(size) => format!("bytes{}", size), - FixedArray(inner, size) => { - format!("{}[{}]", event_param_type_signature(inner), size) - } - Tuple(components) => format!( - "({})", - components - .iter() - .map(event_param_type_signature) - .collect::>() - .join(",") - ), - } - } - self.contract_abi .contract .events() @@ -578,7 +645,9 @@ impl DataSource { }) } - fn contract_function_with_signature(&self, target_signature: &str) -> Option<&Function> { + fn contract_function_with_signature(&self, target_signature: &str) -> Option<&abi::Function> { + use abi::StateMutability; + self.contract_abi .contract .functions() @@ -592,13 +661,13 @@ impl DataSource { let mut arguments = function .inputs .iter() - .map(|input| format!("{}", input.kind)) + .map(|input| input.selector_type().into_owned()) .collect::>() .join(","); // `address,uint256,bool) arguments.push(')'); // `operation(address,uint256,bool)` - let actual_signature = vec![function.name.clone(), arguments].join("("); + let actual_signature = [function.name.clone(), arguments].join("("); target_signature == actual_signature }) } @@ -642,7 +711,7 @@ impl DataSource { MappingTrigger::Block { block: block.cheap_clone(), }, - handler.handler, + handler.handler.clone(), block.block_ptr(), block.timestamp(), ))) @@ -682,11 +751,7 @@ impl DataSource { .into_iter() .filter_map(|(event_handler, event_abi)| { event_abi - .parse_log(RawLog { - topics: log.topics.clone(), - data: log.data.clone().0, - }) - .map(|log| log.params) + .decode_log(&log) .map_err(|e| { trace!( logger, @@ -723,36 +788,47 @@ impl DataSource { // associated transaction and instead have `transaction_hash == block.hash`, // in which case we pass a dummy transaction to the mappings. // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. - let transaction = if log.transaction_hash != block.hash { + // There is another special case in zkSync-era, where the transaction hash in this case would be zero + // See https://docs.zksync.io/zk-stack/concepts/blocks.html#fictive-l2-block-finalizing-the-batch + let transaction = if log.transaction_hash == Some(block.hash()) + || log.transaction_hash == Some(B256::ZERO) + { + create_dummy_transaction( + block.number_u64(), + block.hash(), + log.transaction_index, + log.transaction_hash, + )? + } else { + // This is the general case where the log's transaction hash does not match the block's hash + // and is not a special zero hash, implying a real transaction associated with this log. block .transaction_for_log(&log) .context("Found no transaction for event")? - } else { - // Infer some fields from the log and fill the rest with zeros. - Transaction { - hash: log.transaction_hash.unwrap(), - block_hash: block.hash, - block_number: block.number, - transaction_index: log.transaction_index, - from: Some(H160::zero()), - ..Transaction::default() - } }; let logging_extras = Arc::new(o! { "signature" => event_handler.event.to_string(), - "address" => format!("{}", &log.address), - "transaction" => format!("{}", &transaction.hash), + "address" => format!("{}", &log.address()), + "transaction" => format!("{}", &transaction.tx_hash()), }); + let handler = event_handler.handler.clone(); + let calls = DeclaredCall::from_log_trigger_with_event( + &self.mapping, + &event_handler.calls, + &log, + ¶ms, + )?; Ok(Some(TriggerWithHandler::::new_with_logging_extras( MappingTrigger::Log { block: block.cheap_clone(), transaction: Arc::new(transaction), - log: log, + log, params, receipt: receipt.map(|r| r.cheap_clone()), + calls, }, - event_handler.handler, + handler, block.block_ptr(), block.timestamp(), logging_extras, @@ -778,20 +854,15 @@ impl DataSource { ) })?; - // Parse the inputs - // - // Take the input for the call, chop off the first 4 bytes, then call - // `function.decode_input` to get a vector of `Token`s. Match the `Token`s - // with the `Param`s in `function.inputs` to create a `Vec`. - let tokens = match function_abi.decode_input(&call.input.0[4..]).with_context( - || { + let values = match function_abi + .abi_decode_input(&call.input.0[4..]) + .with_context(|| { format!( "Generating function inputs for the call {:?} failed, raw input: {}", &function_abi, hex::encode(&call.input.0) ) - }, - ) { + }) { Ok(val) => val, // See also 280b0108-a96e-4738-bb37-60ce11eeb5bf Err(err) => { @@ -801,27 +872,22 @@ impl DataSource { }; ensure!( - tokens.len() == function_abi.inputs.len(), + values.len() == function_abi.inputs.len(), "Number of arguments in call does not match \ number of inputs in function signature." ); - let inputs = tokens + let inputs = values .into_iter() .enumerate() - .map(|(i, token)| LogParam { + .map(|(i, value)| abi::DynSolParam { name: function_abi.inputs[i].name.clone(), - value: token, + value, }) .collect::>(); - // Parse the outputs - // - // Take the output for the call, then call `function.decode_output` to - // get a vector of `Token`s. Match the `Token`s with the `Param`s in - // `function.outputs` to create a `Vec`. - let tokens = function_abi - .decode_output(&call.output.0) + let values = function_abi + .abi_decode_output(&call.output.0) .with_context(|| { format!( "Decoding function outputs for the call {:?} failed, raw output: {}", @@ -831,17 +897,17 @@ impl DataSource { })?; ensure!( - tokens.len() == function_abi.outputs.len(), + values.len() == function_abi.outputs.len(), "Number of parameters in the call output does not match \ number of outputs in the function signature." ); - let outputs = tokens + let outputs = values .into_iter() .enumerate() - .map(|(i, token)| LogParam { + .map(|(i, value)| abi::DynSolParam { name: function_abi.outputs[i].name.clone(), - value: token, + value, }) .collect::>(); @@ -853,7 +919,7 @@ impl DataSource { let logging_extras = Arc::new(o! { "function" => handler.function.to_string(), "to" => format!("{}", &call.to), - "transaction" => format!("{}", &transaction.hash), + "transaction" => format!("{}", &transaction.tx_hash()), }); Ok(Some(TriggerWithHandler::::new_with_logging_extras( MappingTrigger::Call { @@ -863,7 +929,7 @@ impl DataSource { inputs, outputs, }, - handler.handler, + handler.handler.clone(), block.block_ptr(), block.timestamp(), logging_extras, @@ -873,6 +939,256 @@ impl DataSource { } } +pub struct DecoderHook { + eth_adapters: Arc, + call_cache: Arc, + eth_call_gas: Option, +} + +impl DecoderHook { + pub fn new( + eth_adapters: Arc, + call_cache: Arc, + eth_call_gas: Option, + ) -> Self { + Self { + eth_adapters, + call_cache, + eth_call_gas, + } + } +} + +impl DecoderHook { + /// Perform a batch of eth_calls, observing the execution time of each + /// call. Returns a list of the call labels for which we received a + /// `None` response, indicating a revert + async fn eth_calls( + &self, + logger: &Logger, + block_ptr: &BlockPtr, + calls_and_metrics: Vec<(Arc, DeclaredCall)>, + ) -> Result, MappingError> { + // This check is not just to speed things up, but is also needed to + // make sure the runner tests don't fail; they don't have declared + // eth calls, but without this check we try to get an eth adapter + // even when there are no calls, which fails in the runner test + // setup + if calls_and_metrics.is_empty() { + return Ok(vec![]); + } + + let start = Instant::now(); + + let (metrics, calls): (Vec<_>, Vec<_>) = calls_and_metrics.into_iter().unzip(); + + let (calls, labels): (Vec<_>, Vec<_>) = calls + .into_iter() + .map(|call| call.as_eth_call(block_ptr.clone(), self.eth_call_gas)) + .unzip(); + + let eth_adapter = self.eth_adapters.call_or_cheapest(Some(&NodeCapabilities { + archive: true, + traces: false, + }))?; + + let call_refs = calls.iter().collect::>(); + let results = eth_adapter + .contract_calls(logger, &call_refs, self.call_cache.cheap_clone()) + .await + .map_err(|e| { + // An error happened, everybody gets charged + let elapsed = start.elapsed().as_secs_f64() / call_refs.len() as f64; + for (metrics, call) in metrics.iter().zip(call_refs) { + metrics.observe_eth_call_execution_time( + elapsed, + &call.contract_name, + &call.function.name, + ); + } + MappingError::from(e) + })?; + + // We don't have time measurements for each call (though that would be nice) + // Use the average time of all calls that we want to observe as the time for + // each call + let to_observe = results + .iter() + .filter(|(_, source)| source.observe()) + .count() as f64; + let elapsed = start.elapsed().as_secs_f64() / to_observe; + + results + .iter() + .zip(metrics) + .zip(calls) + .for_each(|(((_, source), metrics), call)| { + if source.observe() { + metrics.observe_eth_call_execution_time( + elapsed, + &call.contract_name, + &call.function.name, + ); + } + }); + + let labels = results + .iter() + .zip(labels) + .filter_map(|((res, _), label)| if res.is_none() { Some(label) } else { None }) + .map(|s| s.to_string()) + .collect(); + Ok(labels) + } + + fn collect_declared_calls<'a>( + &self, + runnables: &Vec>, + ) -> Vec<(Arc, DeclaredCall)> { + // Extract all hosted triggers from runnables + let all_triggers = runnables + .iter() + .flat_map(|runnable| &runnable.hosted_triggers); + + // Collect calls from both onchain and subgraph triggers + let mut all_calls = Vec::new(); + + for trigger in all_triggers { + let host_metrics = trigger.host.host_metrics(); + + match &trigger.mapping_trigger.trigger { + MappingTriggerType::Onchain(t) => { + if let MappingTrigger::Log { calls, .. } = t { + for call in calls.clone() { + all_calls.push((host_metrics.cheap_clone(), call)); + } + } + } + MappingTriggerType::Subgraph(t) => { + for call in t.calls.clone() { + // Convert subgraph call to the expected DeclaredCall type if needed + // or handle differently based on the types + all_calls.push((host_metrics.cheap_clone(), call)); + } + } + MappingTriggerType::Offchain(_) => {} + } + } + + all_calls + } + + /// Deduplicate calls. Unfortunately, we can't get `DeclaredCall` to + /// implement `Hash` or `Ord` easily, so we can only deduplicate by + /// comparing the whole call not with a `HashSet` or `BTreeSet`. + /// Since that can be inefficient, we don't deduplicate if we have an + /// enormous amount of calls; in that case though, things will likely + /// blow up because of the amount of I/O that many calls cause. + /// Cutting off at 1000 is fairly arbitrary + fn deduplicate_calls( + &self, + calls: Vec<(Arc, DeclaredCall)>, + ) -> Vec<(Arc, DeclaredCall)> { + if calls.len() >= 1000 { + return calls; + } + + let mut uniq_calls = Vec::new(); + for (metrics, call) in calls { + if !uniq_calls.iter().any(|(_, c)| c == &call) { + uniq_calls.push((metrics, call)); + } + } + uniq_calls + } + + /// Log information about failed eth calls. 'Failure' here simply + /// means that the call was reverted; outright errors lead to a real + /// error. For reverted calls, `self.eth_calls` returns the label + /// from the manifest for that call. + /// + /// One reason why declared calls can fail is if they are attached + /// to the wrong handler, or if arguments are specified incorrectly. + /// Calls that revert every once in a while might be ok and what the + /// user intended, but we want to clearly log so that users can spot + /// mistakes in their manifest, which will lead to unnecessary eth + /// calls + fn log_declared_call_results( + logger: &Logger, + failures: &[String], + calls_count: usize, + trigger_count: usize, + elapsed: Duration, + ) { + let fail_count = failures.len(); + + if fail_count > 0 { + let mut counts: Vec<_> = failures.iter().counts().into_iter().collect(); + counts.sort_by_key(|(label, _)| *label); + + let failure_summary = counts + .into_iter() + .map(|(label, count)| { + let times = if count == 1 { "time" } else { "times" }; + format!("{label} ({count} {times})") + }) + .join(", "); + + error!(logger, "Declared calls failed"; + "triggers" => trigger_count, + "calls_count" => calls_count, + "fail_count" => fail_count, + "calls_ms" => elapsed.as_millis(), + "failures" => format!("[{}]", failure_summary) + ); + } else { + debug!(logger, "Declared calls"; + "triggers" => trigger_count, + "calls_count" => calls_count, + "calls_ms" => elapsed.as_millis() + ); + } + } +} + +#[async_trait] +impl blockchain::DecoderHook for DecoderHook { + async fn after_decode<'a>( + &self, + logger: &Logger, + block_ptr: &BlockPtr, + runnables: Vec>, + metrics: &Arc, + ) -> Result>, MappingError> { + if ENV_VARS.mappings.disable_declared_calls { + return Ok(runnables); + } + + let _section = metrics.stopwatch.start_section("declared_ethereum_call"); + + let start = Instant::now(); + // Collect and process declared calls + let calls = self.collect_declared_calls(&runnables); + let deduplicated_calls = self.deduplicate_calls(calls); + + // Execute calls and log results + let calls_count = deduplicated_calls.len(); + let results = self + .eth_calls(logger, block_ptr, deduplicated_calls) + .await?; + + Self::log_declared_call_results( + logger, + &results, + calls_count, + runnables.len(), + start.elapsed(), + ); + + Ok(runnables) + } +} + #[derive(Clone, Debug, Eq, PartialEq, Deserialize)] pub struct UnresolvedDataSource { pub kind: String, @@ -887,9 +1203,11 @@ pub struct UnresolvedDataSource { impl blockchain::UnresolvedDataSource for UnresolvedDataSource { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, manifest_idx: u32, + spec_version: &semver::Version, ) -> Result { let UnresolvedDataSource { kind, @@ -900,7 +1218,7 @@ impl blockchain::UnresolvedDataSource for UnresolvedDataSource { context, } = self; - let mapping = mapping.resolve(resolver, logger).await.with_context(|| { + let mapping = mapping.resolve(deployment_hash, resolver, logger, spec_version).await.with_context(|| { format!( "failed to resolve data source {} with source_address {:?} and source_start_block {}", name, source.address, source.start_block @@ -911,7 +1229,7 @@ impl blockchain::UnresolvedDataSource for UnresolvedDataSource { } } -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] pub struct UnresolvedDataSourceTemplate { pub kind: String, pub network: Option, @@ -934,9 +1252,11 @@ pub struct DataSourceTemplate { impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, manifest_idx: u32, + spec_version: &semver::Version, ) -> Result { let UnresolvedDataSourceTemplate { kind, @@ -947,7 +1267,7 @@ impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTem } = self; let mapping = mapping - .resolve(resolver, logger) + .resolve(deployment_hash, resolver, logger, spec_version) .await .with_context(|| format!("failed to resolve data source template {}", name))?; @@ -984,7 +1304,7 @@ impl blockchain::DataSourceTemplate for DataSourceTemplate { } } -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UnresolvedMapping { pub kind: String, @@ -997,7 +1317,7 @@ pub struct UnresolvedMapping { #[serde(default)] pub call_handlers: Vec, #[serde(default)] - pub event_handlers: Vec, + pub event_handlers: Vec, pub file: Link, } @@ -1029,8 +1349,10 @@ impl Mapping { .iter() .any(|handler| matches!(handler.filter, Some(BlockHandlerFilter::Call))) } +} - pub fn find_abi(&self, abi_name: &str) -> Result, Error> { +impl FindMappingABI for Mapping { + fn find_abi(&self, abi_name: &str) -> Result, Error> { Ok(self .abis .iter() @@ -1043,8 +1365,10 @@ impl Mapping { impl UnresolvedMapping { pub async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, + spec_version: &semver::Version, ) -> Result { let UnresolvedMapping { kind, @@ -1064,67 +1388,57 @@ impl UnresolvedMapping { // resolve each abi abis.into_iter() .map(|unresolved_abi| async { - Result::<_, Error>::Ok(Arc::new( - unresolved_abi.resolve(resolver, logger).await?, - )) + unresolved_abi + .resolve(deployment_hash, resolver, logger) + .await }) .collect::>() .try_collect::>(), async { - let module_bytes = resolver.cat(logger, &link).await?; + let module_bytes = resolver + .cat(&LinkResolverContext::new(deployment_hash, logger), &link) + .await?; Ok(Arc::new(module_bytes)) }, ) .await .with_context(|| format!("failed to resolve mapping {}", link.link))?; + // Resolve event handlers with ABI context + let resolved_event_handlers = event_handlers + .into_iter() + .map(|unresolved_handler| { + // Find the ABI for this event handler + let (_, abi_json) = abis.first().ok_or_else(|| { + anyhow!( + "No ABI found for event '{}' in event handler '{}'", + unresolved_handler.event, + unresolved_handler.handler + ) + })?; + + unresolved_handler.resolve(abi_json, spec_version) + }) + .collect::, anyhow::Error>>()?; + + // Extract just the MappingABIs for the final Mapping struct + let mapping_abis = abis.into_iter().map(|(abi, _)| Arc::new(abi)).collect(); + Ok(Mapping { kind, api_version, language, entities, - abis, + abis: mapping_abis, block_handlers: block_handlers.clone(), call_handlers: call_handlers.clone(), - event_handlers: event_handlers.clone(), + event_handlers: resolved_event_handlers, runtime, link, }) } } -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct UnresolvedMappingABI { - pub name: String, - pub file: Link, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct MappingABI { - pub name: String, - pub contract: Contract, -} - -impl UnresolvedMappingABI { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let contract_bytes = resolver.cat(logger, &self.file).await.with_context(|| { - format!( - "failed to resolve ABI {} from {}", - self.name, self.file.link - ) - })?; - let contract = Contract::load(&*contract_bytes)?; - Ok(MappingABI { - name: self.name, - contract, - }) - } -} - #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] pub struct MappingBlockHandler { pub handler: String, @@ -1162,34 +1476,121 @@ pub struct MappingCallHandler { pub handler: String, } -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingEventHandler { +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedMappingEventHandler { pub event: String, - pub topic0: Option, + pub topic0: Option, + #[serde(deserialize_with = "deserialize_b256_vec", default)] + pub topic1: Option>, + #[serde(deserialize_with = "deserialize_b256_vec", default)] + pub topic2: Option>, + #[serde(deserialize_with = "deserialize_b256_vec", default)] + pub topic3: Option>, pub handler: String, #[serde(default)] pub receipt: bool, + #[serde(default)] + pub calls: UnresolvedCallDecls, +} + +impl UnresolvedMappingEventHandler { + pub fn resolve( + self, + abi_json: &AbiJson, + spec_version: &semver::Version, + ) -> Result { + let resolved_calls = self + .calls + .resolve(abi_json, Some(&self.event), spec_version)?; + + Ok(MappingEventHandler { + event: self.event, + topic0: self.topic0, + topic1: self.topic1, + topic2: self.topic2, + topic3: self.topic3, + handler: self.handler, + receipt: self.receipt, + calls: resolved_calls, + }) + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub struct MappingEventHandler { + pub event: String, + pub topic0: Option, + pub topic1: Option>, + pub topic2: Option>, + pub topic3: Option>, + pub handler: String, + pub receipt: bool, + pub calls: CallDecls, +} + +// Custom deserializer for B256 fields that removes the '0x' prefix before parsing +fn deserialize_b256_vec<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let s: Option> = Option::deserialize(deserializer)?; + + match s { + Some(vec) => { + let mut b256_vec = Vec::new(); + for hex_str in vec { + // Remove '0x' prefix if present + let clean_hex_str = hex_str.trim_start_matches("0x"); + // Ensure the hex string is 64 characters long, after removing '0x' + let padded_hex_str = format!("{:0>64}", clean_hex_str); + // Parse the padded string into H256, handling potential errors + b256_vec.push( + B256::from_str(&padded_hex_str) + .map_err(|e| D::Error::custom(format!("Failed to parse B256: {}", e)))?, + ); + } + Ok(Some(b256_vec)) + } + None => Ok(None), + } } impl MappingEventHandler { - pub fn topic0(&self) -> H256 { + pub fn topic0(&self) -> B256 { self.topic0 - .unwrap_or_else(|| string_to_h256(&self.event.replace("indexed ", ""))) + .unwrap_or_else(|| string_to_b256(&self.event.replace("indexed ", ""))) + } + + pub fn matches(&self, log: &Log) -> bool { + let matches_topic = |index: usize, topic_opt: &Option>| -> bool { + topic_opt.as_ref().is_none_or(|topic_vec| { + log.topics() + .get(index) + .is_some_and(|log_topic| topic_vec.contains(log_topic)) + }) + }; + + if let Some(topic0) = log.topics().first() { + return self.topic0() == *topic0 + && matches_topic(1, &self.topic1) + && matches_topic(2, &self.topic2) + && matches_topic(3, &self.topic3); + } + + // Logs without topic0 should simply be skipped + false + } + + pub fn has_additional_topics(&self) -> bool { + self.topic1.as_ref().is_some_and(|v| !v.is_empty()) + || self.topic2.as_ref().is_some_and(|v| !v.is_empty()) + || self.topic3.as_ref().is_some_and(|v| !v.is_empty()) } } -/// Hashes a string to a H256 hash. -fn string_to_h256(s: &str) -> H256 { - let mut result = [0u8; 32]; - let data = s.replace(' ', "").into_bytes(); - let mut sponge = Keccak::new_keccak256(); - sponge.update(&data); - sponge.finalize(&mut result); - - // This was deprecated but the replacement seems to not be available in the - // version web3 uses. - #[allow(deprecated)] - H256::from_slice(&result) +/// Hashes a string to a B256 hash. +fn string_to_b256(s: &str) -> B256 { + keccak256(s.replace(' ', "").as_bytes()) } #[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] diff --git a/chain/ethereum/src/env.rs b/chain/ethereum/src/env.rs index c4b0a263049..b15ee0ee018 100644 --- a/chain/ethereum/src/env.rs +++ b/chain/ethereum/src/env.rs @@ -1,6 +1,6 @@ use envconfig::Envconfig; use graph::env::EnvVarBoolean; -use graph::prelude::{envconfig, lazy_static, BlockNumber}; +use graph::prelude::{BlockNumber, envconfig, lazy_static}; use std::fmt; use std::time::Duration; @@ -13,9 +13,9 @@ lazy_static! { pub struct EnvVars { /// Additional deterministic errors that have not yet been hardcoded. /// - /// Set by the environment variable `GRAPH_GETH_ETH_CALL_ERRORS`, separated + /// Set by the environment variable `GRAPH_RPC_ETH_CALL_ERRORS`, separated /// by `;`. - pub geth_eth_call_errors: Vec, + pub rpc_eth_call_errors: Vec, /// Set by the environment variable `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS`. The /// default value is 2000. pub get_logs_max_contracts: usize, @@ -33,6 +33,9 @@ pub struct EnvVars { /// Set by the environment variable `ETHEREUM_BLOCK_BATCH_SIZE`. The /// default value is 10 blocks. pub block_batch_size: usize, + /// Set by the environment variable `ETHEREUM_BLOCK_PTR_BATCH_SIZE`. The + /// default value is 10 blocks. + pub block_ptr_batch_size: usize, /// Maximum number of blocks to request in each chunk. /// /// Set by the environment variable `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE`. @@ -46,6 +49,10 @@ pub struct EnvVars { /// Set by the environment variable `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT` /// (expressed in seconds). The default value is 180s. pub json_rpc_timeout: Duration, + + /// Set by the environment variable `GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT` + /// (expressed in seconds). The default value is 10s. + pub block_receipts_check_timeout: Duration, /// This is used for requests that will not fail the subgraph if the limit /// is reached, but will simply restart the syncing step, so it can be low. /// This limit guards against scenarios such as requesting a block hash that @@ -84,6 +91,10 @@ pub struct EnvVars { /// This is a comma separated list of chain ids for which the gas field will not be set /// when calling `eth_call`. pub eth_call_no_gas: Vec, + /// Set by the flag `GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS`. On by default. + /// When enabled, forces the use of RPC instead of Firehose for loading block pointers by numbers. + /// This is used in composable subgraphs. Firehose can be slow for loading block pointers by numbers. + pub force_rpc_for_block_ptrs: bool, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -103,8 +114,8 @@ impl From for EnvVars { fn from(x: Inner) -> Self { Self { get_logs_max_contracts: x.get_logs_max_contracts, - geth_eth_call_errors: x - .geth_eth_call_errors + rpc_eth_call_errors: x + .rpc_eth_call_errors .split(';') .filter(|s| !s.is_empty()) .map(str::to_string) @@ -112,8 +123,12 @@ impl From for EnvVars { trace_stream_step_size: x.trace_stream_step_size, max_event_only_range: x.max_event_only_range, block_batch_size: x.block_batch_size, + block_ptr_batch_size: x.block_ptr_batch_size, max_block_range_size: x.max_block_range_size, json_rpc_timeout: Duration::from_secs(x.json_rpc_timeout_in_secs), + block_receipts_check_timeout: Duration::from_secs( + x.block_receipts_check_timeout_in_seccs, + ), request_retries: x.request_retries, block_ingestor_max_concurrent_json_rpc_calls: x .block_ingestor_max_concurrent_json_rpc_calls, @@ -130,6 +145,7 @@ impl From for EnvVars { .filter(|s| !s.is_empty()) .map(str::to_string) .collect(), + force_rpc_for_block_ptrs: x.force_rpc_for_block_ptrs.0, } } } @@ -142,8 +158,8 @@ impl Default for EnvVars { #[derive(Clone, Debug, Envconfig)] struct Inner { - #[envconfig(from = "GRAPH_GETH_ETH_CALL_ERRORS", default = "")] - geth_eth_call_errors: String, + #[envconfig(from = "GRAPH_RPC_ETH_CALL_ERRORS", default = "")] + rpc_eth_call_errors: String, #[envconfig(from = "GRAPH_ETH_GET_LOGS_MAX_CONTRACTS", default = "2000")] get_logs_max_contracts: usize, @@ -153,10 +169,14 @@ struct Inner { max_event_only_range: BlockNumber, #[envconfig(from = "ETHEREUM_BLOCK_BATCH_SIZE", default = "10")] block_batch_size: usize, + #[envconfig(from = "ETHEREUM_BLOCK_PTR_BATCH_SIZE", default = "100")] + block_ptr_batch_size: usize, #[envconfig(from = "GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE", default = "2000")] max_block_range_size: BlockNumber, #[envconfig(from = "GRAPH_ETHEREUM_JSON_RPC_TIMEOUT", default = "180")] json_rpc_timeout_in_secs: u64, + #[envconfig(from = "GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT", default = "10")] + block_receipts_check_timeout_in_seccs: u64, #[envconfig(from = "GRAPH_ETHEREUM_REQUEST_RETRIES", default = "10")] request_retries: usize, #[envconfig( @@ -175,6 +195,8 @@ struct Inner { target_triggers_per_block_range: u64, #[envconfig(from = "GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER", default = "0")] genesis_block_number: u64, - #[envconfig(from = "GRAPH_ETH_CALL_NO_GAS", default = "421613")] + #[envconfig(from = "GRAPH_ETH_CALL_NO_GAS", default = "421613,421614")] eth_call_no_gas: String, + #[envconfig(from = "GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS", default = "true")] + force_rpc_for_block_ptrs: EnvVarBoolean, } diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index b699b7370db..9c85f0ac551 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -1,39 +1,56 @@ -use futures::future; -use futures::prelude::*; +use async_trait::async_trait; use futures03::{future::BoxFuture, stream::FuturesUnordered}; -use graph::blockchain::client::ChainClient; +use graph::abi; +use graph::abi::DynSolValueExt; +use graph::abi::FunctionExt; use graph::blockchain::BlockHash; use graph::blockchain::ChainIdentifier; +use graph::blockchain::ExtendedBlockPtr; +use graph::blockchain::client::ChainClient; +use graph::components::ethereum::*; use graph::components::transaction_receipt::LightTransactionReceipt; -use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::data::store::ethereum::call; +use graph::data::store::scalar; use graph::data::subgraph::API_VERSION_0_0_7; -use graph::prelude::ethabi::ParamType; -use graph::prelude::ethabi::Token; -use graph::prelude::tokio::try_join; -use graph::slog::o; -use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr, IngestorError}, - prelude::{ - anyhow::{self, anyhow, bail, ensure, Context}, - async_trait, debug, error, ethabi, - futures03::{self, compat::Future01CompatExt, FutureExt, StreamExt, TryStreamExt}, - hex, info, retry, serde_json as json, stream, tiny_keccak, trace, warn, - web3::{ - self, - types::{ - Address, BlockId, BlockNumber as Web3BlockNumber, Bytes, CallRequest, Filter, - FilterBuilder, Log, Transaction, TransactionReceipt, H256, +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::data_source::common::ContractCall; +use graph::derive::CheapClone; +use graph::futures01::Future; +use graph::futures01::Stream; +use graph::futures01::stream; +use graph::futures03::future::try_join_all; +use graph::futures03::{ + self, FutureExt, StreamExt, TryFutureExt, TryStreamExt, compat::Future01CompatExt, +}; +use graph::prelude::{ + alloy::{ + self, + network::TransactionResponse, + primitives::{Address, B256}, + providers::{ + Identity, Provider, RootProvider, + ext::TraceApi, + fillers::{ + BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, }, }, - BlockNumber, ChainStore, CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, - TimeoutError, TryFutureExt, + rpc::types::{ + TransactionInput, TransactionRequest, + trace::{filter::TraceFilter as AlloyTraceFilter, parity::LocalizedTransactionTrace}, + }, + transports::{RpcError, TransportErrorKind}, }, + tokio::try_join, }; +use graph::slog::o; use graph::{ - components::ethereum::*, - prelude::web3::api::Web3, - prelude::web3::transports::Batch, - prelude::web3::types::{Trace, TraceFilter, TraceFilterBuilder, H160}, + blockchain::{BlockPtr, IngestorError, block_stream::BlockWithTriggers}, + prelude::{ + BlockNumber, ChainStore, CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, + TimeoutError, + anyhow::{self, Context, anyhow, bail, ensure}, + debug, error, hex, info, retry, trace, warn, + }, }; use itertools::Itertools; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -41,32 +58,64 @@ use std::convert::TryFrom; use std::iter::FromIterator; use std::pin::Pin; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; +use tokio::time::timeout; -use crate::adapter::ProviderStatus; -use crate::chain::BlockFinality; -use crate::trigger::LogRef; use crate::Chain; use crate::NodeCapabilities; +use crate::TriggerFilter; +use crate::adapter::EthGetLogsFilter; +use crate::adapter::EthereumRpcError; +use crate::adapter::ProviderStatus; +use crate::call_helper::interpret_eth_call_error; +use crate::chain::BlockFinality; +use crate::chain::ChainSettings; +use crate::trigger::{LogPosition, LogRef}; use crate::{ + ENV_VARS, adapter::{ - EthGetLogsFilter, EthereumAdapter as EthereumAdapterTrait, EthereumBlockFilter, - EthereumCallFilter, EthereumContractCall, EthereumContractCallError, EthereumLogFilter, - ProviderEthRpcMetrics, SubgraphEthRpcMetrics, + ContractCallError, EthereumAdapter as EthereumAdapterTrait, EthereumBlockFilter, + EthereumCallFilter, EthereumLogFilter, ProviderEthRpcMetrics, SubgraphEthRpcMetrics, }, transport::Transport, trigger::{EthereumBlockTriggerType, EthereumTrigger}, - TriggerFilter, ENV_VARS, }; -#[derive(Debug, Clone)] +type AlloyProvider = FillProvider< + JoinFill< + Identity, + JoinFill>>, + >, + RootProvider, + AnyNetworkBare, +>; + +#[derive(Clone)] pub struct EthereumAdapter { logger: Logger, provider: String, - web3: Arc>, + alloy: Arc, metrics: Arc, supports_eip_1898: bool, call_only: bool, + supports_block_receipts: Arc>>, + pub(crate) settings: Arc, +} + +impl std::fmt::Debug for EthereumAdapter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EthereumAdapter") + .field("logger", &self.logger) + .field("provider", &self.provider) + .field("alloy", &"") + .field("metrics", &self.metrics) + .field("supports_eip_1898", &self.supports_eip_1898) + .field("call_only", &self.call_only) + .field("supports_block_receipts", &self.supports_block_receipts) + .field("settings", &self.settings) + .finish() + } } impl CheapClone for EthereumAdapter { @@ -74,19 +123,44 @@ impl CheapClone for EthereumAdapter { Self { logger: self.logger.clone(), provider: self.provider.clone(), - web3: self.web3.cheap_clone(), + alloy: self.alloy.clone(), metrics: self.metrics.cheap_clone(), supports_eip_1898: self.supports_eip_1898, call_only: self.call_only, + supports_block_receipts: self.supports_block_receipts.cheap_clone(), + settings: self.settings.clone(), } } } +/// Logger with provider context. Internal methods should take this type +/// to ensure provider info is always included in logs. +#[derive(Clone, CheapClone)] +struct ProviderLogger(Logger); + +impl ProviderLogger { + fn new(logger: &Logger, provider: &str) -> Self { + Self(Logger::new(logger, o!("provider" => provider.to_string()))) + } +} + +impl std::ops::Deref for ProviderLogger { + type Target = Logger; + fn deref(&self) -> &Logger { + &self.0 + } +} + impl EthereumAdapter { pub fn is_call_only(&self) -> bool { self.call_only } + /// Returns a logger with provider context added + fn provider_logger(&self, logger: &Logger) -> ProviderLogger { + ProviderLogger::new(logger, &self.provider) + } + pub async fn new( logger: Logger, provider: String, @@ -94,106 +168,68 @@ impl EthereumAdapter { provider_metrics: Arc, supports_eip_1898: bool, call_only: bool, + settings: Arc, ) -> Self { - let web3 = Arc::new(Web3::new(transport)); - - // Use the client version to check if it is ganache. For compatibility with unit tests, be - // are lenient with errors, defaulting to false. - let is_ganache = web3 - .web3() - .client_version() - .await - .map(|s| s.contains("TestRPC")) - .unwrap_or(false); + let alloy = match &transport { + Transport::RPC(client) => Arc::new( + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_client(client.clone()), + ), + Transport::IPC(ipc_connect) => Arc::new( + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_ipc(ipc_connect.clone()) + .await + .expect("Failed to connect to Ethereum IPC"), + ), + Transport::WS(ws_connect) => Arc::new( + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_ws(ws_connect.clone()) + .await + .expect("Failed to connect to Ethereum WS"), + ), + }; EthereumAdapter { logger, provider, - web3, + alloy, metrics: provider_metrics, - supports_eip_1898: supports_eip_1898 && !is_ganache, + supports_eip_1898, call_only, + supports_block_receipts: Arc::new(RwLock::new(None)), + settings, } } async fn traces( self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, - addresses: Vec, - ) -> Result, Error> { + addresses: Vec
, + ) -> Result, Error> { assert!(!self.call_only); - let eth = self.clone(); let retry_log_message = format!("trace_filter RPC call for block range: [{}..{}]", from, to); + let eth = self.clone(); + retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .redact_log_urls(true) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { - let trace_filter: TraceFilter = match addresses.len() { - 0 => TraceFilterBuilder::default() - .from_block(from.into()) - .to_block(to.into()) - .build(), - _ => TraceFilterBuilder::default() - .from_block(from.into()) - .to_block(to.into()) - .to_address(addresses.clone()) - .build(), - }; - - let eth = eth.cheap_clone(); - let logger_for_triggers = logger.clone(); - let logger_for_error = logger.clone(); - let start = Instant::now(); + let eth = eth.clone(); + let logger = logger.clone(); let subgraph_metrics = subgraph_metrics.clone(); - let provider_metrics = eth.metrics.clone(); - let provider = self.provider.clone(); - + let addresses = addresses.clone(); async move { - let result = eth - .web3 - .trace() - .filter(trace_filter) + eth.execute_trace_filter_request(logger, subgraph_metrics, from, to, addresses) .await - .map(move |traces| { - if !traces.is_empty() { - if to == from { - debug!( - logger_for_triggers, - "Received {} traces for block {}", - traces.len(), - to - ); - } else { - debug!( - logger_for_triggers, - "Received {} traces for blocks [{}, {}]", - traces.len(), - from, - to - ); - } - } - traces - }) - .map_err(Error::from); - - let elapsed = start.elapsed().as_secs_f64(); - provider_metrics.observe_request(elapsed, "trace_filter", &provider); - subgraph_metrics.observe_request(elapsed, "trace_filter", &provider); - if let Err(e) = &result { - provider_metrics.add_error("trace_filter", &provider); - subgraph_metrics.add_error("trace_filter", &provider); - debug!( - logger_for_error, - "Error querying traces error = {:#} from = {} to = {}", e, from, to - ); - } - result } }) .map_err(move |e| { @@ -209,28 +245,177 @@ impl EthereumAdapter { .await } + async fn execute_trace_filter_request( + &self, + logger: ProviderLogger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + addresses: Vec
, + ) -> Result, Error> { + let alloy_trace_filter = Self::build_trace_filter(from, to, &addresses); + let start = Instant::now(); + + let result = self.alloy.trace_filter(&alloy_trace_filter).await; + + if let Ok(traces) = &result { + self.log_trace_results(&logger, from, to, traces.len()); + } + + self.record_trace_metrics( + &subgraph_metrics, + start.elapsed().as_secs_f64(), + &result, + from, + to, + &logger, + ); + + result.map_err(Error::from) + } + + fn build_trace_filter( + from: BlockNumber, + to: BlockNumber, + addresses: &[Address], + ) -> AlloyTraceFilter { + let filter = AlloyTraceFilter::default() + .from_block(from as u64) + .to_block(to as u64); + + if !addresses.is_empty() { + filter.to_address(addresses.to_vec()) + } else { + filter + } + } + + fn log_trace_results( + &self, + logger: &ProviderLogger, + from: BlockNumber, + to: BlockNumber, + trace_len: usize, + ) { + if trace_len > 0 { + if to == from { + debug!(logger, "Received {} traces for block {}", trace_len, to); + } else { + debug!( + logger, + "Received {} traces for blocks [{}, {}]", trace_len, from, to + ); + } + } + } + + fn record_trace_metrics( + &self, + subgraph_metrics: &Arc, + elapsed: f64, + result: &Result, RpcError>, + from: BlockNumber, + to: BlockNumber, + logger: &ProviderLogger, + ) { + self.metrics + .observe_request(elapsed, "trace_filter", &self.provider); + subgraph_metrics.observe_request(elapsed, "trace_filter", &self.provider); + + if let Err(e) = result { + self.metrics.add_error("trace_filter", &self.provider); + subgraph_metrics.add_error("trace_filter", &self.provider); + debug!( + logger, + "Error querying traces error = {:#} from = {} to = {}", e, from, to + ); + } + } + + // This is a lazy check for block receipt support. It is only called once and then the result is + // cached. The result is not used for anything critical, so it is fine to be lazy. + async fn check_block_receipt_support_and_update_cache( + &self, + alloy: Arc, + block_hash: B256, + supports_eip_1898: bool, + call_only: bool, + logger: ProviderLogger, + ) -> bool { + // This is the lazy part. If the result is already in `supports_block_receipts`, we don't need + // to check again. + { + let supports_block_receipts = self.supports_block_receipts.read().await; + if let Some(supports_block_receipts) = *supports_block_receipts { + return supports_block_receipts; + } + } + + info!(logger, "Checking eth_getBlockReceipts support"); + let result = timeout( + ENV_VARS.block_receipts_check_timeout, + check_block_receipt_support(alloy, block_hash, supports_eip_1898, call_only), + ) + .await; + + let result = match result { + Ok(Ok(_)) => { + info!(logger, "Provider supports block receipts"); + true + } + Ok(Err(err)) => { + warn!(logger, "Skipping use of block receipts, reason: {}", err); + false + } + Err(_) => { + warn!( + logger, + "Skipping use of block receipts, reason: Timeout after {} seconds", + ENV_VARS.block_receipts_check_timeout.as_secs() + ); + false + } + }; + + // We set the result in `self.supports_block_receipts` so that the next time this function is called, we don't + // need to check again. + let mut supports_block_receipts = self.supports_block_receipts.write().await; + if supports_block_receipts.is_none() { + *supports_block_receipts = Some(result); + } + + result + } + + /// Alloy-exclusive version of logs_with_sigs using alloy types and methods async fn logs_with_sigs( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, filter: Arc, too_many_logs_fingerprints: &'static [&'static str], - ) -> Result, TimeoutError> { + ) -> Result< + Vec, + TimeoutError>, + > { assert!(!self.call_only); let eth_adapter = self.clone(); let retry_log_message = format!("eth_getLogs RPC call for block range: [{}..{}]", from, to); retry(retry_log_message, &logger) - .when(move |res: &Result<_, web3::error::Error>| match res { - Ok(_) => false, - Err(e) => !too_many_logs_fingerprints - .iter() - .any(|f| e.to_string().contains(f)), - }) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .redact_log_urls(true) + .when( + move |res: &Result<_, RpcError>| match res { + Ok(_) => false, + Err(e) => !too_many_logs_fingerprints + .iter() + .any(|f| e.to_string().contains(f)), + }, + ) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let eth_adapter = eth_adapter.cheap_clone(); let subgraph_metrics = subgraph_metrics.clone(); @@ -241,16 +426,9 @@ impl EthereumAdapter { async move { let start = Instant::now(); - // Create a log filter - let log_filter: Filter = FilterBuilder::default() - .from_block(from.into()) - .to_block(to.into()) - .address(filter.contracts.clone()) - .topics(Some(filter.event_signatures.clone()), None, None, None) - .build(); + let alloy_filter = filter.to_alloy_filter(from, to); - // Request logs from client - let result = eth_adapter.web3.eth().logs(log_filter).boxed().await; + let result = eth_adapter.alloy.get_logs(&alloy_filter).await; let elapsed = start.elapsed().as_secs_f64(); provider_metrics.observe_request(elapsed, "eth_getLogs", &provider); subgraph_metrics.observe_request(elapsed, "eth_getLogs", &provider); @@ -266,12 +444,12 @@ impl EthereumAdapter { fn trace_stream( self, - logger: &Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, - addresses: Vec, - ) -> impl Stream + Send { + addresses: Vec
, + ) -> impl futures03::Stream> + Send { if from > to { panic!( "Can not produce a call stream on a backwards block range: from = {}, to = {}", @@ -285,52 +463,57 @@ impl EthereumAdapter { true => 1, }; - let eth = self; - let logger = logger.clone(); - stream::unfold(from, move |start| { - if start > to { - return None; + let ranges: Vec<(BlockNumber, BlockNumber)> = { + let mut ranges = Vec::new(); + let mut start = from; + while start <= to { + let end = (start + step_size - 1).min(to); + ranges.push((start, end)); + start = end + 1; } - let end = (start + step_size - 1).min(to); - let new_start = end + 1; - if start == end { - debug!(logger, "Requesting traces for block {}", start); - } else { - debug!(logger, "Requesting traces for blocks [{}, {}]", start, end); + ranges + }; + + let block_batch_size = self.settings.block_batch_size; + let eth = self; + + futures03::stream::iter(ranges.into_iter().map(move |(start, end)| { + let eth = eth.clone(); + let logger = logger.clone(); + let subgraph_metrics = subgraph_metrics.clone(); + let addresses = addresses.clone(); + + async move { + if start == end { + debug!(logger, "Requesting traces for block {}", start); + } else { + debug!(logger, "Requesting traces for blocks [{}, {}]", start, end); + } + + eth.traces(logger, subgraph_metrics, start, end, addresses) + .await } - Some(futures::future::ok(( - eth.clone() - .traces( - logger.cheap_clone(), - subgraph_metrics.clone(), - start, - end, - addresses.clone(), - ) - .boxed() - .compat(), - new_start, - ))) - }) - .buffered(ENV_VARS.block_batch_size) - .map(stream::iter_ok) - .flatten() + })) + .buffered(block_batch_size) + .map_ok(|traces| futures03::stream::iter(traces.into_iter().map(Ok))) + .try_flatten() } fn log_stream( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, filter: EthGetLogsFilter, - ) -> DynTryFuture<'static, Vec, Error> { + ) -> DynTryFuture<'static, Vec, Error> { // Codes returned by Ethereum node providers if an eth_getLogs request is too heavy. - // The first one is for Infura when it hits the log limit, the rest for Alchemy timeouts. const TOO_MANY_LOGS_FINGERPRINTS: &[&str] = &[ - "ServerError(-32005)", - "503 Service Unavailable", - "ServerError(-32000)", + "ServerError(-32005)", // Infura + "503 Service Unavailable", // Alchemy + "ServerError(-32000)", // Alchemy + "Try with this block range", // zKSync era + "block range too large", // Monad ]; if from > to { @@ -347,7 +530,7 @@ impl EthereumAdapter { let step = match filter.contracts.is_empty() { // `to - from + 1` blocks will be scanned. false => to - from, - true => (to - from).min(ENV_VARS.max_event_only_range - 1), + true => (to - from).min(self.settings.max_event_only_range - 1), }; // Typically this will loop only once and fetch the entire range in one request. But if the @@ -410,194 +593,257 @@ impl EthereumAdapter { .boxed() } - fn call( + fn block_ptr_to_id(&self, block_ptr: &BlockPtr) -> alloy::rpc::types::BlockId { + if !self.supports_eip_1898 { + alloy::rpc::types::BlockId::number(block_ptr.number as u64) + } else { + alloy::rpc::types::BlockId::hash(block_ptr.hash.as_b256()) + } + } + + async fn code( &self, - logger: Logger, - contract_address: Address, - call_data: Bytes, + logger: ProviderLogger, + address: Address, block_ptr: BlockPtr, - gas: Option, - ) -> impl Future + Send { - let web3 = self.web3.clone(); - let logger = Logger::new(&logger, o!("provider" => self.provider.clone())); - - // Ganache does not support calls by block hash. - // See https://github.com/trufflesuite/ganache-cli/issues/973 - let block_id = if !self.supports_eip_1898 { - BlockId::Number(block_ptr.number.into()) - } else { - BlockId::Hash(block_ptr.hash_as_h256()) - }; - let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); + ) -> Result { + let alloy = self.alloy.clone(); + + let block_id = self.block_ptr_to_id(&block_ptr); + let retry_log_message = format!("eth_getCode RPC call for block {}", block_ptr); + retry(retry_log_message, &logger) - .when(|result| match result { - Ok(_) | Err(EthereumContractCallError::Revert(_)) => false, - Err(_) => true, - }) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .redact_log_urls(true) + .when(|result| result.is_err()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { - let call_data = call_data.clone(); - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); async move { - let req = CallRequest { - to: Some(contract_address), - gas: gas.map(|val| web3::types::U256::from(val)), - data: Some(call_data.clone()), - from: None, - gas_price: None, - value: None, - access_list: None, - max_fee_per_gas: None, - max_priority_fee_per_gas: None, - transaction_type: None, - }; - let result = web3.eth().call(req, Some(block_id)).boxed().await; - - // Try to check if the call was reverted. The JSON-RPC response for reverts is - // not standardized, so we have ad-hoc checks for each Ethereum client. - - // 0xfe is the "designated bad instruction" of the EVM, and Solidity uses it for - // asserts. - const PARITY_BAD_INSTRUCTION_FE: &str = "Bad instruction fe"; - - // 0xfd is REVERT, but on some contracts, and only on older blocks, - // this happens. Makes sense to consider it a revert as well. - const PARITY_BAD_INSTRUCTION_FD: &str = "Bad instruction fd"; - - const PARITY_BAD_JUMP_PREFIX: &str = "Bad jump"; - const PARITY_STACK_LIMIT_PREFIX: &str = "Out of stack"; - - // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61. - const PARITY_OUT_OF_GAS: &str = "Out of gas"; - - const PARITY_VM_EXECUTION_ERROR: i64 = -32015; - const PARITY_REVERT_PREFIX: &str = "Reverted 0x"; - const XDAI_REVERT: &str = "revert"; - - // Deterministic Geth execution errors. We might need to expand this as - // subgraphs come across other errors. See - // https://github.com/ethereum/go-ethereum/blob/cd57d5cd38ef692de8fbedaa56598b4e9fbfbabc/core/vm/errors.go - const GETH_EXECUTION_ERRORS: &[&str] = &[ - // The "revert" substring covers a few known error messages, including: - // Hardhat: "error: transaction reverted", - // Ganache and Moonbeam: "vm exception while processing transaction: revert", - // Geth: "execution reverted" - // And others. - "revert", - "invalid jump destination", - "invalid opcode", - // Ethereum says 1024 is the stack sizes limit, so this is deterministic. - "stack limit reached 1024", - // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61 for why the gas limit is considered deterministic. - "out of gas", - "stack underflow", - ]; - - let env_geth_call_errors = ENV_VARS.geth_eth_call_errors.iter(); - let mut geth_execution_errors = GETH_EXECUTION_ERRORS - .iter() - .copied() - .chain(env_geth_call_errors.map(|s| s.as_str())); - - let as_solidity_revert_with_reason = |bytes: &[u8]| { - let solidity_revert_function_selector = - &tiny_keccak::keccak256(b"Error(string)")[..4]; - - match bytes.len() >= 4 && &bytes[..4] == solidity_revert_function_selector { - false => None, - true => ethabi::decode(&[ParamType::String], &bytes[4..]) - .ok() - .and_then(|tokens| tokens[0].clone().into_string()), - } - }; + let result = alloy.get_code_at(address).block_id(block_id).await; + match result { + Ok(code) => Ok(code), + Err(err) => Err(EthereumRpcError::AlloyError(err)), + } + } + }) + .await + .map_err(|e| e.into_inner().unwrap_or(EthereumRpcError::Timeout)) + } + + async fn balance( + &self, + logger: ProviderLogger, + address: Address, + block_ptr: BlockPtr, + ) -> Result { + let alloy = self.alloy.clone(); + let block_id = self.block_ptr_to_id(&block_ptr); + let retry_log_message = format!("eth_getBalance RPC call for block {}", block_ptr); + + retry(retry_log_message, &logger) + .redact_log_urls(true) + .when(|result| result.is_err()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + let result = alloy.get_balance(address).block_id(block_id).await; match result { - // A successful response. - Ok(bytes) => Ok(bytes), + Ok(balance) => Ok(balance), + Err(err) => Err(EthereumRpcError::AlloyError(err)), + } + } + }) + .await + .map_err(|e| e.into_inner().unwrap_or(EthereumRpcError::Timeout)) + } - // Check for Geth revert. - Err(web3::Error::Rpc(rpc_error)) - if geth_execution_errors - .any(|e| rpc_error.message.to_lowercase().contains(e)) => - { - Err(EthereumContractCallError::Revert(rpc_error.message)) - } + async fn call( + &self, + logger: ProviderLogger, + call_data: call::Request, + block_ptr: BlockPtr, + gas: Option, + ) -> Result { + let alloy = self.alloy.clone(); - // Check for Parity revert. - Err(web3::Error::Rpc(ref rpc_error)) - if rpc_error.code.code() == PARITY_VM_EXECUTION_ERROR => - { - match rpc_error.data.as_ref().and_then(|d| d.as_str()) { - Some(data) - if data.starts_with(PARITY_REVERT_PREFIX) - || data.starts_with(PARITY_BAD_JUMP_PREFIX) - || data.starts_with(PARITY_STACK_LIMIT_PREFIX) - || data == PARITY_BAD_INSTRUCTION_FE - || data == PARITY_BAD_INSTRUCTION_FD - || data == PARITY_OUT_OF_GAS - || data == XDAI_REVERT => - { - let reason = if data == PARITY_BAD_INSTRUCTION_FE { - PARITY_BAD_INSTRUCTION_FE.to_owned() - } else { - let payload = data.trim_start_matches(PARITY_REVERT_PREFIX); - hex::decode(payload) - .ok() - .and_then(|payload| { - as_solidity_revert_with_reason(&payload) - }) - .unwrap_or("no reason".to_owned()) - }; - Err(EthereumContractCallError::Revert(reason)) - } + let alloy_block_id = self.block_ptr_to_id(&block_ptr); + let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); + retry(retry_log_message, &logger) + .redact_log_urls(true) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let call_data = call_data.clone(); + let alloy = alloy.cheap_clone(); + let logger = logger.cheap_clone(); + async move { + let mut req = TransactionRequest::default() + .input(TransactionInput::both(alloy::primitives::Bytes::from( + call_data.encoded_call.to_vec(), + ))) + .to(call_data.address); - // The VM execution error was not identified as a revert. - _ => Err(EthereumContractCallError::Web3Error(web3::Error::Rpc( - rpc_error.clone(), - ))), - } - } + if let Some(gas) = gas { + req = req.gas_limit(gas as u64); + } - // The error was not identified as a revert. - Err(err) => Err(EthereumContractCallError::Web3Error(err)), + let result = alloy.call(req.into()).block(alloy_block_id).await; + + match result { + Ok(bytes) => Ok(call::Retval::Value(scalar::Bytes::from(bytes))), + Err(err) => interpret_eth_call_error(&logger, err), } } }) - .map_err(|e| e.into_inner().unwrap_or(EthereumContractCallError::Timeout)) - .boxed() - .compat() + .await + .map_err(|e| e.into_inner().unwrap_or(ContractCallError::Timeout)) } + async fn call_and_cache( + &self, + logger: &ProviderLogger, + call: &ContractCall, + req: call::Request, + cache: Arc, + ) -> Result { + let result = self + .call( + logger.clone(), + req.cheap_clone(), + call.block_ptr.clone(), + call.gas, + ) + .await?; + if let Err(e) = cache + .set_call( + logger, + req.cheap_clone(), + call.block_ptr.cheap_clone(), + result.clone(), + ) + .await + { + error!(logger, "EthereumAdapter: call cache set error"; + "contract_address" => format!("{:?}", req.address), + "error" => e.to_string()); + } + + Ok(req.response(result, call::Source::Rpc)) + } /// Request blocks by hash through JSON-RPC. fn load_blocks_rpc( &self, logger: Logger, - ids: Vec, - ) -> impl Stream, Error = Error> + Send { - let web3 = self.web3.clone(); - - stream::iter_ok::<_, Error>(ids.into_iter().map(move |hash| { - let web3 = web3.clone(); - retry(format!("load block {}", hash), &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - Box::pin(web3.eth().block_with_txs(BlockId::Hash(hash))) - .compat() - .from_err::() - .and_then(move |block| { - block.map(Arc::new).ok_or_else(|| { - anyhow::anyhow!("Ethereum node did not find block {:?}", hash) - }) + ids: Vec, + ) -> impl futures03::Stream, Error>> + Send { + let alloy = self.alloy.clone(); + let request_retries = self.settings.request_retries; + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); + + futures03::stream::iter(ids.into_iter().map(move |hash| { + let alloy = alloy.clone(); + let logger = logger.clone(); + + async move { + retry(format!("load block {}", hash), &logger) + .redact_log_urls(true) + .limit(request_retries) + .timeout_secs(json_rpc_timeout_secs) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + alloy + .get_block_by_hash(hash) + .full() + .await + .map_err(Error::from) + .and_then(|block| { + block + .map(|b| Arc::new(LightEthereumBlock::new(b))) + .ok_or_else(|| { + anyhow::anyhow!( + "Ethereum node did not find block {:?}", + hash + ) + }) + }) + } + }) + .await + .map_err(|e| { + e.into_inner().unwrap_or_else(|| { + anyhow::anyhow!("Ethereum node took too long to return block {}", hash) }) - .compat() - }) - .boxed() - .compat() - .from_err() + }) + } })) - .buffered(ENV_VARS.block_batch_size) + .buffered(self.settings.block_batch_size) + } + + /// Request blocks by number through JSON-RPC. + pub fn load_block_ptrs_by_numbers_rpc( + &self, + logger: Logger, + numbers: Vec, + ) -> impl futures03::Stream, Error>> + Send { + let alloy = self.alloy.clone(); + let request_retries = self.settings.request_retries; + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); + + futures03::stream::iter(numbers.into_iter().map(move |number| { + let alloy = alloy.clone(); + let logger = logger.clone(); + + async move { + retry(format!("load block {}", number), &logger) + .redact_log_urls(true) + .limit(request_retries) + .timeout_secs(json_rpc_timeout_secs) + .run(move || { + let alloy = alloy.cheap_clone(); + + async move { + let block_result = alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + number as u64, + )) + .await; + + match block_result { + Ok(Some(block)) => { + let ptr = ExtendedBlockPtr::try_from(( + block.header.hash, + i32::try_from(block.header.number).unwrap(), + block.header.parent_hash, + block.header.timestamp, + )) + .map_err(|e| { + anyhow::anyhow!("Failed to convert block: {}", e) + })?; + Ok(Arc::new(ptr)) + } + Ok(None) => Err(anyhow::anyhow!( + "Ethereum node did not find block with number {:?}", + number + )), + Err(e) => Err(anyhow::anyhow!("Failed to fetch block: {}", e)), + } + } + }) + .await + .map_err(|e| match e { + TimeoutError::Elapsed => { + anyhow::anyhow!("Timeout while fetching block {}", number) + } + TimeoutError::Inner(e) => e, + }) + } + })) + .buffered(self.settings.block_ptr_batch_size) } /// Request blocks ptrs for numbers through JSON-RPC. @@ -608,20 +854,23 @@ impl EthereumAdapter { logger: Logger, block_nums: Vec, ) -> impl Stream + Send { - let web3 = self.web3.clone(); + let alloy = self.alloy.clone(); + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); stream::iter_ok::<_, Error>(block_nums.into_iter().map(move |block_num| { - let web3 = web3.clone(); + let alloy = alloy.clone(); retry(format!("load block ptr {}", block_num), &logger) + .redact_log_urls(true) + .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(json_rpc_timeout_secs) .run(move || { - let web3 = web3.clone(); + let alloy = alloy.cheap_clone(); async move { - let block = web3 - .eth() - .block(BlockId::Number(Web3BlockNumber::Number(block_num.into()))) - .boxed() + let block = alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + block_num as u64, + )) .await?; block.ok_or_else(|| { @@ -632,9 +881,17 @@ impl EthereumAdapter { .boxed() .compat() .from_err() + .then(|res| { + if detect_null_block(&res) { + Ok(None) + } else { + Some(res).transpose() + } + }) })) - .buffered(ENV_VARS.block_batch_size) - .map(|b| b.into()) + .buffered(self.settings.block_batch_size) + .filter_map(|b| b) + .map(|b| BlockPtr::from((b.header.hash, b.header.number))) } /// Check if `block_ptr` refers to a block that is on the main chain, according to the Ethereum @@ -652,13 +909,12 @@ impl EthereumAdapter { logger: &Logger, block_ptr: BlockPtr, ) -> Result { - let block_hash = self - .block_hash_by_block_number(logger, block_ptr.number) - .compat() + // TODO: This considers null blocks, but we could instead bail if we encounter one as a + // small optimization. + let canonical_block = self + .next_existing_ptr_to_number(logger, block_ptr.number) .await?; - block_hash - .ok_or_else(|| anyhow!("Ethereum node is missing block #{}", block_ptr.number)) - .map(|block_hash| block_hash == block_ptr.hash_as_h256()) + Ok(canonical_block == block_ptr) } pub(crate) fn logs_in_block_range( @@ -668,21 +924,26 @@ impl EthereumAdapter { from: BlockNumber, to: BlockNumber, log_filter: EthereumLogFilter, - ) -> DynTryFuture<'static, Vec, Error> { + ) -> DynTryFuture<'static, Vec, Error> { let eth: Self = self.cheap_clone(); - let logger = logger.clone(); - - futures03::stream::iter(log_filter.eth_get_logs_filters().map(move |filter| { - eth.cheap_clone().log_stream( - logger.cheap_clone(), - subgraph_metrics.cheap_clone(), - from, - to, - filter, - ) - })) + let logger = self.provider_logger(logger); + + let max_contracts = eth.settings.get_logs_max_contracts; + futures03::stream::iter( + log_filter + .eth_get_logs_filters(max_contracts) + .map(move |filter| { + eth.cheap_clone().log_stream( + logger.cheap_clone(), + subgraph_metrics.cheap_clone(), + from, + to, + filter, + ) + }), + ) // Real limits on the number of parallel requests are imposed within the adapter. - .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls) + .buffered(self.settings.block_ingestor_max_concurrent_json_rpc_calls) .try_concat() .boxed() } @@ -696,19 +957,20 @@ impl EthereumAdapter { call_filter: &'a EthereumCallFilter, ) -> Box + Send + 'a> { let eth = self.clone(); + let logger = self.provider_logger(logger); let EthereumCallFilter { contract_addresses_function_signatures, wildcard_signatures, } = call_filter; - let mut addresses: Vec = contract_addresses_function_signatures + let mut addresses: Vec
= contract_addresses_function_signatures .iter() .filter(|(_addr, (start_block, _fsigs))| start_block <= &to) .map(|(addr, (_start_block, _fsigs))| *addr) - .collect::>() + .collect::>() .into_iter() - .collect::>(); + .collect::>(); if addresses.is_empty() && wildcard_signatures.is_empty() { // The filter has no started data sources in the requested range, nothing to do. @@ -725,14 +987,13 @@ impl EthereumAdapter { Box::new( eth.trace_stream(logger, subgraph_metrics, from, to, addresses) - .filter_map(|trace| EthereumCall::try_from_trace(&trace)) - .filter(move |call| { - // `trace_filter` can only filter by calls `to` an address and - // a block range. Since subgraphs are subscribing to calls - // for a specific contract function an additional filter needs - // to be applied - call_filter.matches(call) - }), + .try_filter_map(move |trace| { + let maybe_call = EthereumCall::try_from_trace(&trace) + .filter(|call| call_filter.matches(call)); + futures03::future::ready(Ok(maybe_call)) + }) + .boxed() + .compat(), ) } @@ -751,54 +1012,38 @@ impl EthereumAdapter { ) -> Pin< Box< dyn std::future::Future, anyhow::Error>> - + std::marker::Send, + + std::marker::Send + + '_, >, > { - // Create a HashMap of block numbers to Vec + // Create a HashMap of block numbers to Vec. let matching_blocks = (from..=to) .filter_map(|block_number| { - filter - .polling_intervals - .iter() - .find_map(|(start_block, interval)| { - let has_once_trigger = (*interval == 0) && (block_number == *start_block); - let has_polling_trigger = block_number >= *start_block - && *interval > 0 - && ((block_number - start_block) % *interval) == 0; - - if has_once_trigger || has_polling_trigger { - let mut triggers = Vec::new(); - if has_once_trigger { - triggers.push(EthereumBlockTriggerType::Start); - } - if has_polling_trigger { - triggers.push(EthereumBlockTriggerType::End); - } - Some((block_number, triggers)) - } else { - None - } - }) + let triggers = + block_trigger_types_from_intervals(block_number, &filter.polling_intervals); + if triggers.is_empty() { + None + } else { + Some((block_number, triggers)) + } }) .collect::>(); let blocks_matching_polling_filter = self.load_ptrs_for_blocks( logger.clone(), - matching_blocks.iter().map(|(k, _)| *k).collect_vec(), + matching_blocks.keys().cloned().collect_vec(), ); let block_futures = blocks_matching_polling_filter.map(move |ptrs| { ptrs.into_iter() .flat_map(|ptr| { - let triggers = matching_blocks + matching_blocks .get(&ptr.number) // Safe to unwrap since we are iterating over ptrs which was created from // the keys of matching_blocks .unwrap() .iter() - .map(move |trigger| EthereumTrigger::Block(ptr.clone(), trigger.clone())); - - triggers + .map(move |trigger| EthereumTrigger::Block(ptr.clone(), trigger.clone())) }) .collect::>() }); @@ -811,11 +1056,12 @@ impl EthereumAdapter { logger: &Logger, subgraph_metrics: Arc, block_number: BlockNumber, - block_hash: H256, + block_hash: alloy::primitives::B256, ) -> Result, Error> { let eth = self.clone(); + let logger = self.provider_logger(logger); let addresses = Vec::new(); - let traces = eth + let traces: Vec = eth .trace_stream( logger, subgraph_metrics.clone(), @@ -823,8 +1069,7 @@ impl EthereumAdapter { block_number, addresses, ) - .collect() - .compat() + .try_collect() .await?; // `trace_stream` returns all of the traces for the block, and this @@ -842,7 +1087,7 @@ impl EthereumAdapter { // all the traces for the block, we need to ensure that the // block hash for the traces is equal to the desired block hash. // Assume all traces are for the same block. - if traces.iter().nth(0).unwrap().block_hash != block_hash { + if traces.first().unwrap().block_hash != Some(block_hash) { return Err(anyhow!( "Trace stream returned traces for an unexpected block: \ number = `{}`, hash = `{}`", @@ -863,7 +1108,7 @@ impl EthereumAdapter { logger: Logger, from: BlockNumber, to: BlockNumber, - ) -> Box, Error = Error> + Send> { + ) -> Box, Error = Error> + Send + '_> { // Currently we can't go to the DB for this because there might be duplicate entries for // the same block number. debug!(&logger, "Requesting hashes for blocks [{}, {}]", from, to); @@ -877,7 +1122,7 @@ impl EthereumAdapter { &self, logger: Logger, blocks: Vec, - ) -> Box, Error = Error> + Send> { + ) -> Box, Error = Error> + Send + '_> { // Currently we can't go to the DB for this because there might be duplicate entries for // the same block number. debug!(&logger, "Requesting hashes for blocks {:?}", blocks); @@ -886,18 +1131,27 @@ impl EthereumAdapter { pub async fn chain_id(&self) -> Result { let logger = self.logger.clone(); - let web3 = self.web3.clone(); - u64::try_from( - retry("chain_id RPC call", &logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { web3.eth().chain_id().await } - }) - .await?, - ) - .map_err(Error::msg) + let alloy = self.alloy.clone(); + retry("chain_id RPC call", &logger) + .redact_log_urls(true) + .no_limit() + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { alloy.get_chain_id().await.map_err(Error::from) } + }) + .await + .map_err(|e| e.into_inner().unwrap_or(EthereumRpcError::Timeout.into())) + } +} + +// Detects null blocks as can occur on Filecoin EVM chains, by checking for the FEVM-specific +// error returned when requesting such a null round. Ideally there should be a defined reponse or +// message for this case, or a check that is less dependent on the Filecoin implementation. +fn detect_null_block(res: &Result) -> bool { + match res { + Ok(_) => false, + Err(e) => e.to_string().contains("requested epoch was a null round"), } } @@ -910,18 +1164,19 @@ impl EthereumAdapterTrait for EthereumAdapter { async fn net_identifiers(&self) -> Result { let logger = self.logger.clone(); - let web3 = self.web3.clone(); + let alloy = self.alloy.clone(); let metrics = self.metrics.clone(); let provider = self.provider().to_string(); let net_version_future = retry("net_version RPC call", &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(20) .run(move || { - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); let metrics = metrics.cheap_clone(); let provider = provider.clone(); async move { - web3.net().version().await.map_err(|e| { + alloy.get_net_version().await.map_err(|e| { metrics.set_status(ProviderStatus::VersionFail, &provider); e.into() }) @@ -934,31 +1189,32 @@ impl EthereumAdapterTrait for EthereumAdapter { }) .boxed(); - let web3 = self.web3.clone(); + let alloy_provider = self.alloy.clone(); let metrics = self.metrics.clone(); let provider = self.provider().to_string(); + let genesis_block_number = self.settings.genesis_block_number; let retry_log_message = format!( "eth_getBlockByNumber({}, false) RPC call", - ENV_VARS.genesis_block_number + genesis_block_number ); let gen_block_hash_future = retry(retry_log_message, &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(30) .run(move || { - let web3 = web3.cheap_clone(); + let alloy_genesis = alloy_provider.cheap_clone(); let metrics = metrics.cheap_clone(); let provider = provider.clone(); async move { - web3.eth() - .block(BlockId::Number(Web3BlockNumber::Number( - ENV_VARS.genesis_block_number.into(), - ))) + alloy_genesis + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + genesis_block_number, + )) .await - .map_err(|e| { + .inspect_err(|_| { metrics.set_status(ProviderStatus::GenesisFail, &provider); - e })? - .and_then(|gen_block| gen_block.hash.map(BlockHash::from)) + .map(|gen_block| BlockHash::from(gen_block.header.hash)) .ok_or_else(|| anyhow!("Ethereum node could not find genesis block")) } }) @@ -977,7 +1233,7 @@ impl EthereumAdapterTrait for EthereumAdapter { })?; let ident = ChainIdentifier { - net_version, + net_version: net_version.to_string(), genesis_block_hash, }; @@ -986,362 +1242,412 @@ impl EthereumAdapterTrait for EthereumAdapter { Ok(ident) } - fn latest_block_header( - &self, - logger: &Logger, - ) -> Box, Error = IngestorError> + Send> { - let web3 = self.web3.clone(); - Box::new( - retry("eth_getBlockByNumber(latest) no txs RPC call", logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { - let block_opt = web3 - .eth() - .block(Web3BlockNumber::Latest.into()) - .await - .map_err(|e| { - anyhow!("could not get latest block from Ethereum: {}", e) - })?; + async fn latest_block_ptr(&self, logger: &Logger) -> Result { + let alloy = self.alloy.clone(); + retry("eth_getBlockByNumber(latest) no txs RPC call", logger) + .redact_log_urls(true) + .limit(ENV_VARS.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + let block_opt = alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Latest) + .await + .map_err(|e| anyhow!("could not get latest block from Ethereum: {}", e))?; - block_opt - .ok_or_else(|| anyhow!("no latest block returned from Ethereum").into()) - } - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!("Ethereum node took too long to return latest block").into() - }) - }) - .boxed() - .compat(), - ) - } + let block = block_opt + .ok_or_else(|| anyhow!("no latest block returned from Ethereum"))?; - fn latest_block( - &self, - logger: &Logger, - ) -> Box + Send + Unpin> { - let web3 = self.web3.clone(); - Box::new( - retry("eth_getBlockByNumber(latest) with txs RPC call", logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { - let block_opt = web3 - .eth() - .block_with_txs(Web3BlockNumber::Latest.into()) - .await - .map_err(|e| { - anyhow!("could not get latest block from Ethereum: {}", e) - })?; - block_opt - .ok_or_else(|| anyhow!("no latest block returned from Ethereum").into()) - } - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!("Ethereum node took too long to return latest block").into() - }) + Ok(BlockPtr::from((block.header.hash, block.header.number))) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return latest block").into() }) - .boxed() - .compat(), - ) + }) + .await } - fn load_block( - &self, - logger: &Logger, - block_hash: H256, - ) -> Box + Send> { - Box::new( - self.block_by_hash(logger, block_hash) - .and_then(move |block_opt| { - block_opt.ok_or_else(move || { - anyhow!( - "Ethereum node could not find block with hash {}", - block_hash - ) - }) - }), - ) + async fn is_reachable(&self) -> bool { + let alloy = self.alloy.clone(); + tokio::time::timeout(Duration::from_secs(10), alloy.get_block_number()) + .await + .map(|r| r.is_ok()) + .unwrap_or(false) } - fn block_by_hash( + async fn block_by_hash( &self, logger: &Logger, - block_hash: H256, - ) -> Box, Error = Error> + Send> { - let web3 = self.web3.clone(); + block_hash: B256, + ) -> Result, Error> { + let alloy = self.alloy.clone(); let logger = logger.clone(); let retry_log_message = format!( "eth_getBlockByHash RPC call for block hash {:?}", block_hash ); - Box::new( - retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - Box::pin(web3.eth().block_with_txs(BlockId::Hash(block_hash))) - .compat() - .from_err() - .compat() - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!("Ethereum node took too long to return block {}", block_hash) - }) + + retry(retry_log_message, &logger) + .redact_log_urls(true) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + alloy + .get_block_by_hash(block_hash) + .full() + .await + .map_err(Error::from) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return block {}", block_hash) }) - .boxed() - .compat(), - ) + }) + .await } - fn block_by_number( + async fn block_by_number( &self, logger: &Logger, block_number: BlockNumber, - ) -> Box, Error = Error> + Send> { - let web3 = self.web3.clone(); + ) -> Result, Error> { + let alloy = self.alloy.clone(); let logger = logger.clone(); let retry_log_message = format!( "eth_getBlockByNumber RPC call for block number {}", block_number ); - Box::new( - retry(retry_log_message, &logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { - web3.eth() - .block_with_txs(BlockId::Number(block_number.into())) - .await - .map_err(Error::from) - } - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!( - "Ethereum node took too long to return block {}", - block_number - ) - }) + retry(retry_log_message, &logger) + .redact_log_urls(true) + .no_limit() + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.clone(); + async move { + alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + block_number as u64, + )) + .full() + .await + .map_err(Error::from) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!( + "Ethereum node took too long to return block {}", + block_number + ) }) - .boxed() - .compat(), - ) + }) + .await } - fn load_full_block( + async fn load_full_block( &self, logger: &Logger, - block: LightEthereumBlock, - ) -> Pin> + Send>> - { - let web3 = Arc::clone(&self.web3); - let logger = logger.clone(); - let block_hash = block.hash.expect("block is missing block hash"); + block: AnyBlock, + ) -> Result { + let alloy = self.alloy.clone(); + let logger = self.provider_logger(logger); + let block_hash = block.header.hash; // The early return is necessary for correctness, otherwise we'll // request an empty batch which is not valid in JSON-RPC. if block.transactions.is_empty() { trace!(logger, "Block {} contains no transactions", block_hash); - return Box::pin(std::future::ready(Ok(EthereumBlock { - block: Arc::new(block), + return Ok(EthereumBlock { + block: Arc::new(LightEthereumBlock::new(block)), transaction_receipts: Vec::new(), - }))); + }); } - let hashes: Vec<_> = block.transactions.iter().map(|txn| txn.hash).collect(); - let receipts_future = if ENV_VARS.fetch_receipts_in_batches { - // Deprecated batching retrieval of transaction receipts. - fetch_transaction_receipts_in_batch_with_retry(web3, hashes, block_hash, logger).boxed() - } else { - let hash_stream = graph::tokio_stream::iter(hashes); - let receipt_stream = graph::tokio_stream::StreamExt::map(hash_stream, move |tx_hash| { - fetch_transaction_receipt_with_retry( - web3.cheap_clone(), - tx_hash, - block_hash, - logger.cheap_clone(), - ) - }) - .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls); - graph::tokio_stream::StreamExt::collect::< - Result>, IngestorError>, - >(receipt_stream) - .boxed() - }; + let hashes: Vec<_> = block.transactions.hashes().collect(); - let block_future = - futures03::TryFutureExt::map_ok(receipts_future, move |transaction_receipts| { - EthereumBlock { - block: Arc::new(block), - transaction_receipts, - } - }); + let supports_block_receipts = self + .check_block_receipt_support_and_update_cache( + alloy.clone(), + block_hash, + self.supports_eip_1898, + self.call_only, + logger.clone(), + ) + .await; - Box::pin(block_future) + fetch_receipts_with_retry( + alloy, + hashes, + block_hash, + logger, + supports_block_receipts, + &self.settings, + ) + .await + .map(|transaction_receipts| EthereumBlock { + block: Arc::new(LightEthereumBlock::new(block)), + transaction_receipts, + }) } - fn block_pointer_from_number( + async fn get_balance( &self, logger: &Logger, - block_number: BlockNumber, - ) -> Box + Send> { - Box::new( - self.block_hash_by_block_number(logger, block_number) - .and_then(move |block_hash_opt| { - block_hash_opt.ok_or_else(|| { - anyhow!( - "Ethereum node could not find start block hash by block number {}", - &block_number - ) - }) - }) - .from_err() - .map(move |block_hash| BlockPtr::from((block_hash, block_number))), - ) + address: Address, + block_ptr: BlockPtr, + ) -> Result { + let logger = self.provider_logger(logger); + debug!( + logger, "eth_getBalance"; + "address" => format!("{}", address), + "block" => format!("{}", block_ptr) + ); + self.balance(logger, address, block_ptr).await } - fn block_hash_by_block_number( + async fn get_code( &self, logger: &Logger, - block_number: BlockNumber, - ) -> Box, Error = Error> + Send> { - let web3 = self.web3.clone(); - let retry_log_message = format!( - "eth_getBlockByNumber RPC call for block number {}", - block_number + address: Address, + block_ptr: BlockPtr, + ) -> Result { + let logger = self.provider_logger(logger); + debug!( + logger, "eth_getCode"; + "address" => format!("{}", address), + "block" => format!("{}", block_ptr) ); - Box::new( - retry(retry_log_message, logger) + self.code(logger, address, block_ptr).await + } + + async fn next_existing_ptr_to_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Result { + let mut next_number = block_number; + loop { + let retry_log_message = format!( + "eth_getBlockByNumber RPC call for block number {}", + next_number + ); + let alloy = self.alloy.clone(); + let logger = logger.clone(); + let res = retry(retry_log_message, &logger) + .redact_log_urls(true) + .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); async move { - web3.eth() - .block(BlockId::Number(block_number.into())) + alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + next_number as u64, + )) .await - .map(|block_opt| block_opt.and_then(|block| block.hash)) + .map(|block_opt| { + block_opt.map(|block| BlockHash::from(block.header.hash.0.to_vec())) + }) .map_err(Error::from) } }) - .boxed() - .compat() + .await .map_err(move |e| { e.into_inner().unwrap_or_else(move || { anyhow!( "Ethereum node took too long to return data for block #{}", - block_number + next_number ) }) - }), - ) + }); + if detect_null_block(&res) { + next_number += 1; + continue; + } + return match res { + Ok(Some(hash)) => Ok(BlockPtr::new(hash, next_number)), + Ok(None) => Err(anyhow!("Block {} does not contain hash", next_number)), + Err(e) => Err(e), + }; + } } - fn contract_call( + async fn contract_call( &self, logger: &Logger, - call: EthereumContractCall, + inp_call: &ContractCall, cache: Arc, - ) -> Box, Error = EthereumContractCallError> + Send> { - // Emit custom error for type mismatches. - for (token, kind) in call - .args - .iter() - .zip(call.function.inputs.iter().map(|p| &p.kind)) - { - if !token.type_check(kind) { - return Box::new(future::err(EthereumContractCallError::TypeError( - token.clone(), - kind.clone(), - ))); + ) -> Result<(Option>, call::Source), ContractCallError> { + let mut result = self.contract_calls(logger, &[inp_call], cache).await?; + // unwrap: self.contract_calls returns as many results as there were calls + Ok(result.pop().unwrap()) + } + + async fn contract_calls( + &self, + logger: &Logger, + calls: &[&ContractCall], + cache: Arc, + ) -> Result>, call::Source)>, ContractCallError> { + fn as_req( + logger: &ProviderLogger, + call: &ContractCall, + index: u32, + ) -> Result { + // Emit custom error for type mismatches. + for (val, kind) in call + .args + .iter() + .zip(call.function.inputs.iter().map(|p| p.selector_type())) + { + let kind: abi::DynSolType = kind.parse().map_err(|err| { + ContractCallError::ABIError(anyhow!( + "failed to parse function input type '{kind}': {err}" + )) + })?; + + if !val.type_check(&kind) { + return Err(ContractCallError::TypeError(val.clone(), kind.clone())); + } + } + + // Encode the call parameters according to the ABI + let req = { + let encoded_call = call + .function + .abi_encode_input(&call.args) + .map_err(ContractCallError::EncodingError)?; + call::Request::new(call.address, encoded_call, index) + }; + + trace!(logger, "eth_call"; + "fn" => &call.function.name, + "address" => hex::encode(call.address), + "data" => hex::encode(req.encoded_call.as_ref()), + "block_hash" => call.block_ptr.hash_hex(), + "block_number" => call.block_ptr.block_number() + ); + Ok(req) + } + + fn decode( + logger: &ProviderLogger, + resp: call::Response, + call: &ContractCall, + ) -> (Option>, call::Source) { + let call::Response { + retval, + source, + req: _, + } = resp; + match retval { + call::Retval::Value(output) => match call.function.abi_decode_output(&output) { + Ok(tokens) => (Some(tokens), source), + Err(e) => { + // Decode failures are reverts. The reasoning is that if Solidity fails to + // decode an argument, that's a revert, so the same goes for the output. + let reason = format!("failed to decode output: {}", e); + info!(logger, "Contract call reverted"; "reason" => reason); + (None, call::Source::Rpc) + } + }, + call::Retval::Null => { + // We got a `0x` response. For old Geth, this can mean a revert. It can also be + // that the contract actually returned an empty response. A view call is meant + // to return something, so we treat empty responses the same as reverts. + info!(logger, "Contract call reverted"; "reason" => "empty response"); + (None, call::Source::Rpc) + } + } + } + + fn log_call_error(logger: &ProviderLogger, e: &ContractCallError, call: &ContractCall) { + match e { + ContractCallError::AlloyError(e) => error!( + logger, + "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", + call.function.name, + call.contract_name, + e + ), + ContractCallError::Timeout => error!( + logger, + "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", + call.function.name, + call.contract_name + ), + _ => error!( + logger, + "Failed to call function \"{}\" of contract \"{}\": {}", + call.function.name, + call.contract_name, + e + ), } } - // Encode the call parameters according to the ABI - let call_data = match call.function.encode_input(&call.args) { - Ok(data) => data, - Err(e) => return Box::new(future::err(EthereumContractCallError::EncodingError(e))), - }; + let logger = self.provider_logger(logger); - debug!(logger, "eth_call"; - "address" => hex::encode(call.address), - "data" => hex::encode(&call_data), - "block_hash" => call.block_ptr.hash_hex(), - "block_number" => call.block_ptr.block_number() - ); + if calls.is_empty() { + return Ok(Vec::new()); + } - // Check if we have it cached, if not do the call and cache. - Box::new( - match cache - .get_call(call.address, &call_data, call.block_ptr.clone()) - .map_err(|e| error!(logger, "call cache get error"; "error" => e.to_string())) - .ok() - .flatten() - { - Some(result) => { - Box::new(future::ok(result)) as Box + Send> - } - None => { - let cache = cache.clone(); - let call = call.clone(); - let logger = logger.clone(); - Box::new( - self.call( - logger.clone(), - call.address, - Bytes(call_data.clone()), - call.block_ptr.clone(), - call.gas, - ) - .map(move |result| { - // Don't block handler execution on writing to the cache. - let for_cache = result.0.clone(); - if !result.0.is_empty() { - let _ = graph::spawn_blocking_allow_panic(move || { - cache - .set_call( - call.address, - &call_data, - call.block_ptr, - &for_cache, - ) - .map_err(|e| { - error!(logger, "call cache set error"; - "error" => e.to_string()) - }) - }); - } - result.0 - }), - ) + let block_ptr = calls.first().unwrap().block_ptr.clone(); + if calls.iter().any(|call| call.block_ptr != block_ptr) { + return Err(ContractCallError::Internal( + "all calls must have the same block pointer".to_string(), + )); + } + + let reqs: Vec<_> = calls + .iter() + .enumerate() + .map(|(index, call)| as_req(&logger, call, index as u32)) + .collect::>()?; + + let (mut resps, missing) = cache + .get_calls(&reqs, block_ptr) + .await + .map_err(|e| error!(logger, "call cache get error"; "error" => e.to_string())) + .unwrap_or_else(|_| (Vec::new(), reqs)); + + let futs = missing.into_iter().map(|req| { + let cache = cache.clone(); + let logger = logger.clone(); + async move { + let call = calls[req.index as usize]; + match self.call_and_cache(&logger, call, req, cache.clone()).await { + Ok(resp) => Ok(resp), + Err(e) => { + log_call_error(&logger, &e, call); + Err(e) + } } } - // Decode the return values according to the ABI - .and_then(move |output| { - if output.is_empty() { - // We got a `0x` response. For old Geth, this can mean a revert. It can also be - // that the contract actually returned an empty response. A view call is meant - // to return something, so we treat empty responses the same as reverts. - Err(EthereumContractCallError::Revert("empty response".into())) - } else { - // Decode failures are reverts. The reasoning is that if Solidity fails to - // decode an argument, that's a revert, so the same goes for the output. - call.function.decode_output(&output).map_err(|e| { - EthereumContractCallError::Revert(format!("failed to decode output: {}", e)) - }) - } - }), - ) + }); + resps.extend(try_join_all(futs).await?); + + // If we make it here, we have a response for every call. + debug_assert_eq!(resps.len(), calls.len()); + + // Bring the responses into the same order as the calls + resps.sort_by_key(|resp| resp.req.index); + + let decoded: Vec<_> = resps + .into_iter() + .map(|res| { + let call = &calls[res.req.index as usize]; + decode(&logger, res, call) + }) + .collect(); + + Ok(decoded) } /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. @@ -1349,56 +1655,53 @@ impl EthereumAdapterTrait for EthereumAdapter { &self, logger: Logger, chain_store: Arc, - block_hashes: HashSet, - ) -> Box, Error = Error> + Send> { + block_hashes: HashSet, + ) -> Result>, Error> { let block_hashes: Vec<_> = block_hashes.iter().cloned().collect(); // Search for the block in the store first then use json-rpc as a backup. - let mut blocks: Vec> = chain_store + let mut blocks: Vec<_> = chain_store .cheap_clone() .blocks(block_hashes.iter().map(|&b| b.into()).collect::>()) .await .map_err(|e| error!(&logger, "Error accessing block cache {}", e)) .unwrap_or_default() - .into_iter() - .filter_map(|value| json::from_value(value).ok()) - .map(Arc::new) + .iter() + .map(|b| b.to_light_block()) .collect(); let missing_blocks = Vec::from_iter( block_hashes .into_iter() - .filter(|hash| !blocks.iter().any(|b| b.hash == Some(*hash))), + .filter(|hash| !blocks.iter().any(|b| b.hash() == *hash)), ); // Return a stream that lazily loads batches of blocks. debug!(logger, "Requesting {} block(s)", missing_blocks.len()); - Box::new( - self.load_blocks_rpc(logger.clone(), missing_blocks) - .collect() - .map(move |new_blocks| { - let upsert_blocks: Vec<_> = new_blocks - .iter() - .map(|block| BlockFinality::Final(block.clone())) - .collect(); - let block_refs: Vec<_> = upsert_blocks - .iter() - .map(|block| block as &dyn graph::blockchain::Block) - .collect(); - if let Err(e) = chain_store.upsert_light_blocks(block_refs.as_slice()) { - error!(logger, "Error writing to block cache {}", e); - } - blocks.extend(new_blocks); - blocks.sort_by_key(|block| block.number); - stream::iter_ok(blocks) - }) - .flatten_stream(), - ) + let new_blocks: Vec<_> = self + .load_blocks_rpc(logger.clone(), missing_blocks) + .try_collect() + .await?; + let upsert_blocks: Vec<_> = new_blocks + .iter() + .map(|block| BlockFinality::Final(block.clone())) + .collect(); + let block_refs: Vec<_> = upsert_blocks + .iter() + .map(|block| block as &dyn graph::blockchain::Block) + .collect(); + if let Err(e) = chain_store.upsert_light_blocks(block_refs.as_slice()).await { + error!(logger, "Error writing to block cache {}", e); + } + blocks.extend(new_blocks); + blocks.sort_by_key(|block| block.number()); + Ok(blocks) } } -/// Returns blocks with triggers, corresponding to the specified range and filters. +/// Returns blocks with triggers, corresponding to the specified range and filters; and the resolved +/// `to` block, which is the nearest non-null block greater than or equal to the passed `to` block. /// If a block contains no triggers, there may be no corresponding item in the stream. -/// However the `to` block will always be present, even if triggers are empty. +/// However the (resolved) `to` block will always be present, even if triggers are empty. /// /// Careful: don't use this function without considering race conditions. /// Chain reorgs could happen at any time, and could affect the answer received. @@ -1418,22 +1721,31 @@ pub(crate) async fn blocks_with_triggers( to: BlockNumber, filter: &TriggerFilter, unified_api_version: UnifiedMappingApiVersion, -) -> Result>, Error> { +) -> Result<(Vec>, BlockNumber), Error> { // Each trigger filter needs to be queried for the same block range // and the blocks yielded need to be deduped. If any error occurs // while searching for a trigger type, the entire operation fails. let eth = adapter.clone(); let call_filter = EthereumCallFilter::from(&filter.block); + let logger = ProviderLogger::new(&logger, eth.provider()); // Scan the block range to find relevant triggers let trigger_futs: FuturesUnordered, anyhow::Error>>> = FuturesUnordered::new(); + // Resolve the nearest non-null "to" block + debug!(logger, "Finding nearest valid `to` block to {}", to); + + let to_ptr = eth.next_existing_ptr_to_number(&logger, to).await?; + let to_hash = to_ptr.hash.as_b256(); + let to = to_ptr.block_number(); + // This is for `start` triggers which can be initialization handlers which needs to be run // before all other triggers if filter.block.trigger_every_block { + let logger = logger.clone(); let block_future = eth - .block_range_to_ptrs(logger.clone(), from, to) + .block_range_to_ptrs((*logger).clone(), from, to) .map(move |ptrs| { ptrs.into_iter() .flat_map(|ptr| { @@ -1449,7 +1761,7 @@ pub(crate) async fn blocks_with_triggers( trigger_futs.push(block_future) } else if !filter.block.polling_intervals.is_empty() { let block_futures_matching_once_filter = - eth.blocks_matching_polling_intervals(logger.clone(), from, to, &filter.block); + eth.blocks_matching_polling_intervals((*logger).clone(), from, to, &filter.block); trigger_futs.push(block_futures_matching_once_filter); } @@ -1457,7 +1769,7 @@ pub(crate) async fn blocks_with_triggers( if !filter.log.is_empty() { let logs_future = get_logs_and_transactions( ð, - &logger, + logger.clone(), subgraph_metrics.clone(), from, to, @@ -1497,31 +1809,16 @@ pub(crate) async fn blocks_with_triggers( trigger_futs.push(block_future) } - // Get hash for "to" block - let to_hash_fut = eth - .block_hash_by_block_number(&logger, to) - .and_then(|hash| match hash { - Some(hash) => Ok(hash), - None => { - warn!(logger, - "Ethereum endpoint is behind"; - "url" => eth.provider() - ); - bail!("Block {} not found in the chain", to) - } - }) - .compat(); - - // Join on triggers and block hash resolution - let (triggers, to_hash) = futures03::join!(trigger_futs.try_concat(), to_hash_fut); - - // Unpack and handle possible errors in the previously joined futures - let triggers = - triggers.with_context(|| format!("Failed to obtain triggers for block {}", to))?; - let to_hash = to_hash.with_context(|| format!("Failed to infer hash for block {}", to))?; + // Join on triggers, unpack and handle possible errors + let triggers = trigger_futs + .try_concat() + .await + .with_context(|| format!("Failed to obtain triggers for block {}", to))?; - let mut block_hashes: HashSet = - triggers.iter().map(EthereumTrigger::block_hash).collect(); + let mut block_hashes: HashSet = triggers + .iter() + .map(|trigger| trigger.block_hash()) + .collect(); let mut triggers_by_block: HashMap> = triggers.into_iter().fold(HashMap::new(), |mut map, t| { map.entry(t.block_number()).or_default().push(t); @@ -1536,11 +1833,12 @@ pub(crate) async fn blocks_with_triggers( let logger2 = logger.cheap_clone(); - let blocks = eth - .load_blocks(logger.cheap_clone(), chain_store.clone(), block_hashes) - .await - .and_then( - move |block| match triggers_by_block.remove(&(block.number() as BlockNumber)) { + let blocks: Vec<_> = eth + .load_blocks((*logger).cheap_clone(), chain_store.clone(), block_hashes) + .await? + .into_iter() + .map( + move |block| match triggers_by_block.remove(&(block.number())) { Some(triggers) => Ok(BlockWithTriggers::new( BlockFinality::Final(block), triggers, @@ -1552,9 +1850,7 @@ pub(crate) async fn blocks_with_triggers( )), }, ) - .collect() - .compat() - .await?; + .collect::>()?; // Filter out call triggers that come from unsuccessful transactions let futures = blocks.into_iter().map(|block| { @@ -1583,7 +1879,7 @@ pub(crate) async fn blocks_with_triggers( )); } - Ok(blocks) + Ok((blocks, to)) } pub(crate) async fn get_calls( @@ -1612,13 +1908,13 @@ pub(crate) async fn get_calls( } else { client .rpc()? - .cheapest_with(capabilities)? + .cheapest_with(capabilities) + .await? .calls_in_block( &logger, subgraph_metrics.clone(), - BlockNumber::try_from(ethereum_block.block.number.unwrap().as_u64()) - .unwrap(), - ethereum_block.block.hash.unwrap(), + ethereum_block.block.number(), + ethereum_block.block.hash(), ) .await? }; @@ -1627,6 +1923,9 @@ pub(crate) async fn get_calls( calls: Some(calls), })) } + BlockFinality::Ptr(_) => { + unreachable!("get_calls called with BlockFinality::Ptr") + } } } @@ -1638,15 +1937,36 @@ pub(crate) fn parse_log_triggers( return vec![]; } - block + let total_logs: usize = block .transaction_receipts .iter() - .flat_map(move |receipt| { - receipt.logs.iter().enumerate().map(move |(index, _)| { - EthereumTrigger::Log(LogRef::LogPosition(index, receipt.cheap_clone())) - }) - }) - .collect() + .map(|r| r.logs().len()) + .sum(); + let mut triggers = Vec::with_capacity(total_logs); + + for receipt in &block.transaction_receipts { + for (index, log) in receipt.logs().iter().enumerate() { + let requires_transaction_receipt = log + .topics() + .first() + .map(|signature| { + log_filter.requires_transaction_receipt( + signature, + Some(&log.address()), + log.topics(), + ) + }) + .unwrap_or(false); + + triggers.push(EthereumTrigger::Log(LogRef::LogPosition(LogPosition { + index, + receipt: receipt.cheap_clone(), + requires_transaction_receipt, + }))); + } + } + + triggers } pub(crate) fn parse_call_triggers( @@ -1674,6 +1994,36 @@ pub(crate) fn parse_call_triggers( } } +/// For a given `block_number`, return the block trigger types that fire +/// based on the rules in `polling_intervals`. Each entry is `(start_block, +/// interval)` where `interval == 0` encodes a `once` rule and `interval > 0` +/// encodes a `polling every interval` rule. Both rule kinds can fire at the +/// same block (e.g. a once and polling rule sharing a `start_block`), so the +/// returned Vec may contain `Start`, `End`, both, or neither. +pub(crate) fn block_trigger_types_from_intervals( + block_number: i32, + polling_intervals: &HashSet<(i32, i32)>, +) -> Vec { + let has_once_trigger = polling_intervals + .iter() + .any(|(start_block, interval)| *interval == 0 && block_number == *start_block); + + let has_polling_trigger = polling_intervals.iter().any(|(start_block, interval)| { + *interval > 0 + && block_number >= *start_block + && (block_number - start_block) % *interval == 0 + }); + + let mut triggers = Vec::new(); + if has_once_trigger { + triggers.push(EthereumBlockTriggerType::Start); + } + if has_polling_trigger { + triggers.push(EthereumBlockTriggerType::End); + } + triggers +} + /// This method does not parse block triggers with `once` filters. /// This is because it is to be run before any other triggers are run. /// So we have `parse_initialization_triggers` for that. @@ -1685,7 +2035,7 @@ pub(crate) fn parse_block_triggers( return vec![]; } - let block_ptr = BlockPtr::from(&block.ethereum_block); + let block_ptr = block.ethereum_block.block.block_ptr(); let trigger_every_block = block_filter.trigger_every_block; let call_filter = EthereumCallFilter::from(block_filter); let block_ptr2 = block_ptr.cheap_clone(); @@ -1715,38 +2065,12 @@ pub(crate) fn parse_block_triggers( EthereumBlockTriggerType::End, )); } else if !block_filter.polling_intervals.is_empty() { - let has_polling_trigger = - &block_filter - .polling_intervals - .iter() - .any(|(start_block, interval)| match interval { - 0 => false, - _ => { - block_number >= *start_block - && (block_number - *start_block) % *interval == 0 - } - }); - - let has_once_trigger = - &block_filter - .polling_intervals - .iter() - .any(|(start_block, interval)| match interval { - 0 => block_number == *start_block, - _ => false, - }); - - if *has_once_trigger { - triggers.push(EthereumTrigger::Block( - block_ptr3.clone(), - EthereumBlockTriggerType::Start, - )); - } - - if *has_polling_trigger { + for trigger_type in + block_trigger_types_from_intervals(block_number, &block_filter.polling_intervals) + { triggers.push(EthereumTrigger::Block( - block_ptr3, - EthereumBlockTriggerType::End, + block_ptr3.cheap_clone(), + trigger_type, )); } } @@ -1755,9 +2079,9 @@ pub(crate) fn parse_block_triggers( async fn fetch_receipt_from_ethereum_client( eth: &EthereumAdapter, - transaction_hash: &H256, -) -> anyhow::Result { - match eth.web3.eth().transaction_receipt(*transaction_hash).await { + transaction_hash: B256, +) -> anyhow::Result { + match eth.alloy.get_transaction_receipt(transaction_hash).await { Ok(Some(receipt)) => Ok(receipt), Ok(None) => bail!("Could not find transaction receipt"), Err(error) => bail!("Failed to fetch transaction receipt: {}", error), @@ -1768,7 +2092,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( mut block: BlockWithTriggers, eth: &EthereumAdapter, chain_store: &Arc, - logger: &Logger, + logger: &ProviderLogger, ) -> anyhow::Result> { // Return early if there is no trigger data if block.trigger_data.is_empty() { @@ -1778,14 +2102,14 @@ async fn filter_call_triggers_from_unsuccessful_transactions( let initial_number_of_triggers = block.trigger_data.len(); // Get the transaction hash from each call trigger - let transaction_hashes: BTreeSet = block + let transaction_hashes: BTreeSet = block .trigger_data .iter() - .filter_map(|trigger| match trigger { - EthereumTrigger::Call(call_trigger) => Some(call_trigger.transaction_hash), + .filter_map(|trigger| match trigger.as_chain() { + Some(EthereumTrigger::Call(call_trigger)) => Some(call_trigger.transaction_hash), _ => None, }) - .collect::>>() + .collect::>>() .ok_or(anyhow!( "failed to obtain transaction hash from call triggers" ))?; @@ -1796,18 +2120,24 @@ async fn filter_call_triggers_from_unsuccessful_transactions( } // And obtain all Transaction values for the calls in this block. - let transactions: Vec<&Transaction> = { + let transactions: Vec<&AnyTransaction> = { match &block.block { - BlockFinality::Final(ref block) => block - .transactions + BlockFinality::Final(block) => block + .transactions() + .ok_or_else(|| anyhow!("Block transactions not available"))? .iter() - .filter(|transaction| transaction_hashes.contains(&transaction.hash)) + .filter(|transaction| transaction_hashes.contains(&transaction.tx_hash())) .collect(), BlockFinality::NonFinal(_block_with_calls) => { unreachable!( "this function should not be called when dealing with non-final blocks" ) } + BlockFinality::Ptr(_block) => { + unreachable!( + "this function should not be called when dealing with header-only blocks" + ) + } } }; @@ -1818,21 +2148,21 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // We'll also need the receipts for those transactions. In this step we collect all receipts // we have in store for the current block. - let mut receipts = chain_store - .transaction_receipts_in_block(&block.ptr().hash_as_h256()) + let mut receipts: BTreeMap = chain_store + .transaction_receipts_in_block(&block.ptr().hash.as_b256()) .await? .into_iter() .map(|receipt| (receipt.transaction_hash, receipt)) - .collect::>(); + .collect::>(); // Do we have a receipt for each transaction under analysis? - let mut receipts_and_transactions: Vec<(&Transaction, LightTransactionReceipt)> = Vec::new(); - let mut transactions_without_receipt: Vec<&Transaction> = Vec::new(); + let mut receipts_and_transactions: Vec<(&AnyTransaction, LightTransactionReceipt)> = Vec::new(); + let mut transactions_without_receipt: Vec<&AnyTransaction> = Vec::new(); for transaction in transactions.iter() { - if let Some(receipt) = receipts.remove(&transaction.hash) { - receipts_and_transactions.push((transaction, receipt)); + if let Some(receipt) = receipts.remove(&transaction.tx_hash()) { + receipts_and_transactions.push((*transaction, receipt)); } else { - transactions_without_receipt.push(transaction); + transactions_without_receipt.push(*transaction); } } @@ -1840,7 +2170,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( let futures = transactions_without_receipt .iter() .map(|transaction| async move { - fetch_receipt_from_ethereum_client(eth, &transaction.hash) + fetch_receipt_from_ethereum_client(eth, transaction.tx_hash()) .await .map(|receipt| (transaction, receipt)) }); @@ -1855,12 +2185,9 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // additional Ethereum API calls for future scans on this block. // With all transactions and receipts in hand, we can evaluate the success of each transaction - let mut transaction_success: BTreeMap<&H256, bool> = BTreeMap::new(); + let mut transaction_success: BTreeMap = BTreeMap::new(); for (transaction, receipt) in receipts_and_transactions.into_iter() { - transaction_success.insert( - &transaction.hash, - evaluate_transaction_status(receipt.status), - ); + transaction_success.insert(transaction.tx_hash(), receipt.status); } // Confidence check: Did we inspect the status of all transactions? @@ -1873,7 +2200,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // Filter call triggers from unsuccessful transactions block.trigger_data.retain(|trigger| { - if let EthereumTrigger::Call(call_trigger) = trigger { + if let Some(EthereumTrigger::Call(call_trigger)) = trigger.as_chain() { // Unwrap: We already checked that those values exist transaction_success[&call_trigger.transaction_hash.unwrap()] } else { @@ -1902,75 +2229,242 @@ async fn filter_call_triggers_from_unsuccessful_transactions( /// Deprecated. Wraps the [`fetch_transaction_receipts_in_batch`] in a retry loop. async fn fetch_transaction_receipts_in_batch_with_retry( - web3: Arc>, - hashes: Vec, - block_hash: H256, - logger: Logger, -) -> Result>, IngestorError> { + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { let retry_log_message = format!( "batch eth_getTransactionReceipt RPC call for block {:?}", block_hash ); retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) + .redact_log_urls(true) + .limit(settings.request_retries) .no_logging() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(settings.json_rpc_timeout.as_secs()) .run(move || { - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); let hashes = hashes.clone(); let logger = logger.cheap_clone(); - fetch_transaction_receipts_in_batch(web3, hashes, block_hash, logger).boxed() + fetch_transaction_receipts_in_batch(alloy, hashes, block_hash, logger).boxed() }) .await .map_err(|_timeout| anyhow!(block_hash).into()) } -/// Deprecated. Attempts to fetch multiple transaction receipts in a batching contex. +/// Deprecated. Attempts to fetch multiple transaction receipts in a batching context. async fn fetch_transaction_receipts_in_batch( - web3: Arc>, - hashes: Vec, - block_hash: H256, - logger: Logger, -) -> Result>, IngestorError> { - let batching_web3 = Web3::new(Batch::new(web3.transport().clone())); - let eth = batching_web3.eth(); - let receipt_futures = hashes - .into_iter() - .map(move |hash| { - let logger = logger.cheap_clone(); - eth.transaction_receipt(hash) - .map_err(IngestorError::from) - .and_then(move |some_receipt| async move { - resolve_transaction_receipt(some_receipt, hash, block_hash, logger) - }) + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, +) -> Result>, IngestorError> { + // Use the batch method to get all receipts at once + let receipts = batch_get_transaction_receipts(alloy, hashes.clone()) + .await + .map_err(|e| { + IngestorError::Unknown(anyhow::anyhow!("Batch receipt fetch failed: {}", e)) + })?; + + let mut result = Vec::new(); + for (receipt, hash) in receipts.into_iter().zip(hashes.iter()) { + if let Some(receipt) = receipt { + let validated_receipt = resolve_transaction_receipt( + Some(receipt), + *hash, + block_hash, + logger.cheap_clone(), + )?; + result.push(Arc::new(validated_receipt)); + } else { + return Err(IngestorError::ReceiptUnavailable(block_hash, *hash)); + } + } + + Ok(result) +} + +async fn batch_get_transaction_receipts( + provider: Arc, + tx_hashes: Vec, +) -> Result>, Box> { + let mut batch = alloy::rpc::client::BatchRequest::new(provider.client()); + let mut receipt_futures = Vec::new(); + + // Add all receipt requests to batch + for tx_hash in &tx_hashes { + let receipt_future = batch.add_call::<(B256,), Option>( + "eth_getTransactionReceipt", + &(*tx_hash,), + )?; + receipt_futures.push(receipt_future); + } + + // Execute batch + batch.send().await?; + + // Collect results in order + let mut results = Vec::new(); + for receipt_future in receipt_futures { + let receipt = receipt_future.await?; + results.push(receipt); + } + + Ok(results) +} + +pub(crate) async fn check_block_receipt_support( + alloy: Arc, + block_hash: B256, + supports_eip_1898: bool, + call_only: bool, +) -> Result<(), Error> { + use alloy::rpc::types::BlockId; + if call_only { + return Err(anyhow!("Provider is call-only")); + } + + if !supports_eip_1898 { + return Err(anyhow!("Provider does not support EIP 1898")); + } + + // Fetch block receipts from the provider for the latest block. + let block_receipts_result = alloy.get_block_receipts(BlockId::from(block_hash)).await; + + // Determine if the provider supports block receipts based on the fetched result. + match block_receipts_result { + Ok(Some(receipts)) if !receipts.is_empty() => Ok(()), + Ok(_) => Err(anyhow!("Block receipts are empty")), + Err(err) => Err(anyhow!("Error fetching block receipts: {}", err)), + } +} + +// Fetches transaction receipts with retries. This function acts as a dispatcher +// based on whether block receipts are supported or individual transaction receipts +// need to be fetched. +async fn fetch_receipts_with_retry( + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + supports_block_receipts: bool, + settings: &ChainSettings, +) -> Result>, IngestorError> { + if supports_block_receipts { + return fetch_block_receipts_with_retry(alloy, hashes, block_hash, logger, settings).await; + } + fetch_individual_receipts_with_retry(alloy, hashes, block_hash, logger, settings).await +} + +// Fetches receipts for each transaction in the block individually. +async fn fetch_individual_receipts_with_retry( + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { + if ENV_VARS.fetch_receipts_in_batches { + return fetch_transaction_receipts_in_batch_with_retry( + alloy, hashes, block_hash, logger, settings, + ) + .await; + } + + let request_retries = settings.request_retries; + let json_rpc_timeout = settings.json_rpc_timeout; + let concurrent_requests = settings.block_ingestor_max_concurrent_json_rpc_calls; + + // Use a stream to fetch receipts individually + let hash_stream = tokio_stream::iter(hashes); + let receipt_stream = hash_stream + .map(move |tx_hash| { + fetch_transaction_receipt_with_retry( + alloy.cheap_clone(), + tx_hash, + block_hash, + logger.cheap_clone(), + request_retries, + json_rpc_timeout, + ) }) - .collect::>(); + .buffered(concurrent_requests); - batching_web3.transport().submit_batch().await?; + tokio_stream::StreamExt::collect::>, IngestorError>>( + receipt_stream, + ) + .await +} - let mut collected = vec![]; - for receipt in receipt_futures.into_iter() { - collected.push(Arc::new(receipt.await?)) +/// Fetches transaction receipts of all transactions in a block with `eth_getBlockReceipts` call. +async fn fetch_block_receipts_with_retry( + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { + use graph::prelude::alloy::rpc::types::BlockId; + let retry_log_message = format!("eth_getBlockReceipts RPC call for block {:?}", block_hash); + + // Perform the retry operation + let receipts_option = retry(retry_log_message, &logger) + .redact_log_urls(true) + .limit(settings.request_retries) + .timeout_secs(settings.json_rpc_timeout.as_secs()) + .run(move || alloy.get_block_receipts(BlockId::from(block_hash)).boxed()) + .await + .map_err(|_timeout| -> IngestorError { anyhow!(block_hash).into() })?; + + // Check if receipts are available, and transform them if they are + match receipts_option { + Some(receipts) => { + // Create a HashSet from the transaction hashes of the receipts + let receipt_hashes_set: HashSet<_> = + receipts.iter().map(|r| r.transaction_hash).collect(); + + // Check if the set contains all the hashes and has the same length as the hashes vec + if hashes.len() == receipt_hashes_set.len() + && hashes.iter().all(|hash| receipt_hashes_set.contains(hash)) + { + let transformed_receipts = receipts.into_iter().map(Arc::new).collect(); + Ok(transformed_receipts) + } else { + // If there's a mismatch in numbers or a missing hash, return an error + Err(IngestorError::BlockReceiptsMismatched(block_hash)) + } + } + None => { + // If no receipts are found, return an error + Err(IngestorError::BlockReceiptsUnavailable(block_hash)) + } } - Ok(collected) } -/// Retries fetching a single transaction receipt. +/// Retries fetching a single transaction receipt using alloy, then converts to web3 format. async fn fetch_transaction_receipt_with_retry( - web3: Arc>, - transaction_hash: H256, - block_hash: H256, - logger: Logger, -) -> Result, IngestorError> { - let logger = logger.cheap_clone(); + alloy: Arc, + transaction_hash: B256, + block_hash: B256, + logger: ProviderLogger, + request_retries: usize, + json_rpc_timeout: Duration, +) -> Result, IngestorError> { let retry_log_message = format!( "eth_getTransactionReceipt RPC call for transaction {:?}", transaction_hash ); + retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || web3.eth().transaction_receipt(transaction_hash).boxed()) + .redact_log_urls(true) + .limit(request_retries) + .timeout_secs(json_rpc_timeout.as_secs()) + .run(move || { + let alloy_clone = alloy.clone(); + async move { alloy_clone.get_transaction_receipt(transaction_hash).await }.boxed() + }) .await .map_err(|_timeout| anyhow!(block_hash).into()) .and_then(move |some_receipt| { @@ -1980,11 +2474,11 @@ async fn fetch_transaction_receipt_with_retry( } fn resolve_transaction_receipt( - transaction_receipt: Option, - transaction_hash: H256, - block_hash: H256, - logger: Logger, -) -> Result { + transaction_receipt: Option, + transaction_hash: B256, + block_hash: B256, + logger: ProviderLogger, +) -> Result { match transaction_receipt { // A receipt might be missing because the block was uncled, and the transaction never // made it back into the main chain. @@ -2037,7 +2531,7 @@ fn resolve_transaction_receipt( /// Retrieves logs and the associated transaction receipts, if required by the [`EthereumLogFilter`]. async fn get_logs_and_transactions( adapter: &Arc, - logger: &Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -2047,7 +2541,7 @@ async fn get_logs_and_transactions( // Obtain logs externally let logs = adapter .logs_in_block_range( - logger, + &logger, subgraph_metrics.cheap_clone(), from, to, @@ -2057,12 +2551,16 @@ async fn get_logs_and_transactions( // Not all logs have associated transaction hashes, nor do all triggers require them. // We also restrict receipts retrieval for some api versions. - let transaction_hashes_by_block: HashMap> = logs + let transaction_hashes_by_block: HashMap> = logs .iter() .filter(|_| unified_api_version.equal_or_greater_than(&API_VERSION_0_0_7)) .filter(|log| { - if let Some(signature) = log.topics.first() { - log_filter.requires_transaction_receipt(signature, Some(&log.address)) + if let Some(signature) = log.topics().first() { + log_filter.requires_transaction_receipt( + signature, + Some(&log.address()), + log.topics(), + ) } else { false } @@ -2077,7 +2575,7 @@ async fn get_logs_and_transactions( } }) .fold( - HashMap::>::new(), + HashMap::>::new(), |mut acc, (block_hash, txn_hash)| { acc.entry(block_hash).or_default().insert(txn_hash); acc @@ -2099,6 +2597,7 @@ async fn get_logs_and_transactions( let optional_receipt = log .transaction_hash .and_then(|txn| transaction_receipts_by_hash.get(&txn).cloned()); + let value = EthereumTrigger::Log(LogRef::FullLog(Arc::new(log), optional_receipt)); log_triggers.push(value); } @@ -2109,13 +2608,13 @@ async fn get_logs_and_transactions( /// Tries to retrive all transaction receipts for a set of transaction hashes. async fn get_transaction_receipts_for_transaction_hashes( adapter: &EthereumAdapter, - transaction_hashes_by_block: &HashMap>, + transaction_hashes_by_block: &HashMap>, subgraph_metrics: Arc, - logger: Logger, -) -> Result>, anyhow::Error> { + logger: ProviderLogger, +) -> Result>, anyhow::Error> { use std::collections::hash_map::Entry::Vacant; - let mut receipts_by_hash: HashMap> = HashMap::new(); + let mut receipts_by_hash: HashMap> = HashMap::new(); // Return early if input set is empty if transaction_hashes_by_block.is_empty() { @@ -2124,20 +2623,22 @@ async fn get_transaction_receipts_for_transaction_hashes( // Keep a record of all unique transaction hashes for which we'll request receipts. We will // later use this to check if we have collected the receipts from all required transactions. - let mut unique_transaction_hashes: HashSet<&H256> = HashSet::new(); + let mut unique_transaction_hashes: HashSet<&B256> = HashSet::new(); // Request transaction receipts concurrently let receipt_futures = FuturesUnordered::new(); - let web3 = Arc::clone(&adapter.web3); + let alloy = Arc::clone(&adapter.alloy); for (block_hash, transaction_hashes) in transaction_hashes_by_block { for transaction_hash in transaction_hashes { unique_transaction_hashes.insert(transaction_hash); let receipt_future = fetch_transaction_receipt_with_retry( - web3.cheap_clone(), + alloy.cheap_clone(), *transaction_hash, *block_hash, logger.cheap_clone(), + adapter.settings.request_retries, + adapter.settings.json_rpc_timeout, ); receipt_futures.push(receipt_future) } @@ -2191,24 +2692,28 @@ mod tests { use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger}; - use super::{parse_block_triggers, EthereumBlock, EthereumBlockFilter, EthereumBlockWithCalls}; + use super::{ + EthereumBlock, EthereumBlockFilter, EthereumBlockWithCalls, + block_trigger_types_from_intervals, check_block_receipt_support, parse_block_triggers, + }; use graph::blockchain::BlockPtr; - use graph::prelude::ethabi::ethereum_types::U64; - use graph::prelude::web3::types::{Address, Block, Bytes, H256}; - use graph::prelude::EthereumCall; + use graph::components::ethereum::AnyNetworkBare; + use graph::prelude::alloy::primitives::{Address, B256, Bytes}; + use graph::prelude::alloy::providers::ProviderBuilder; + use graph::prelude::alloy::providers::mock::Asserter; + use graph::prelude::{EthereumCall, LightEthereumBlock, create_minimal_block_for_test}; + use jsonrpc_core::serde_json::{self, Value}; use std::collections::HashSet; use std::iter::FromIterator; use std::sync::Arc; #[test] fn parse_block_triggers_every_block() { + let block = create_minimal_block_for_test(2, hash(2)); + let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(Block { - hash: Some(hash(2)), - number: Some(U64::from(2)), - ..Default::default() - }), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2238,15 +2743,121 @@ mod tests { ); } + #[graph::test] + async fn test_check_block_receipts_support() { + let json_receipts = r#"[{ + "blockHash": "0x23f785604642e91613881fc3c9d16740ee416e340fd36f3fa2239f203d68fd33", + "blockNumber": "0x12f7f81", + "contractAddress": null, + "cumulativeGasUsed": "0x26f66", + "effectiveGasPrice": "0x140a1bd03", + "from": "0x56fc0708725a65ebb633efdaec931c0600a9face", + "gasUsed": "0x26f66", + "logs": [], + "logsBloom": "0x00000000010000000000000000000000000000000000000000000000040000000000000000000000000008000000000002000000080020000000040000000000000000000000000808000008000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000010000800000000000000000000000000000000000000000000010000000000000000000000000000000000200000000000000000000000000000000000002000000008000000000002000000000000000000000000000000000400000000000000000000000000200000000000000010000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x51c72848c68a965f66fa7a88855f9f7784502a7f", + "transactionHash": "0xabfe9e82d71c843a91251fd1272b0dd80bc0b8d94661e3a42c7bb9e7f55789cf", + "transactionIndex": "0x0", + "type": "0x2" + }]"#; + + let json_empty = r#"[]"#; + + // Helper function to run a single test case + async fn run_test_case( + json_response: &str, + expected_err: Option<&str>, + supports_eip_1898: bool, + call_only: bool, + ) -> Result<(), anyhow::Error> { + let json_value: Value = serde_json::from_str(json_response).unwrap(); + + let asserter = Asserter::new(); + let provider = ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_mocked_client(asserter.clone()); + + asserter.push_success(&json_value); + + let result = check_block_receipt_support( + Arc::new(provider), + B256::ZERO, + supports_eip_1898, + call_only, + ) + .await; + + match expected_err { + Some(err_msg) => match result { + Ok(_) => panic!("Expected error but got Ok"), + Err(e) => { + assert!(e.to_string().contains(err_msg)); + } + }, + None => match result { + Ok(_) => (), + Err(e) => { + eprintln!("Error: {}", e); + panic!("Unexpected error: {}", e); + } + }, + } + Ok(()) + } + + // Test case 1: Valid block receipts + run_test_case(json_receipts, None, true, false) + .await + .unwrap(); + + // Test case 2: Empty block receipts + run_test_case(json_empty, Some("Block receipts are empty"), true, false) + .await + .unwrap(); + + // Test case 3: Null response + run_test_case("null", Some("Block receipts are empty"), true, false) + .await + .unwrap(); + + // Test case 3: Simulating an RPC error + // Note: In the context of this test, we cannot directly simulate an RPC error. + // Instead, we simulate a response that would cause a decoding error, such as an unexpected key("error"). + // The function should handle this as an error case. + run_test_case( + r#"{"error":"RPC Error"}"#, + Some("Error fetching block receipts:"), + true, + false, + ) + .await + .unwrap(); + + // Test case 5: Does not support EIP-1898 + run_test_case( + json_receipts, + Some("Provider does not support EIP 1898"), + false, + false, + ) + .await + .unwrap(); + + // Test case 5: Does not support Call only adapters + run_test_case(json_receipts, Some("Provider is call-only"), true, true) + .await + .unwrap(); + } + #[test] fn parse_block_triggers_specific_call_not_found() { + let block = create_minimal_block_for_test(2, hash(2)); + + #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(Block { - hash: Some(hash(2)), - number: Some(U64::from(2)), - ..Default::default() - }), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2272,13 +2883,12 @@ mod tests { #[test] fn parse_block_triggers_specific_call_found() { + let block = create_minimal_block_for_test(2, hash(2)); + + #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(Block { - hash: Some(hash(2)), - number: Some(U64::from(2)), - ..Default::default() - }), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2305,12 +2915,121 @@ mod tests { ); } + #[test] + fn block_trigger_types_once_only() { + let intervals = HashSet::from_iter(vec![(100, 0)]); + + assert_eq!( + vec![EthereumBlockTriggerType::Start], + block_trigger_types_from_intervals(100, &intervals), + "once rule fires Start at start_block" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(99, &intervals), + "once rule does not fire before start_block" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(101, &intervals), + "once rule does not fire after start_block" + ); + } + + #[test] + fn block_trigger_types_polling_only() { + let intervals = HashSet::from_iter(vec![(100, 10)]); + + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(100, &intervals), + "polling rule fires End at start_block" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(110, &intervals), + "polling rule fires End at start_block + interval" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(105, &intervals), + "polling rule does not fire off-interval" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(90, &intervals), + "polling rule does not fire before start_block" + ); + } + + #[test] + fn block_trigger_types_once_and_polling_same_start_block() { + // A single data source with both a `once` handler and a `polling` handler + // contributes both entries at its start_block. Both must fire at start_block. + let intervals = HashSet::from_iter(vec![(100, 0), (100, 10)]); + + assert_eq!( + vec![ + EthereumBlockTriggerType::Start, + EthereumBlockTriggerType::End, + ], + block_trigger_types_from_intervals(100, &intervals), + "both Start and End should fire when once and polling rules share start_block" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(110, &intervals), + "only polling fires at later interval matches" + ); + } + + #[test] + fn block_trigger_types_cross_datasource_collision() { + // Two data sources: DS-A with once at 100, DS-B with polling every 10 from 50. + // Block 100 satisfies DS-A's once rule and also (100-50) % 10 == 0, so both fire. + let intervals = HashSet::from_iter(vec![(100, 0), (50, 10)]); + + assert_eq!( + vec![ + EthereumBlockTriggerType::Start, + EthereumBlockTriggerType::End, + ], + block_trigger_types_from_intervals(100, &intervals), + "both triggers fire when a once rule and an unrelated polling rule collide" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(60, &intervals), + "only polling fires at a block where only the polling rule matches" + ); + } + + #[test] + fn block_trigger_types_no_match() { + let intervals = HashSet::from_iter(vec![(100, 0), (100, 10)]); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(99, &intervals), + "no triggers when block matches neither rule" + ); + } + + #[test] + fn block_trigger_types_empty_intervals() { + let intervals: HashSet<(i32, i32)> = HashSet::new(); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(100, &intervals), + "empty intervals yields no triggers" + ); + } + fn address(id: u64) -> Address { - Address::from_low_u64_be(id) + Address::left_padding_from(&id.to_be_bytes()) } - fn hash(id: u8) -> H256 { - H256::from([id; 32]) + fn hash(id: u8) -> B256 { + B256::from_slice(&[id; 32]) } fn bytes(value: Vec) -> Bytes { diff --git a/chain/ethereum/src/ingestor.rs b/chain/ethereum/src/ingestor.rs index 94c8c6a7d55..dcbcc42c6f8 100644 --- a/chain/ethereum/src/ingestor.rs +++ b/chain/ethereum/src/ingestor.rs @@ -1,10 +1,19 @@ -use crate::{chain::BlockFinality, EthereumAdapter, EthereumAdapterTrait, ENV_VARS}; +use crate::{ENV_VARS, chain::BlockFinality}; +use crate::{EthereumAdapter, EthereumAdapterTrait as _}; +use async_trait::async_trait; +use futures::future::select_ok; +use graph::blockchain::BlockchainKind; +use graph::blockchain::client::ChainClient; +use graph::components::network_provider::ChainName; +use graph::prelude::alloy::primitives::B256; +use graph::slog::o; +use graph::util::backoff::ExponentialBackoff; use graph::{ blockchain::{BlockHash, BlockIngestor, BlockPtr, IngestorError}, cheap_clone::CheapClone, prelude::{ - async_trait, error, ethabi::ethereum_types::H256, info, tokio, trace, warn, ChainStore, - Error, EthereumBlockWithCalls, Future01CompatExt, LogCode, Logger, + ChainStore, Error, EthereumBlockWithCalls, LogCode, Logger, debug, error, info, tokio, + trace, warn, }, }; use std::{sync::Arc, time::Duration}; @@ -12,33 +21,37 @@ use std::{sync::Arc, time::Duration}; pub struct PollingBlockIngestor { logger: Logger, ancestor_count: i32, - eth_adapter: Arc, + chain_client: Arc>, chain_store: Arc, polling_interval: Duration, - network_name: String, + network_name: ChainName, } impl PollingBlockIngestor { pub fn new( logger: Logger, ancestor_count: i32, - eth_adapter: Arc, + chain_client: Arc>, chain_store: Arc, polling_interval: Duration, - network_name: String, + network_name: ChainName, ) -> Result { Ok(PollingBlockIngestor { logger, ancestor_count, - eth_adapter, + chain_client, chain_store, polling_interval, network_name, }) } - fn cleanup_cached_blocks(&self) { - match self.chain_store.cleanup_cached_blocks(self.ancestor_count) { + async fn cleanup_cached_blocks(&self) { + match self + .chain_store + .cleanup_cached_blocks(self.ancestor_count) + .await + { Ok(Some((min_block, count))) => { if count > 0 { info!( @@ -58,8 +71,12 @@ impl PollingBlockIngestor { } } - async fn do_poll(&self) -> Result<(), IngestorError> { - trace!(self.logger, "BlockIngestor::do_poll"); + async fn do_poll( + &self, + logger: &Logger, + eth_adapter: Arc, + ) -> Result<(), IngestorError> { + trace!(&logger, "BlockIngestor::do_poll"); // Get chain head ptr from store let head_block_ptr_opt = self.chain_store.cheap_clone().chain_head_ptr().await?; @@ -67,7 +84,7 @@ impl PollingBlockIngestor { // To check if there is a new block or not, fetch only the block header since that's cheaper // than the full block. This is worthwhile because most of the time there won't be a new // block, as we expect the poll interval to be much shorter than the block time. - let latest_block = self.latest_block().await?; + let latest_block = self.latest_block(logger, ð_adapter).await?; if let Some(head_block) = head_block_ptr_opt.as_ref() { // If latest block matches head block in store, nothing needs to be done @@ -79,7 +96,7 @@ impl PollingBlockIngestor { // An ingestor might wait or move forward, but it never // wavers and goes back. More seriously, this keeps us from // later trying to ingest a block with the same number again - warn!(self.logger, + warn!(&logger, "Provider went backwards - ignoring this latest block"; "current_block_head" => head_block.number, "latest_block_head" => latest_block.number); @@ -91,7 +108,7 @@ impl PollingBlockIngestor { match head_block_ptr_opt { None => { info!( - self.logger, + &logger, "Downloading latest blocks from Ethereum, this may take a few minutes..." ); } @@ -107,7 +124,7 @@ impl PollingBlockIngestor { }; if distance > 0 { info!( - self.logger, + &logger, "Syncing {} blocks from Ethereum", blocks_needed; "current_block_head" => head_number, @@ -124,7 +141,9 @@ impl PollingBlockIngestor { // Might be a no-op if latest block is one that we have seen. // ingest_blocks will return a (potentially incomplete) list of blocks that are // missing. - let mut missing_block_hash = self.ingest_block(&latest_block.hash).await?; + let mut missing_block_hash = self + .ingest_block(logger, ð_adapter, &latest_block.hash) + .await?; // Repeatedly fetch missing parent blocks, and ingest them. // ingest_blocks will continue to tell us about more missing parent @@ -145,29 +164,25 @@ impl PollingBlockIngestor { // iteration will have at most block number N-1. // - Therefore, the loop will iterate at most ancestor_count times. while let Some(hash) = missing_block_hash { - missing_block_hash = self.ingest_block(&hash).await?; + missing_block_hash = self.ingest_block(logger, ð_adapter, &hash).await?; } Ok(()) } async fn ingest_block( &self, + logger: &Logger, + eth_adapter: &Arc, block_hash: &BlockHash, ) -> Result, IngestorError> { - // TODO: H256::from_slice can panic - let block_hash = H256::from_slice(block_hash.as_slice()); + let block_hash = B256::from_slice(block_hash.as_slice()); // Get the fully populated block - let block = self - .eth_adapter - .block_by_hash(&self.logger, block_hash) - .compat() + let block = eth_adapter + .block_by_hash(logger, block_hash) .await? .ok_or(IngestorError::BlockUnavailable(block_hash))?; - let ethereum_block = self - .eth_adapter - .load_full_block(&self.logger, block) - .await?; + let ethereum_block = eth_adapter.load_full_block(logger, block).await?; // We need something that implements `Block` to store the block; the // store does not care whether the block is final or not @@ -187,44 +202,428 @@ impl PollingBlockIngestor { .await .map(|missing| missing.map(|h256| h256.into())) .map_err(|e| { - error!(self.logger, "failed to update chain head"); + error!(logger, "failed to update chain head"); IngestorError::Unknown(e) }) } - async fn latest_block(&self) -> Result { - self.eth_adapter - .latest_block_header(&self.logger) - .compat() - .await - .map(|block| block.into()) + async fn latest_block( + &self, + logger: &Logger, + eth_adapter: &Arc, + ) -> Result { + eth_adapter.latest_block_ptr(logger).await + } + + /// Executes one polling iteration. On failure delegates to `on_poll_failure` for + /// probe+switch logic. + async fn poll_once( + &self, + providers: &[Arc], + current_provider: &mut Option, + ) { + // Resolve by name; resets to first provider if the tracked one left the list. + let eth_adapter = resolve_provider(providers, current_provider, &self.logger).clone(); + // Pin the name so the next iteration knows which provider is active. + let provider_name = eth_adapter.provider().to_string(); + let logger = self.logger.new(o!("provider" => provider_name.clone())); + *current_provider = Some(provider_name); + + if let Err(err) = self.do_poll(&logger, eth_adapter).await { + error!(logger, "Trying again after block polling failed: {}", err); + on_poll_failure(providers, current_provider, &self.logger).await; + } + } +} + +/// Returns the currently-tracked provider from `providers`. +/// +/// If the tracked provider is no longer in the list (it became invalid and was removed by +/// `ProviderManager`), logs a warning, resets the state, and returns the first available +/// provider. +fn resolve_provider<'a, A: crate::EthereumAdapterTrait>( + providers: &'a [Arc], + current_provider: &mut Option, + logger: &Logger, +) -> &'a Arc { + if let Some(name) = current_provider.as_ref() { + if let Some(found) = providers.iter().find(|p| p.provider() == name) { + return found; + } + warn!( + logger, + "Current RPC provider is no longer available, resetting to first provider"; + "provider" => name, + ); + *current_provider = None; + } + &providers[0] +} + +async fn on_poll_failure( + providers: &[Arc], + current_provider: &mut Option, + logger: &Logger, +) { + if providers.len() <= 1 { + return; + } + + let current_name = match current_provider.as_ref() { + Some(name) => name.clone(), + None => return, + }; + + // Probe the current provider before trying alternatives. do_poll() can fail for + // reasons unrelated to RPC availability (e.g. DB errors from chain_head_ptr or + // attempt_chain_head_update, or a BlockUnavailable from a chain reorg). All of + // these surface as IngestorError::Unknown, indistinguishable from an RPC failure + // at the match level. If the current provider responds to eth_blockNumber, the + // failure was not caused by provider unavailability — switching cannot help. + let current = providers.iter().find(|p| p.provider() == current_name); + if let Some(current) = current + && current.is_reachable().await + { + return; + } + + // Probe all alternatives in parallel; switch to the first that responds. + let futs = providers + .iter() + .filter(|p| p.provider() != current_name) + .map(|p| { + let name = p.provider().to_string(); + debug!(logger, "Trying RPC provider"; "provider" => &name); + Box::pin(async move { + if p.is_reachable().await { + Ok(name) + } else { + Err(()) + } + }) + }); + + match select_ok(futs).await { + Ok((next_name, _)) => { + warn!( + logger, + "Switching RPC provider for block ingestor"; + "from" => ¤t_name, + "to" => &next_name, + ); + *current_provider = Some(next_name); + } + Err(_) => { + warn!( + logger, + "All RPC providers unreachable, continuing on current provider"; + "provider" => ¤t_name, + ); + } } } #[async_trait] impl BlockIngestor for PollingBlockIngestor { async fn run(self: Box) { + let mut backoff = + ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); + // Name of the provider currently in use. `None` until the first poll. + let mut current_provider: Option = None; + loop { - match self.do_poll().await { - // Some polls will fail due to transient issues - Err(err) => { - error!( - self.logger, - "Trying again after block polling failed: {}", err - ); - } - Ok(()) => (), + let providers = self + .chain_client + .rpc() + .expect("PollingBlockIngestor is only created for RPC chains") + .all_cheapest() + .await; + + if providers.is_empty() { + error!(self.logger, "No RPC providers available for block ingestor"); + backoff.sleep_async().await; + continue; } + backoff.reset(); + + self.poll_once(&providers, &mut current_provider).await; if ENV_VARS.cleanup_blocks { - self.cleanup_cached_blocks() + self.cleanup_cached_blocks().await; } tokio::time::sleep(self.polling_interval).await; } } - fn network_name(&self) -> String { + fn network_name(&self) -> ChainName { self.network_name.clone() } + + fn kind(&self) -> BlockchainKind { + BlockchainKind::Ethereum + } +} + +#[cfg(test)] +mod tests { + use super::on_poll_failure; + use super::*; + use crate::adapter::{ + ContractCallError, EthereumAdapter as EthereumAdapterTrait, EthereumRpcError, + }; + use async_trait::async_trait; + use graph::blockchain::{BlockPtr, ChainIdentifier}; + use graph::components::ethereum::AnyBlock; + use graph::components::ethereum::LightEthereumBlock; + use graph::data::store::ethereum::call; + use graph::data_source::common::ContractCall; + use graph::prelude::alloy::primitives::{Address, B256, Bytes, U256}; + use graph::prelude::{BlockNumber, Error, EthereumCallCache, Logger}; + use graph::slog::Discard; + use std::collections::HashSet; + use std::sync::Arc; + + struct MockEthAdapter { + provider_name: String, + reachable: bool, + } + + impl MockEthAdapter { + fn new(name: &str, reachable: bool) -> Arc { + Arc::new(Self { + provider_name: name.to_string(), + reachable, + }) + } + } + + #[async_trait] + impl EthereumAdapterTrait for MockEthAdapter { + fn provider(&self) -> &str { + &self.provider_name + } + + async fn is_reachable(&self) -> bool { + self.reachable + } + + async fn net_identifiers(&self) -> Result { + unimplemented!() + } + async fn latest_block_ptr( + &self, + _: &Logger, + ) -> Result { + unimplemented!() + } + async fn load_blocks( + &self, + _: Logger, + _: Arc, + _: HashSet, + ) -> Result>, Error> { + unimplemented!() + } + async fn block_by_hash(&self, _: &Logger, _: B256) -> Result, Error> { + unimplemented!() + } + async fn block_by_number( + &self, + _: &Logger, + _: BlockNumber, + ) -> Result, Error> { + unimplemented!() + } + async fn load_full_block( + &self, + _: &Logger, + _: AnyBlock, + ) -> Result { + unimplemented!() + } + async fn next_existing_ptr_to_number( + &self, + _: &Logger, + _: BlockNumber, + ) -> Result { + unimplemented!() + } + async fn contract_call( + &self, + _: &Logger, + _: &ContractCall, + _: Arc, + ) -> Result<(Option>, call::Source), ContractCallError> + { + unimplemented!() + } + async fn contract_calls( + &self, + _: &Logger, + _: &[&ContractCall], + _: Arc, + ) -> Result>, call::Source)>, ContractCallError> + { + unimplemented!() + } + async fn get_balance( + &self, + _: &Logger, + _: Address, + _: BlockPtr, + ) -> Result { + unimplemented!() + } + async fn get_code( + &self, + _: &Logger, + _: Address, + _: BlockPtr, + ) -> Result { + unimplemented!() + } + } + + fn discard_logger() -> Logger { + Logger::root(Discard, o!()) + } + + #[test] + fn test_current_provider_unavailable_resets_to_first() { + // p0 left the validated list; only p1 and p2 remain (p1 is now at index 0). + let providers: Vec> = vec![ + MockEthAdapter::new("p1", true), + MockEthAdapter::new("p2", true), + ]; + let mut current_provider = Some("p0".to_string()); + let resolved = resolve_provider(&providers, &mut current_provider, &discard_logger()); + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, None); + } + + #[tokio::test] + async fn test_current_reachable_no_switch() { + // Current provider is reachable: on_poll_failure should return early without + // switching, even if alternatives are also reachable. + let providers: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + #[tokio::test] + async fn test_successful_probe_switches_provider() { + let providers: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p1".to_string())); + } + + #[tokio::test] + async fn test_all_unreachable_stays_on_current() { + let providers: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", false), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + #[tokio::test] + async fn test_single_provider_no_switch() { + let providers: Vec> = vec![MockEthAdapter::new("p0", true)]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + // --- Provider-list churn tests --- + // + // These tests simulate the state carried across loop iterations where + // all_cheapest() returns a different subset each time. + + /// Current provider survives a list shrink; its index shifts but identity is preserved. + /// + /// Before: [p0, p1, p2], active = p1 (index 1) + /// After: [p1, p2], active = p1 (index 0) + /// + /// With index-based tracking this would silently move to p2. Name-based tracking + /// must keep us on p1. + #[test] + fn test_list_shrinks_current_provider_index_remaps() { + let providers: Vec> = vec![ + MockEthAdapter::new("p1", true), + MockEthAdapter::new("p2", true), + ]; + let mut current_provider = Some("p1".to_string()); + let resolved = resolve_provider(&providers, &mut current_provider, &discard_logger()); + // p1 is now at index 0 — must not drift to p2 + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, Some("p1".to_string())); + } + + /// After failing over to p1, p1 itself disappears from the validated list in the + /// next iteration. The ingestor must fall back to the first available provider (p0). + /// + /// Iteration 1: [p0(unreachable), p1(reachable)] — p0 fails, switch to p1 + /// Iteration 2: [p0(reachable), p2(reachable)] — p1 gone, reset to p0 + #[tokio::test] + async fn test_failover_target_then_disappears_resets_to_first() { + let logger = discard_logger(); + + // Iteration 1: p0 fails, switch to p1. + let providers_iter1: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers_iter1, &mut current_provider, &logger).await; + assert_eq!(current_provider, Some("p1".to_string())); + + // Iteration 2: p1 has dropped from the validated list. + let providers_iter2: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p2", true), + ]; + let resolved = resolve_provider(&providers_iter2, &mut current_provider, &logger); + assert_eq!(resolved.provider(), "p0"); // reset to first + assert_eq!(current_provider, None); + } + + /// After failing over to p1, the original provider p0 reappears in the list. + /// The ingestor must stay on p1 — there is no proactive return to p0. + /// + /// Iteration 1: [p0(unreachable), p1(reachable)] — switch to p1 + /// Iteration 2: [p0(reachable), p1(reachable)] — p0 is back; must stay on p1 + #[tokio::test] + async fn test_original_provider_reappears_no_involuntary_return() { + let logger = discard_logger(); + + // Iteration 1: switch to p1. + let providers_iter1: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers_iter1, &mut current_provider, &logger).await; + assert_eq!(current_provider, Some("p1".to_string())); + + // Iteration 2: p0 is back in the validated list alongside p1. + let providers_iter2: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p1", true), + ]; + let resolved = resolve_provider(&providers_iter2, &mut current_provider, &logger); + // Must resolve to p1, not drift back to p0. + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, Some("p1".to_string())); + } } diff --git a/chain/ethereum/src/lib.rs b/chain/ethereum/src/lib.rs index cc9c24e79a4..b21eb4717ae 100644 --- a/chain/ethereum/src/lib.rs +++ b/chain/ethereum/src/lib.rs @@ -1,22 +1,29 @@ mod adapter; +mod buffered_call_cache; +mod call_helper; mod capabilities; pub mod codec; mod data_source; mod env; mod ethereum_adapter; mod ingestor; +mod polling_block_stream; pub mod runtime; mod transport; pub use self::capabilities::NodeCapabilities; pub use self::ethereum_adapter::EthereumAdapter; pub use self::runtime::RuntimeAdapter; -pub use self::transport::Transport; +pub use self::transport::{Compression, Transport}; pub use env::ENV_VARS; +pub use buffered_call_cache::BufferedCallCache; + // ETHDEP: These concrete types should probably not be exposed. pub use data_source::{ - BlockHandlerFilter, DataSource, DataSourceTemplate, Mapping, MappingABI, TemplateSource, + BlockHandlerFilter, DataSource, DataSourceTemplate, Mapping, MappingBlockHandler, + MappingCallHandler, TemplateSource, UnresolvedDataSource, UnresolvedDataSourceTemplate, + UnresolvedMapping, UnresolvedMappingEventHandler, }; pub mod chain; @@ -25,11 +32,10 @@ pub mod network; pub mod trigger; pub use crate::adapter::{ - EthereumAdapter as EthereumAdapterTrait, EthereumContractCall, EthereumContractCallError, - ProviderEthRpcMetrics, SubgraphEthRpcMetrics, TriggerFilter, + ContractCallError, EthereumAdapter as EthereumAdapterTrait, ProviderEthRpcMetrics, + SubgraphEthRpcMetrics, TriggerFilter, }; pub use crate::chain::Chain; -pub use crate::network::EthereumNetworks; pub use graph::blockchain::BlockIngestor; #[cfg(test)] diff --git a/chain/ethereum/src/network.rs b/chain/ethereum/src/network.rs index 747a55e8886..ad9cd0a83c0 100644 --- a/chain/ethereum/src/network.rs +++ b/chain/ethereum/src/network.rs @@ -1,19 +1,23 @@ use anyhow::{anyhow, bail}; -use graph::cheap_clone::CheapClone; +use async_trait::async_trait; +use graph::blockchain::ChainIdentifier; +use graph::components::network_provider::ChainName; +use graph::components::network_provider::NetworkDetails; +use graph::components::network_provider::ProviderManager; +use graph::components::network_provider::ProviderName; use graph::endpoint::EndpointMetrics; use graph::firehose::{AvailableCapacity, SubgraphLimit}; use graph::prelude::rand::seq::IteratorRandom; use graph::prelude::rand::{self, Rng}; -use std::cmp::Ordering; -use std::collections::HashMap; +use itertools::Itertools; use std::sync::Arc; pub use graph::impl_slog_value; use graph::prelude::Error; +use crate::EthereumAdapter; use crate::adapter::EthereumAdapter as _; use crate::capabilities::NodeCapabilities; -use crate::EthereumAdapter; pub const DEFAULT_ADAPTER_ERROR_RETEST_PERCENT: f64 = 0.2; @@ -29,7 +33,37 @@ pub struct EthereumNetworkAdapter { limit: SubgraphLimit, } +#[async_trait] +impl NetworkDetails for EthereumNetworkAdapter { + fn provider_name(&self) -> ProviderName { + self.adapter.provider().into() + } + + async fn chain_identifier(&self) -> Result { + self.adapter.net_identifiers().await + } + + async fn provides_extended_blocks(&self) -> Result { + Ok(true) + } +} + impl EthereumNetworkAdapter { + pub fn new( + endpoint_metrics: Arc, + capabilities: NodeCapabilities, + adapter: Arc, + limit: SubgraphLimit, + ) -> Self { + Self { + endpoint_metrics, + capabilities, + adapter, + limit, + } + } + + #[cfg(debug_assertions)] fn is_call_only(&self) -> bool { self.adapter.is_call_only() } @@ -48,64 +82,130 @@ impl EthereumNetworkAdapter { #[derive(Debug, Clone)] pub struct EthereumNetworkAdapters { - pub adapters: Vec, + chain_id: ChainName, + manager: ProviderManager, call_only_adapters: Vec, // Percentage of request that should be used to retest errored adapters. retest_percent: f64, } -impl Default for EthereumNetworkAdapters { - fn default() -> Self { - Self::new(None) - } -} - impl EthereumNetworkAdapters { - pub fn new(retest_percent: Option) -> Self { + pub fn empty_for_testing() -> Self { Self { - adapters: vec![], + chain_id: "".into(), + manager: ProviderManager::default(), call_only_adapters: vec![], - retest_percent: retest_percent.unwrap_or(DEFAULT_ADAPTER_ERROR_RETEST_PERCENT), + retest_percent: DEFAULT_ADAPTER_ERROR_RETEST_PERCENT, } } - pub fn push_adapter(&mut self, adapter: EthereumNetworkAdapter) { - if adapter.is_call_only() { - self.call_only_adapters.push(adapter); - } else { - self.adapters.push(adapter); + #[cfg(debug_assertions)] + pub async fn for_testing( + mut adapters: Vec, + call_only: Vec, + ) -> Self { + use std::cmp::Ordering; + + use graph::components::network_provider::ProviderCheckStrategy; + use graph::slog::{Discard, Logger, o}; + + let chain_id: ChainName = "testing".into(); + adapters.sort_by(|a, b| { + a.capabilities + .partial_cmp(&b.capabilities) + .unwrap_or(Ordering::Equal) + }); + + let provider = ProviderManager::new( + Logger::root(Discard, o!()), + vec![(chain_id.clone(), adapters)], + ProviderCheckStrategy::MarkAsValid, + ); + + Self::new(chain_id, provider, call_only, None) + } + + pub fn new( + chain_id: ChainName, + manager: ProviderManager, + call_only_adapters: Vec, + retest_percent: Option, + ) -> Self { + #[cfg(debug_assertions)] + call_only_adapters.iter().for_each(|a| { + a.is_call_only(); + }); + + Self { + chain_id, + manager, + call_only_adapters, + retest_percent: retest_percent.unwrap_or(DEFAULT_ADAPTER_ERROR_RETEST_PERCENT), } } - pub fn all_cheapest_with( - &self, + + fn available_with_capabilities<'a>( + input: Vec<&'a EthereumNetworkAdapter>, required_capabilities: &NodeCapabilities, - ) -> impl Iterator + '_ { - let cheapest_sufficient_capability = self - .adapters + ) -> impl Iterator + 'a { + let cheapest_sufficient_capability = input .iter() .find(|adapter| &adapter.capabilities >= required_capabilities) .map(|adapter| &adapter.capabilities); - self.adapters - .iter() + input + .into_iter() .filter(move |adapter| Some(&adapter.capabilities) == cheapest_sufficient_capability) .filter(|adapter| adapter.get_capacity() > AvailableCapacity::Unavailable) } - pub fn cheapest_with( + /// returns all the available adapters that meet the required capabilities + /// if no adapters are available at the time or none that meet the capabilities then + /// an empty iterator is returned. + pub async fn all_cheapest_with( &self, required_capabilities: &NodeCapabilities, + ) -> impl Iterator + '_ { + let all = self + .manager + .providers(&self.chain_id) + .await + .map(|adapters| adapters.collect_vec()) + .unwrap_or_default(); + + Self::available_with_capabilities(all, required_capabilities) + } + + // get all the adapters, don't trigger the ProviderManager's validations because we want + // this function to remain sync. If no adapters are available an empty iterator is returned. + pub(crate) fn all_unverified_cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> impl Iterator + '_ { + let all = self + .manager + .providers_unchecked(&self.chain_id) + .collect_vec(); + + Self::available_with_capabilities(all, required_capabilities) + } + + // handle adapter selection from a list, implements the availability checking with an abstracted + // source of the adapter list. + fn cheapest_from( + input: Vec<&EthereumNetworkAdapter>, + required_capabilities: &NodeCapabilities, + retest_percent: f64, ) -> Result, Error> { - let retest_rng: f64 = (&mut rand::thread_rng()).gen(); - let cheapest = self - .all_cheapest_with(required_capabilities) - .choose_multiple(&mut rand::thread_rng(), 3); + let retest_rng: f64 = rand::rng().random(); + + let cheapest = input.into_iter().choose_multiple(&mut rand::rng(), 3); let cheapest = cheapest.iter(); // If request falls below the retest threshold, use this request to try and // reset the failed adapter. If a request succeeds the adapter will be more // likely to be selected afterwards. - if retest_rng < self.retest_percent { + if retest_rng < retest_percent { cheapest.max_by_key(|adapter| adapter.current_error_count()) } else { // The assumption here is that most RPC endpoints will not have limits @@ -123,19 +223,54 @@ impl EthereumNetworkAdapters { )) } - pub fn cheapest(&self) -> Option> { + pub(crate) fn unverified_cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> Result, Error> { + let cheapest = self.all_unverified_cheapest_with(required_capabilities); + + Self::cheapest_from( + cheapest.choose_multiple(&mut rand::rng(), 3), + required_capabilities, + self.retest_percent, + ) + } + + /// This is the public entry point and should always use verified adapters + pub async fn cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> Result, Error> { + let cheapest = self + .all_cheapest_with(required_capabilities) + .await + .choose_multiple(&mut rand::rng(), 3); + + Self::cheapest_from(cheapest, required_capabilities, self.retest_percent) + } + + /// Returns all validated providers. Unvalidated providers are excluded. + pub async fn all_cheapest(&self) -> Vec> { + self.manager + .providers(&self.chain_id) + .await + .map(|adapters| adapters.map(|a| a.adapter.clone()).collect()) + .unwrap_or_default() + } + + pub async fn cheapest(&self) -> Option> { // EthereumAdapters are sorted by their NodeCapabilities when the EthereumNetworks // struct is instantiated so they do not need to be sorted here - self.adapters - .first() + self.manager + .providers(&self.chain_id) + .await + .map(|mut adapters| adapters.next()) + .unwrap_or_default() .map(|ethereum_network_adapter| ethereum_network_adapter.adapter.clone()) } - pub fn remove(&mut self, provider: &str) { - self.adapters - .retain(|adapter| adapter.adapter.provider() != provider); - } - + /// call_or_cheapest will bypass ProviderManagers' validation in order to remain non async. + /// ideally this should only be called for already validated providers. pub fn call_or_cheapest( &self, capabilities: Option<&NodeCapabilities>, @@ -145,11 +280,13 @@ impl EthereumNetworkAdapters { // so we will ignore this error and return whatever comes out of `cheapest_with` match self.call_only_adapter() { Ok(Some(adapter)) => Ok(adapter), - _ => self.cheapest_with(capabilities.unwrap_or(&NodeCapabilities { - // Archive is required for call_only - archive: true, - traces: false, - })), + _ => { + self.unverified_cheapest_with(capabilities.unwrap_or(&NodeCapabilities { + // Archive is required for call_only + archive: true, + traces: false, + })) + } } } @@ -179,115 +316,33 @@ impl EthereumNetworkAdapters { } } -#[derive(Clone)] -pub struct EthereumNetworks { - pub metrics: Arc, - pub networks: HashMap, -} - -impl EthereumNetworks { - pub fn new(metrics: Arc) -> EthereumNetworks { - EthereumNetworks { - networks: HashMap::new(), - metrics, - } - } - - pub fn insert_empty(&mut self, name: String) { - self.networks.entry(name).or_default(); - } - - pub fn insert( - &mut self, - name: String, - capabilities: NodeCapabilities, - adapter: Arc, - limit: SubgraphLimit, - ) { - let network_adapters = self.networks.entry(name).or_default(); - - network_adapters.push_adapter(EthereumNetworkAdapter { - capabilities, - adapter, - limit, - endpoint_metrics: self.metrics.cheap_clone(), - }); - } - - pub fn remove(&mut self, name: &str, provider: &str) { - if let Some(adapters) = self.networks.get_mut(name) { - adapters.remove(provider); - } - } - - pub fn extend(&mut self, other_networks: EthereumNetworks) { - self.networks.extend(other_networks.networks); - } - - pub fn flatten(&self) -> Vec<(String, NodeCapabilities, Arc)> { - self.networks - .iter() - .flat_map(|(network_name, network_adapters)| { - network_adapters - .adapters - .iter() - .map(move |network_adapter| { - ( - network_name.clone(), - network_adapter.capabilities, - network_adapter.adapter.clone(), - ) - }) - }) - .collect() - } - - pub fn sort(&mut self) { - for adapters in self.networks.values_mut() { - adapters.adapters.sort_by(|a, b| { - a.capabilities - .partial_cmp(&b.capabilities) - // We can't define a total ordering over node capabilities, - // so incomparable items are considered equal and end up - // near each other. - .unwrap_or(Ordering::Equal) - }) - } - } - - pub fn adapter_with_capabilities( - &self, - network_name: String, - requirements: &NodeCapabilities, - ) -> Result, Error> { - self.networks - .get(&network_name) - .ok_or(anyhow!("network not supported: {}", &network_name)) - .and_then(|adapters| adapters.cheapest_with(requirements)) - } -} - #[cfg(test)] mod tests { + use graph::cheap_clone::CheapClone; + use graph::components::network_provider::ProviderCheckStrategy; + use graph::components::network_provider::ProviderManager; + use graph::components::network_provider::ProviderName; + use graph::data::value::Word; + + use graph::http::HeaderMap; use graph::{ - endpoint::{EndpointMetrics, Provider}, + endpoint::EndpointMetrics, firehose::SubgraphLimit, prelude::MetricsRegistry, - slog::{o, Discard, Logger}, - tokio, + slog::{Discard, Logger, o}, url::Url, }; - use http::HeaderMap; use std::sync::Arc; - use uuid::Uuid; use crate::{ - EthereumAdapter, EthereumAdapterTrait, EthereumNetworks, ProviderEthRpcMetrics, Transport, + Compression, EthereumAdapter, EthereumAdapterTrait, ProviderEthRpcMetrics, Transport, + chain::ChainSettings, }; use super::{EthereumNetworkAdapter, EthereumNetworkAdapters, NodeCapabilities}; #[test] + #[allow(clippy::neg_cmp_op_on_partial_ord)] fn ethereum_capabilities_comparison() { let archive = NodeCapabilities { archive: true, @@ -311,41 +366,40 @@ mod tests { }; // Test all real combinations of capability comparisons - assert_eq!(false, &full >= &archive); - assert_eq!(false, &full >= &traces); - assert_eq!(false, &full >= &archive_traces); - assert_eq!(true, &full >= &full); - assert_eq!(false, &full >= &full_traces); - - assert_eq!(true, &archive >= &archive); - assert_eq!(false, &archive >= &traces); - assert_eq!(false, &archive >= &archive_traces); - assert_eq!(true, &archive >= &full); - assert_eq!(false, &archive >= &full_traces); - - assert_eq!(false, &traces >= &archive); - assert_eq!(true, &traces >= &traces); - assert_eq!(false, &traces >= &archive_traces); - assert_eq!(true, &traces >= &full); - assert_eq!(true, &traces >= &full_traces); - - assert_eq!(true, &archive_traces >= &archive); - assert_eq!(true, &archive_traces >= &traces); - assert_eq!(true, &archive_traces >= &archive_traces); - assert_eq!(true, &archive_traces >= &full); - assert_eq!(true, &archive_traces >= &full_traces); - - assert_eq!(false, &full_traces >= &archive); - assert_eq!(true, &full_traces >= &traces); - assert_eq!(false, &full_traces >= &archive_traces); - assert_eq!(true, &full_traces >= &full); - assert_eq!(true, &full_traces >= &full_traces); + assert!(!(full >= archive)); + assert!(!(full >= traces)); + assert!(!(full >= archive_traces)); + assert!(full >= full); + assert!(!(full >= full_traces)); + + assert!(archive >= archive); + assert!(!(archive >= traces)); + assert!(!(archive >= archive_traces)); + assert!(archive >= full); + assert!(!(archive >= full_traces)); + + assert!(!(traces >= archive)); + assert!(traces >= traces); + assert!(!(traces >= archive_traces)); + assert!(traces >= full); + assert!(traces >= full_traces); + + assert!(archive_traces >= archive); + assert!(archive_traces >= traces); + assert!(archive_traces >= archive_traces); + assert!(archive_traces >= full); + assert!(archive_traces >= full_traces); + + assert!(!(full_traces >= archive)); + assert!(full_traces >= traces); + assert!(!(full_traces >= archive_traces)); + assert!(full_traces >= full); + assert!(full_traces >= full_traces); } - #[tokio::test] + #[graph::test] async fn adapter_selector_selects_eth_call() { let metrics = Arc::new(EndpointMetrics::mock()); - let chain = "mainnet".to_string(); let logger = graph::log::logger(true); let mock_registry = Arc::new(MetricsRegistry::mock()); let transport = Transport::new_rpc( @@ -353,6 +407,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -364,6 +420,7 @@ mod tests { provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -376,44 +433,47 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let mut adapters = { - let mut ethereum_networks = EthereumNetworks::new(metrics); - ethereum_networks.insert( - chain.clone(), + let mut adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, - eth_call_adapter.clone(), + eth_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.insert( - chain.clone(), + )], + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, - eth_adapter.clone(), + eth_call_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + )], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_call_adapter), 2); assert_eq!(Arc::strong_count(ð_adapter), 2); { // Not Found - assert!(adapters - .cheapest_with(&NodeCapabilities { - archive: false, - traces: true, - }) - .is_err()); + assert!( + adapters + .cheapest_with(&NodeCapabilities { + archive: false, + traces: true, + }) + .await + .is_err() + ); // Check cheapest is not call only let adapter = adapters @@ -421,18 +481,16 @@ mod tests { archive: true, traces: false, }) + .await .unwrap(); - assert_eq!(adapter.is_call_only(), false); + assert!(!adapter.is_call_only()); } // Check limits { let adapter = adapters.call_or_cheapest(None).unwrap(); assert!(adapter.is_call_only()); - assert_eq!( - adapters.call_or_cheapest(None).unwrap().is_call_only(), - false - ); + assert!(!adapters.call_or_cheapest(None).unwrap().is_call_only()); } // Check empty falls back to call only @@ -444,14 +502,13 @@ mod tests { traces: false, })) .unwrap(); - assert_eq!(adapter.is_call_only(), false); + assert!(!adapter.is_call_only()); } } - #[tokio::test] + #[graph::test] async fn adapter_selector_unlimited() { let metrics = Arc::new(EndpointMetrics::mock()); - let chain = "mainnet".to_string(); let logger = graph::log::logger(true); let mock_registry = Arc::new(MetricsRegistry::mock()); let transport = Transport::new_rpc( @@ -459,6 +516,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -470,6 +529,7 @@ mod tests { provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -482,47 +542,48 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let adapters = { - let mut ethereum_networks = EthereumNetworks::new(metrics); - ethereum_networks.insert( - chain.clone(), + let adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_call_adapter.clone(), SubgraphLimit::Unlimited, - ); - ethereum_networks.insert( - chain.clone(), + )], + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_adapter.clone(), - SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + SubgraphLimit::Limit(2), + )], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_call_adapter), 2); assert_eq!(Arc::strong_count(ð_adapter), 2); - let keep: Vec> = vec![0; 10] + // verify that after all call_only were exhausted, we can still + // get normal adapters + let keep: Vec> = [0; 10] .iter() .map(|_| adapters.call_or_cheapest(None).unwrap()) .collect(); - assert_eq!(keep.iter().any(|a| !a.is_call_only()), false); + assert!(!keep.iter().any(|a| !a.is_call_only())); } - #[tokio::test] + #[graph::test] async fn adapter_selector_disable_call_only_fallback() { let metrics = Arc::new(EndpointMetrics::mock()); - let chain = "mainnet".to_string(); let logger = graph::log::logger(true); let mock_registry = Arc::new(MetricsRegistry::mock()); let transport = Transport::new_rpc( @@ -530,6 +591,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -541,6 +604,7 @@ mod tests { provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -553,45 +617,41 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let adapters = { - let mut ethereum_networks = EthereumNetworks::new(metrics); - ethereum_networks.insert( - chain.clone(), + let adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_call_adapter.clone(), SubgraphLimit::Disabled, - ); - ethereum_networks.insert( - chain.clone(), + )], + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + )], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_call_adapter), 2); assert_eq!(Arc::strong_count(ð_adapter), 2); - assert_eq!( - adapters.call_or_cheapest(None).unwrap().is_call_only(), - false - ); + assert!(!adapters.call_or_cheapest(None).unwrap().is_call_only()); } - #[tokio::test] + #[graph::test] async fn adapter_selector_no_call_only_fallback() { let metrics = Arc::new(EndpointMetrics::mock()); - let chain = "mainnet".to_string(); let logger = graph::log::logger(true); let mock_registry = Arc::new(MetricsRegistry::mock()); let transport = Transport::new_rpc( @@ -599,6 +659,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -610,64 +672,59 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let adapters = { - let mut ethereum_networks = EthereumNetworks::new(metrics); - ethereum_networks.insert( - chain.clone(), + let adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + )], + vec![], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_adapter), 2); - assert_eq!( - adapters.call_or_cheapest(None).unwrap().is_call_only(), - false - ); + assert!(!adapters.call_or_cheapest(None).unwrap().is_call_only()); } - #[tokio::test] + #[graph::test] async fn eth_adapter_selection_multiple_adapters() { let logger = Logger::root(Discard, o!()); - let unavailable_provider = Uuid::new_v4().to_string(); - let error_provider = Uuid::new_v4().to_string(); - let no_error_provider = Uuid::new_v4().to_string(); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; let mock_registry = Arc::new(MetricsRegistry::mock()); let metrics = Arc::new(EndpointMetrics::new( logger, - &[ - unavailable_provider.clone(), - error_provider.clone(), - no_error_provider.clone(), - ], + &[unavailable_provider, error_provider, no_error_provider], mock_registry.clone(), )); let logger = graph::log::logger(true); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); + let chain_id: Word = "chain_id".into(); - let adapters = vec![ + let adapters = [ fake_adapter( &logger, - &unavailable_provider, + unavailable_provider, &provider_metrics, &metrics, false, ) .await, - fake_adapter(&logger, &error_provider, &provider_metrics, &metrics, false).await, + fake_adapter(&logger, error_provider, &provider_metrics, &metrics, false).await, fake_adapter( &logger, - &no_error_provider, + no_error_provider, &provider_metrics, &metrics, false, @@ -676,10 +733,11 @@ mod tests { ]; // Set errors - metrics.report_for_test(&Provider::from(error_provider.clone()), false); + metrics.report_for_test(&ProviderName::from(error_provider), false); + + let mut no_retest_adapters = vec![]; + let mut always_retest_adapters = vec![]; - let mut no_retest_adapters = EthereumNetworkAdapters::new(Some(0f64)); - let mut always_retest_adapters = EthereumNetworkAdapters::new(Some(1f64)); adapters.iter().cloned().for_each(|adapter| { let limit = if adapter.provider() == unavailable_provider { SubgraphLimit::Disabled @@ -687,7 +745,7 @@ mod tests { SubgraphLimit::Unlimited }; - no_retest_adapters.adapters.push(EthereumNetworkAdapter { + no_retest_adapters.push(EthereumNetworkAdapter { endpoint_metrics: metrics.clone(), capabilities: NodeCapabilities { archive: true, @@ -696,18 +754,35 @@ mod tests { adapter: adapter.clone(), limit: limit.clone(), }); - always_retest_adapters - .adapters - .push(EthereumNetworkAdapter { - endpoint_metrics: metrics.clone(), - capabilities: NodeCapabilities { - archive: true, - traces: false, - }, - adapter, - limit, - }); + always_retest_adapters.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter, + limit, + }); }); + let manager = ProviderManager::::new( + logger, + vec![( + chain_id.clone(), + no_retest_adapters + .iter() + .cloned() + .chain(always_retest_adapters.iter().cloned()) + .collect(), + )] + .into_iter(), + ProviderCheckStrategy::MarkAsValid, + ); + + let no_retest_adapters = + EthereumNetworkAdapters::new(chain_id.clone(), manager.clone(), vec![], Some(0f64)); + + let always_retest_adapters = + EthereumNetworkAdapters::new(chain_id, manager.clone(), vec![], Some(1f64)); assert_eq!( no_retest_adapters @@ -715,6 +790,7 @@ mod tests { archive: true, traces: false, }) + .await .unwrap() .provider(), no_error_provider @@ -725,89 +801,111 @@ mod tests { archive: true, traces: false, }) + .await .unwrap() .provider(), error_provider ); } - #[tokio::test] + #[graph::test] async fn eth_adapter_selection_single_adapter() { let logger = Logger::root(Discard, o!()); - let unavailable_provider = Uuid::new_v4().to_string(); - let error_provider = Uuid::new_v4().to_string(); - let no_error_provider = Uuid::new_v4().to_string(); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; let mock_registry = Arc::new(MetricsRegistry::mock()); let metrics = Arc::new(EndpointMetrics::new( logger, - &[ - unavailable_provider, - error_provider.clone(), - no_error_provider.clone(), - ], + &[unavailable_provider, error_provider, no_error_provider], mock_registry.clone(), )); + let chain_id: Word = "chain_id".into(); let logger = graph::log::logger(true); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); // Set errors - metrics.report_for_test(&Provider::from(error_provider.clone()), false); + metrics.report_for_test(&ProviderName::from(error_provider), false); - let mut no_retest_adapters = EthereumNetworkAdapters::new(Some(0f64)); - no_retest_adapters.adapters.push(EthereumNetworkAdapter { + let mut no_retest_adapters = vec![]; + no_retest_adapters.push(EthereumNetworkAdapter { endpoint_metrics: metrics.clone(), capabilities: NodeCapabilities { archive: true, traces: false, }, - adapter: fake_adapter(&logger, &error_provider, &provider_metrics, &metrics, false) + adapter: fake_adapter(&logger, error_provider, &provider_metrics, &metrics, false) .await, limit: SubgraphLimit::Unlimited, }); + + let mut always_retest_adapters = vec![]; + always_retest_adapters.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter: fake_adapter( + &logger, + no_error_provider, + &provider_metrics, + &metrics, + false, + ) + .await, + limit: SubgraphLimit::Unlimited, + }); + let manager = ProviderManager::::new( + logger.clone(), + always_retest_adapters + .iter() + .cloned() + .map(|a| (chain_id.clone(), vec![a])), + ProviderCheckStrategy::MarkAsValid, + ); + + let always_retest_adapters = + EthereumNetworkAdapters::new(chain_id.clone(), manager.clone(), vec![], Some(1f64)); + assert_eq!( - no_retest_adapters + always_retest_adapters .cheapest_with(&NodeCapabilities { archive: true, traces: false, }) + .await .unwrap() .provider(), - error_provider + no_error_provider ); - let mut always_retest_adapters = EthereumNetworkAdapters::new(Some(1f64)); - always_retest_adapters - .adapters - .push(EthereumNetworkAdapter { - endpoint_metrics: metrics.clone(), - capabilities: NodeCapabilities { - archive: true, - traces: false, - }, - adapter: fake_adapter( - &logger, - &no_error_provider, - &provider_metrics, - &metrics, - false, - ) - .await, - limit: SubgraphLimit::Unlimited, - }); + let manager = ProviderManager::::new( + logger.clone(), + no_retest_adapters + .iter() + .cloned() + .map(|a| (chain_id.clone(), vec![a])), + ProviderCheckStrategy::MarkAsValid, + ); + + let no_retest_adapters = + EthereumNetworkAdapters::new(chain_id.clone(), manager, vec![], Some(0f64)); assert_eq!( - always_retest_adapters + no_retest_adapters .cheapest_with(&NodeCapabilities { archive: true, traces: false, }) + .await .unwrap() .provider(), - no_error_provider + error_provider ); - let mut no_available_adapter = EthereumNetworkAdapters::default(); - no_available_adapter.adapters.push(EthereumNetworkAdapter { + let mut no_available_adapter = vec![]; + no_available_adapter.push(EthereumNetworkAdapter { endpoint_metrics: metrics.clone(), capabilities: NodeCapabilities { archive: true, @@ -815,7 +913,7 @@ mod tests { }, adapter: fake_adapter( &logger, - &no_error_provider, + no_error_provider, &provider_metrics, &metrics, false, @@ -823,10 +921,19 @@ mod tests { .await, limit: SubgraphLimit::Disabled, }); - let res = no_available_adapter.cheapest_with(&NodeCapabilities { - archive: true, - traces: false, - }); + let manager = ProviderManager::new( + logger, + vec![(chain_id.clone(), no_available_adapter.to_vec())].into_iter(), + ProviderCheckStrategy::MarkAsValid, + ); + + let no_available_adapter = EthereumNetworkAdapters::new(chain_id, manager, vec![], None); + let res = no_available_adapter + .cheapest_with(&NodeCapabilities { + archive: true, + traces: false, + }) + .await; assert!(res.is_err(), "{:?}", res); } @@ -838,10 +945,12 @@ mod tests { call_only: bool, ) -> Arc { let transport = Transport::new_rpc( - Url::parse(&"http://127.0.0.1").unwrap(), + Url::parse("http://127.0.0.1").unwrap(), HeaderMap::new(), endpoint_metrics.clone(), "", + false, + Compression::None, ); Arc::new( @@ -852,6 +961,7 @@ mod tests { provider_metrics.clone(), true, call_only, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ) diff --git a/graph/src/blockchain/polling_block_stream.rs b/chain/ethereum/src/polling_block_stream.rs similarity index 84% rename from graph/src/blockchain/polling_block_stream.rs rename to chain/ethereum/src/polling_block_stream.rs index a9c52294c18..339386207f1 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/chain/ethereum/src/polling_block_stream.rs @@ -1,5 +1,4 @@ -use anyhow::Error; -use futures03::{stream::Stream, Future, FutureExt}; +use anyhow::{Error, anyhow}; use std::cmp; use std::collections::VecDeque; use std::pin::Pin; @@ -7,23 +6,24 @@ use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; -use super::block_stream::{ - BlockStream, BlockStreamEvent, BlockWithTriggers, ChainHeadUpdateStream, FirehoseCursor, - TriggersAdapter, BUFFERED_BLOCK_STREAM_SIZE, +use graph::blockchain::block_stream::{ + BUFFERED_BLOCK_STREAM_SIZE, BlockStream, BlockStreamError, BlockStreamEvent, BlockWithTriggers, + ChainHeadUpdateStream, FirehoseCursor, TriggersAdapterWrapper, }; -use super::{Block, BlockPtr, Blockchain}; +use graph::blockchain::{Block, BlockPtr, TriggerFilterWrapper}; +use graph::futures03::{Future, FutureExt, stream::Stream}; +use graph::prelude::{BLOCK_NUMBER_MAX, DeploymentHash}; +use graph::slog::{Logger, info, trace, warn}; -use crate::components::store::BlockNumber; -use crate::data::subgraph::UnifiedMappingApiVersion; -use crate::prelude::*; +use graph::components::store::BlockNumber; +use graph::data::subgraph::UnifiedMappingApiVersion; + +use crate::Chain; // A high number here forces a slow start. const STARTING_PREVIOUS_TRIGGERS_PER_BLOCK: f64 = 1_000_000.0; -enum BlockStreamState -where - C: Blockchain, -{ +enum BlockStreamState { /// Starting or restarting reconciliation. /// /// Valid next states: Reconciliation @@ -32,13 +32,13 @@ where /// The BlockStream is reconciling the subgraph store state with the chain store state. /// /// Valid next states: YieldingBlocks, Idle, BeginReconciliation (in case of revert) - Reconciliation(Pin, Error>> + Send>>), + Reconciliation(Pin> + Send>>), /// The BlockStream is emitting blocks that must be processed in order to bring the subgraph /// store up to date with the chain store. /// /// Valid next states: BeginReconciliation - YieldingBlocks(Box>>), + YieldingBlocks(VecDeque>), /// The BlockStream experienced an error and is pausing before attempting to produce /// blocks again. @@ -55,16 +55,13 @@ where /// A single next step to take in reconciling the state of the subgraph store with the state of the /// chain store. -enum ReconciliationStep -where - C: Blockchain, -{ +enum ReconciliationStep { /// Revert(to) the block the subgraph should be reverted to, so it becomes the new subgraph /// head. Revert(BlockPtr), /// Move forwards, processing one or more blocks. Second element is the block range size. - ProcessDescendantBlocks(Vec>, BlockNumber), + ProcessDescendantBlocks(Vec>, BlockNumber), /// This step is a no-op, but we need to check again for a next step. Retry, @@ -74,18 +71,13 @@ where Done, } -struct PollingBlockStreamContext -where - C: Blockchain, -{ - chain_store: Arc, - adapter: Arc>, - node_id: NodeId, +struct PollingBlockStreamContext { + adapter: Arc>, subgraph_id: DeploymentHash, // This is not really a block number, but the (unsigned) difference // between two block numbers reorg_threshold: BlockNumber, - filter: Arc, + filter: Arc>, start_blocks: Vec, logger: Logger, previous_triggers_per_block: f64, @@ -98,12 +90,10 @@ where current_block: Option, } -impl Clone for PollingBlockStreamContext { +impl Clone for PollingBlockStreamContext { fn clone(&self) -> Self { Self { - chain_store: self.chain_store.cheap_clone(), adapter: self.adapter.clone(), - node_id: self.node_id.clone(), subgraph_id: self.subgraph_id.clone(), reorg_threshold: self.reorg_threshold, filter: self.filter.clone(), @@ -119,37 +109,29 @@ impl Clone for PollingBlockStreamContext { } } -pub struct PollingBlockStream { - state: BlockStreamState, +pub struct PollingBlockStream { + state: BlockStreamState, consecutive_err_count: u32, chain_head_update_stream: ChainHeadUpdateStream, - ctx: PollingBlockStreamContext, + ctx: PollingBlockStreamContext, } // This is the same as `ReconciliationStep` but without retries. -enum NextBlocks -where - C: Blockchain, -{ +enum NextBlocks { /// Blocks and range size - Blocks(VecDeque>, BlockNumber), + Blocks(VecDeque>, BlockNumber), // The payload is block the subgraph should be reverted to, so it becomes the new subgraph head. Revert(BlockPtr), Done, } -impl PollingBlockStream -where - C: Blockchain, -{ +impl PollingBlockStream { pub fn new( - chain_store: Arc, chain_head_update_stream: ChainHeadUpdateStream, - adapter: Arc>, - node_id: NodeId, + adapter: Arc>, subgraph_id: DeploymentHash, - filter: Arc, + filter: Arc>, start_blocks: Vec, reorg_threshold: BlockNumber, logger: Logger, @@ -164,9 +146,7 @@ where chain_head_update_stream, ctx: PollingBlockStreamContext { current_block: start_block, - chain_store, adapter, - node_id, subgraph_id, reorg_threshold, logger, @@ -182,12 +162,9 @@ where } } -impl PollingBlockStreamContext -where - C: Blockchain, -{ +impl PollingBlockStreamContext { /// Perform reconciliation steps until there are blocks to yield or we are up-to-date. - async fn next_blocks(&self) -> Result, Error> { + async fn next_blocks(&self) -> Result { let ctx = self.clone(); loop { @@ -205,20 +182,20 @@ where return Ok(NextBlocks::Done); } ReconciliationStep::Revert(parent_ptr) => { - return Ok(NextBlocks::Revert(parent_ptr)) + return Ok(NextBlocks::Revert(parent_ptr)); } } } } /// Determine the next reconciliation step. Does not modify Store or ChainStore. - async fn get_next_step(&self) -> Result, Error> { + async fn get_next_step(&self) -> Result { let ctx = self.clone(); let start_blocks = self.start_blocks.clone(); let max_block_range_size = self.max_block_range_size; // Get pointers from database for comparison - let head_ptr_opt = ctx.chain_store.chain_head_ptr().await?; + let head_ptr_opt = ctx.adapter.chain_head_ptr().await?; let subgraph_ptr = self.current_block.clone(); // If chain head ptr is not set yet @@ -247,10 +224,10 @@ where // Only continue if the subgraph block ptr is behind the head block ptr. // subgraph_ptr > head_ptr shouldn't happen, but if it does, it's safest to just stop. - if let Some(ptr) = &subgraph_ptr { - if ptr.number >= head_ptr.number { - return Ok(ReconciliationStep::Done); - } + if let Some(ptr) = &subgraph_ptr + && ptr.number >= head_ptr.number + { + return Ok(ReconciliationStep::Done); } // Subgraph ptr is behind head ptr. @@ -363,22 +340,41 @@ where // 1000 triggers found, 2 per block, range_size = 1000 / 2 = 500 let range_size_upper_limit = max_block_range_size.min(ctx.previous_block_range_size * 10); - let range_size = if ctx.previous_triggers_per_block == 0.0 { + let target_range_size = if ctx.previous_triggers_per_block == 0.0 { range_size_upper_limit } else { (self.target_triggers_per_block_range as f64 / ctx.previous_triggers_per_block) .max(1.0) .min(range_size_upper_limit as f64) as BlockNumber }; - let to = cmp::min(from + range_size - 1, to_limit); + let to = cmp::min(from + target_range_size - 1, to_limit); info!( ctx.logger, "Scanning blocks [{}, {}]", from, to; - "range_size" => range_size + "target_range_size" => target_range_size ); - let blocks = self.adapter.scan_triggers(from, to, &self.filter).await?; + // Update with actually scanned range, to account for any skipped null blocks. + let (blocks, to) = self + .adapter + .scan_triggers(&self.logger, from, to, &self.filter) + .await?; + let range_size = to - from + 1; + + // If the target block (`to`) is within the reorg threshold, indicating no non-null finalized blocks are + // greater than or equal to `to`, we retry later. This deferment allows the chain head to advance, + // ensuring the target block range becomes finalized. It effectively minimizes the risk of chain reorg + // affecting the processing by waiting for a more stable set of blocks. + if to > head_ptr.number - reorg_threshold { + return Ok(ReconciliationStep::Retry); + } + + info!( + ctx.logger, + "Scanned blocks [{}, {}]", from, to; + "range_size" => range_size + ); Ok(ReconciliationStep::ProcessDescendantBlocks( blocks, range_size, @@ -406,6 +402,7 @@ where // block number, and checking to see if the block we found matches the // subgraph_ptr. + #[allow(clippy::unnecessary_unwrap)] let subgraph_ptr = subgraph_ptr.expect("subgraph block pointer should not be `None` here"); @@ -415,7 +412,10 @@ where // In principle this block should be in the store, but we have seen this error for deep // reorgs in ropsten. - let head_ancestor_opt = self.adapter.ancestor_block(head_ptr, offset).await?; + let head_ancestor_opt = self + .adapter + .ancestor_block(head_ptr, offset, Some(subgraph_ptr.hash.clone())) + .await?; match head_ancestor_opt { None => { @@ -427,6 +427,15 @@ where Ok(ReconciliationStep::Retry) } Some(head_ancestor) => { + // Check if there was an interceding skipped (null) block. + if head_ancestor.number() != subgraph_ptr.number + 1 { + warn!( + ctx.logger, + "skipped block detected: {}", + subgraph_ptr.number + 1 + ); + } + // We stopped one block short, so we'll compare the parent hash to the // subgraph ptr. if head_ancestor.parent_hash().as_ref() == Some(&subgraph_ptr.hash) { @@ -463,14 +472,14 @@ where } } -impl BlockStream for PollingBlockStream { +impl BlockStream for PollingBlockStream { fn buffer_size_hint(&self) -> usize { BUFFERED_BLOCK_STREAM_SIZE } } -impl Stream for PollingBlockStream { - type Item = Result, Error>; +impl Stream for PollingBlockStream { + type Item = Result, BlockStreamError>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let result = loop { @@ -503,15 +512,17 @@ impl Stream for PollingBlockStream { total_triggers as f64 / block_range_size as f64; self.ctx.previous_block_range_size = block_range_size; if total_triggers > 0 { - debug!( + info!( self.ctx.logger, - "Processing {} triggers", total_triggers + "Found {} triggers in {} blocks with a block range of {}", + total_triggers, + next_blocks.len(), + block_range_size ); } // Switch to yielding state until next_blocks is depleted - self.state = - BlockStreamState::YieldingBlocks(Box::new(next_blocks)); + self.state = BlockStreamState::YieldingBlocks(next_blocks); // Yield the first block in next_blocks continue; @@ -558,7 +569,7 @@ impl Stream for PollingBlockStream { } // Yielding blocks from reconciliation process - BlockStreamState::YieldingBlocks(ref mut next_blocks) => { + BlockStreamState::YieldingBlocks(next_blocks) => { match next_blocks.pop_front() { // Yield one block Some(next_block) => { @@ -578,7 +589,7 @@ impl Stream for PollingBlockStream { } // Pausing after an error, before looking for more blocks - BlockStreamState::RetryAfterDelay(ref mut delay) => match delay.as_mut().poll(cx) { + BlockStreamState::RetryAfterDelay(delay) => match delay.as_mut().poll(cx) { Poll::Ready(Ok(..)) | Poll::Ready(Err(_)) => { self.state = BlockStreamState::BeginReconciliation; } @@ -599,8 +610,8 @@ impl Stream for PollingBlockStream { // Chain head update stream ended Poll::Ready(None) => { // Should not happen - return Poll::Ready(Some(Err(anyhow::anyhow!( - "chain head update stream ended unexpectedly" + return Poll::Ready(Some(Err(BlockStreamError::from( + anyhow::anyhow!("chain head update stream ended unexpectedly"), )))); } @@ -610,6 +621,6 @@ impl Stream for PollingBlockStream { } }; - result + result.map_err(BlockStreamError::from) } } diff --git a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs index 1e6b7841c8d..4c4d8be6c90 100644 --- a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs +++ b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs @@ -1,84 +1,179 @@ -#[allow(clippy::derive_partial_eq_without_eq)] +// This file is @generated by prost-build. +/// Block is the representation of the tracing of a block in the Ethereum +/// blockchain. A block is a collection of \[TransactionTrace\] that are grouped +/// together and processed as an atomic unit. Each \[TransactionTrace\] is composed +/// of a series of \[Call\] (a.k.a internal transactions) and there is also at +/// least one call per transaction a.k.a the root call which essentially has the +/// same parameters as the transaction itself (e.g. `from`, `to`, `gas`, `value`, +/// etc.). +/// +/// The exact tracing method used to build the block must be checked against +/// \[DetailLevel\] field. There is two levels of details available, `BASE` and +/// `EXTENDED`. The `BASE` level has been extracted using archive node RPC calls +/// and will contain only the block header, transaction receipts and event logs. +/// Refers to the Firehose service provider to know which blocks are offered on +/// each network. +/// +/// The `EXTENDED` level has been extracted using the Firehose tracer and all +/// fields are available in this Protobuf. +/// +/// The Ethereum block model is used across many chains which means that it +/// happen that certain fields are not available in one chain but are available +/// in another. Each field should be documented when necesssary if it's available +/// on a subset of chains. +/// +/// One major concept to get about the Block is the concept of 'ordinal'. The +/// ordinal is a number that is used to globally order every element of execution +/// that happened throughout the processing of the block like +/// \[TransactionTracer\], \[Call\], \[Log\], \[BalanceChange\], \[StateChange\], etc. +/// Element that have a start and end interval, \[Transaction\] and \[Call\], will +/// have two ordinals: `begin_ordinal` and `end_ordinal`. Element that are +/// executed as "point in time" \[Log\], \[BalanceChange\], \[StateChange\], etc. will +/// have only one ordinal named `ordinal`. If you take all of the message in the +/// Block that have an 'ordinal' field in an array and you sort each element +/// against the `ordinal` field, you will get the exact order of execution of +/// each element in the block. +/// +/// All the 'ordinal' fields in a block are globally unique for the given block, +/// it is **not** a chain-wide global ordering. Furthermore, caution must be take +/// with reverted elements due to execution failure. For anything attached to a +/// \[Call\] that has a `state_reverted` field set to `true`, the `ordinal` field +/// is not reliable and should not be used to order the element against other +/// elements in the block as those element might have 0 as the ordinal. Only +/// successful calls have a reliable `ordinal` field. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { - #[prost(int32, tag = "1")] - pub ver: i32, + /// Hash is the block's hash. #[prost(bytes = "vec", tag = "2")] pub hash: ::prost::alloc::vec::Vec, + /// Number is the block's height at which this block was mined. #[prost(uint64, tag = "3")] pub number: u64, + /// Size is the size in bytes of the RLP encoding of the block according to Ethereum + /// rules. #[prost(uint64, tag = "4")] pub size: u64, + /// Header contain's the block's header information like its parent hash, the merkel root hash + /// and all other information the form a block. #[prost(message, optional, tag = "5")] pub header: ::core::option::Option, - /// Uncles represents block produced with a valid solution but were not actually choosen + /// Uncles represents block produced with a valid solution but were not actually chosen /// as the canonical block for the given height so they are mostly "forked" blocks. /// /// If the Block has been produced using the Proof of Stake consensus algorithm, this /// field will actually be always empty. #[prost(message, repeated, tag = "6")] pub uncles: ::prost::alloc::vec::Vec, + /// TransactionTraces hold the execute trace of all the transactions that were executed + /// in this block. In in there that you will find most of the Ethereum data model. + /// + /// They are ordered by the order of execution of the transaction in the block. #[prost(message, repeated, tag = "10")] pub transaction_traces: ::prost::alloc::vec::Vec, + /// BalanceChanges here is the array of ETH transfer that happened at the block level + /// outside of the normal transaction flow of a block. The best example of this is mining + /// reward for the block mined, the transfer of ETH to the miner happens outside the normal + /// transaction flow of the chain and is recorded as a `BalanceChange` here since we cannot + /// attached it to any transaction. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, repeated, tag = "11")] pub balance_changes: ::prost::alloc::vec::Vec, + /// DetailLevel affects the data available in this block. + /// + /// ## DetailLevel_EXTENDED + /// + /// Describes the most complete block, with traces, balance changes, storage + /// changes. It is extracted during the execution of the block. + /// + /// ## DetailLevel_BASE + /// + /// Describes a block that contains only the block header, transaction receipts + /// and event logs: everything that can be extracted using the base JSON-RPC + /// interface + /// () + /// Furthermore, the eth_getTransactionReceipt call has been avoided because it + /// brings only minimal improvements at the cost of requiring an archive node + /// or a full node with complete transaction index. + #[prost(enumeration = "block::DetailLevel", tag = "12")] + pub detail_level: i32, + /// CodeChanges here is the array of smart code change that happened that happened at the block level + /// outside of the normal transaction flow of a block. Some Ethereum's fork like BSC and Polygon + /// has some capabilities to upgrade internal smart contracts used usually to track the validator + /// list. + /// + /// On hard fork, some procedure runs to upgrade the smart contract code to a new version. In those + /// network, a `CodeChange` for each modified smart contract on upgrade would be present here. Note + /// that this happen rarely, so the vast majority of block will have an empty list here. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, repeated, tag = "20")] pub code_changes: ::prost::alloc::vec::Vec, + /// System calls are introduced in Cancun, along with blobs. They are executed outside of transactions but affect the state. + /// + /// Only available in DetailLevel: EXTENDED + #[prost(message, repeated, tag = "21")] + pub system_calls: ::prost::alloc::vec::Vec, + /// Withdrawals represents the list of validator balance withdrawals processed in this block. + /// Introduced in the Shanghai hard fork (EIP-4895). + /// + /// This field has been added because Geth blocks include withdrawals after Shanghai fork, + /// but our previous Firehose model didn't capture this data. Currently experimental - + /// NOT ready for production use yet as we validate the tracing implementation. + /// + /// Only available when Shanghai fork is active on the chain. + #[prost(message, repeated, tag = "22")] + pub withdrawals: ::prost::alloc::vec::Vec, + /// Ver represents that data model version of the block, it is used internally by Firehose on Ethereum + /// as a validation that we are reading the correct version. + #[prost(int32, tag = "1")] + pub ver: i32, } -/// HeaderOnlyBlock is used to optimally unpack the \[Block\] structure (note the -/// corresponding message number for the `header` field) while consuming less -/// memory, when only the `header` is desired. -/// -/// WARN: this is a client-side optimization pattern and should be moved in the -/// consuming code. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag = "5")] - pub header: ::core::option::Option, -} -/// BlockWithRefs is a lightweight block, with traces and transactions -/// purged from the `block` within, and only. It is used in transports -/// to pass block data around. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockWithRefs { - #[prost(string, tag = "1")] - pub id: ::prost::alloc::string::String, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub transaction_trace_refs: ::core::option::Option, - #[prost(bool, tag = "4")] - pub irreversible: bool, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionRefs { - #[prost(bytes = "vec", repeated, tag = "1")] - pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct UnclesHeaders { - #[prost(message, repeated, tag = "1")] - pub uncles: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockRef { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "2")] - pub number: u64, +/// Nested message and enum types in `Block`. +pub mod block { + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum DetailLevel { + DetaillevelExtended = 0, + /// DETAILLEVEL_TRACE = 1; // TBD + DetaillevelBase = 2, + } + impl DetailLevel { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::DetaillevelExtended => "DETAILLEVEL_EXTENDED", + Self::DetaillevelBase => "DETAILLEVEL_BASE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "DETAILLEVEL_EXTENDED" => Some(Self::DetaillevelExtended), + "DETAILLEVEL_BASE" => Some(Self::DetaillevelBase), + _ => None, + } + } + } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockHeader { #[prost(bytes = "vec", tag = "1")] pub parent_hash: ::prost::alloc::vec::Vec, - /// Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also - /// referred as `ommers` in EIP specification. + /// Uncle hash of the block, some reference it as `sha3Uncles`, but ```sha3`` is badly worded, so we prefer ```uncle_hash`, also referred as `ommers\` in EIP specification. /// /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake /// consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. @@ -100,12 +195,10 @@ pub struct BlockHeader { /// consensus algorithm, this field will actually be constant and set to `0x00`. #[prost(message, optional, tag = "8")] pub difficulty: ::core::option::Option, - /// TotalDifficulty is the sum of all previous blocks difficulty including this block difficulty. + /// TotalDifficulty used to be the sum of all previous blocks difficulty including this block difficulty. /// - /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake - /// consensus algorithm, this field will actually be constant and set to the terminal total difficulty - /// that was required to transition to Proof of Stake algorithm, which varies per network. It is set to - /// 58 750 000 000 000 000 000 000 on Ethereum Mainnet and to 10 790 000 on Ethereum Testnet Goerli. + /// It has been deprecated in geth v1.15.0 but was already removed from the JSON-RPC interface for a while + #[deprecated] #[prost(message, optional, tag = "17")] pub total_difficulty: ::core::option::Option, #[prost(uint64, tag = "9")] @@ -121,7 +214,7 @@ pub struct BlockHeader { /// forks are using bigger values to carry special consensus data. /// /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake - /// consensus algorithm, this field is strictly enforced to be <= 32 bytes. + /// consensus algorithm, this field is strictly enforced to be \<= 32 bytes. #[prost(bytes = "vec", tag = "13")] pub extra_data: ::prost::alloc::vec::Vec, /// MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been @@ -137,38 +230,102 @@ pub struct BlockHeader { pub nonce: u64, /// Hash is the hash of the block which is actually the computation: /// - /// Keccak256(rlp([ - /// parent_hash, - /// uncle_hash, - /// coinbase, - /// state_root, - /// transactions_root, - /// receipt_root, - /// logs_bloom, - /// difficulty, - /// number, - /// gas_limit, - /// gas_used, - /// timestamp, - /// extra_data, - /// mix_hash, - /// nonce, - /// base_fee_per_gas - /// ])) - /// + /// Keccak256(rlp(\[ + /// parent_hash, + /// uncle_hash, + /// coinbase, + /// state_root, + /// transactions_root, + /// receipt_root, + /// logs_bloom, + /// difficulty, + /// number, + /// gas_limit, + /// gas_used, + /// timestamp, + /// extra_data, + /// mix_hash, + /// nonce, + /// base_fee_per_gas (to be included only if London fork is active) + /// withdrawals_root (to be included only if Shangai fork is active) + /// blob_gas_used (to be included only if Cancun fork is active) + /// excess_blob_gas (to be included only if Cancun fork is active) + /// parent_beacon_root (to be included only if Cancun fork is active) + /// requests_hash (to be included only if Prague fork is active) + /// \])) #[prost(bytes = "vec", tag = "16")] pub hash: ::prost::alloc::vec::Vec, /// Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. #[prost(message, optional, tag = "18")] pub base_fee_per_gas: ::core::option::Option, + /// Withdrawals root hash according to EIP-4895 (e.g. Shangai Fork) rules, only set if Shangai is present/active on the chain. + /// + /// Only available in DetailLevel: EXTENDED + #[prost(bytes = "vec", tag = "19")] + pub withdrawals_root: ::prost::alloc::vec::Vec, + /// TxDependency is list of transaction indexes that are dependent on each other in the block + /// header. This is metadata only that was used by the internal Polygon parallel execution engine. + /// + /// This field was available in a few versions on Polygon Mainnet and Polygon Mumbai chains. It was actually + /// removed and is not populated anymore. It's now embedded in the `extraData` field, refer to Polygon source + /// code to determine how to extract it if you need it. + /// + /// Only available in DetailLevel: EXTENDED + #[prost(message, optional, tag = "20")] + pub tx_dependency: ::core::option::Option, + /// BlobGasUsed was added by EIP-4844 and is ignored in legacy headers. + #[prost(uint64, optional, tag = "22")] + pub blob_gas_used: ::core::option::Option, + /// ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers. + #[prost(uint64, optional, tag = "23")] + pub excess_blob_gas: ::core::option::Option, + /// ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers. + #[prost(bytes = "vec", tag = "24")] + pub parent_beacon_root: ::prost::alloc::vec::Vec, + /// RequestsHash was added by EIP-7685 and is ignored in legacy headers. + #[prost(bytes = "vec", tag = "25")] + pub requests_hash: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct Uint64NestedArray { + #[prost(message, repeated, tag = "1")] + pub val: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Uint64Array { + #[prost(uint64, repeated, tag = "1")] + pub val: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] +/// TransactionTrace is full trace of execution of the transaction when the +/// it actually executed on chain. +/// +/// It contains all the transaction details like `from`, `to`, `gas`, etc. +/// as well as all the internal calls that were made during the transaction. +/// +/// The `calls` vector contains Call objects which have balance changes, events +/// storage changes, etc. +/// +/// If ordering is important between elements, almost each message like `Log`, +/// `Call`, `StorageChange`, etc. have an ordinal field that is represents "execution" +/// order of the said element against all other elements in this block. +/// +/// Due to how the call tree works doing "naively", looping through all calls then +/// through a Call's element like `logs` while not yielding the elements in the order +/// they were executed on chain. A log in call could have been done before or after +/// another in another call depending on the actual call tree. +/// +/// The `calls` are ordered by creation order and the call tree can be re-computing +/// using fields found in `Call` object (parent/child relationship). +/// +/// Another important thing to note is that even if a transaction succeed, some calls +/// within it could have been reverted internally, if this is important to you, you must +/// check the field `state_reverted` on the `Call` to determine if it was fully committed +/// to the chain or not. #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionTrace { /// consensus @@ -213,7 +370,7 @@ pub struct TransactionTrace { /// The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions. #[prost(enumeration = "transaction_trace::Type", tag = "12")] pub r#type: i32, - /// AcccessList represents the storage access this transaction has agreed to do in which case those storage + /// AccessList represents the storage access this transaction has agreed to do in which case those storage /// access cost less gas unit per access. /// /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which @@ -224,6 +381,8 @@ pub struct TransactionTrace { /// /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only /// if London fork is active on the chain. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, optional, tag = "11")] pub max_fee_per_gas: ::core::option::Option, /// MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's @@ -231,6 +390,8 @@ pub struct TransactionTrace { /// /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only /// if London fork is active on the chain. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, optional, tag = "13")] pub max_priority_fee_per_gas: ::core::option::Option, /// meta @@ -240,20 +401,98 @@ pub struct TransactionTrace { pub hash: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "22")] pub from: ::prost::alloc::vec::Vec, + /// Only available in DetailLevel: EXTENDED + /// Known Issues + /// + /// * Version 3: + /// Field not populated. It will be empty. + /// + /// Fixed in `Version 4`, see for information about block versions. #[prost(bytes = "vec", tag = "23")] pub return_data: ::prost::alloc::vec::Vec, + /// Only available in DetailLevel: EXTENDED #[prost(bytes = "vec", tag = "24")] pub public_key: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the transaction started executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "25")] pub begin_ordinal: u64, + /// The block's global ordinal when the transaction finished executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "26")] pub end_ordinal: u64, + /// TransactionTraceStatus is the status of the transaction execution and will let you know if the transaction + /// was successful or not. + /// + /// ## Explanation relevant only for blocks with `DetailLevel: EXTENDED` + /// + /// A successful transaction has been recorded to the blockchain's state for calls in it that were successful. + /// This means it's possible only a subset of the calls were properly recorded, refer to \[calls\[\].state_reverted\] field + /// to determine which calls were reverted. + /// + /// A quirks of the Ethereum protocol is that a transaction `FAILED` or `REVERTED` still affects the blockchain's + /// state for **some** of the state changes. Indeed, in those cases, the transactions fees are still paid to the miner + /// which means there is a balance change for the transaction's emitter (e.g. `from`) to pay the gas fees, an optional + /// balance change for gas refunded to the transaction's emitter (e.g. `from`) and a balance change for the miner who + /// received the transaction fees. There is also a nonce change for the transaction's emitter (e.g. `from`). + /// + /// This means that to properly record the state changes for a transaction, you need to conditionally procees the + /// transaction's status. + /// + /// For a `SUCCEEDED` transaction, you iterate over the `calls` array and record the state changes for each call for + /// which `state_reverted == false` (if a transaction succeeded, the call at #0 will always `state_reverted == false` + /// because it aligns with the transaction). + /// + /// For a `FAILED` or `REVERTED` transaction, you iterate over the root call (e.g. at #0, will always exist) for + /// balance changes you process those where `reason` is either `REASON_GAS_BUY`, `REASON_GAS_REFUND` or + /// `REASON_REWARD_TRANSACTION_FEE` and for nonce change, still on the root call, you pick the nonce change which the + /// smallest ordinal (if more than one). #[prost(enumeration = "TransactionTraceStatus", tag = "30")] pub status: i32, #[prost(message, optional, tag = "31")] pub receipt: ::core::option::Option, + /// Only available in DetailLevel: EXTENDED #[prost(message, repeated, tag = "32")] pub calls: ::prost::alloc::vec::Vec, + /// BlobGas is the amount of gas the transaction is going to pay for the blobs, this is a computed value + /// equivalent to `self.blob_gas_fee_cap * len(self.blob_hashes)` and provided in the model for convenience. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(uint64, optional, tag = "33")] + pub blob_gas: ::core::option::Option, + /// BlobGasFeeCap is the maximum fee per data gas the user is willing to pay for the data gas used. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(message, optional, tag = "34")] + pub blob_gas_fee_cap: ::core::option::Option, + /// BlobHashes field represents a list of hash outputs from 'kzg_to_versioned_hash' which + /// essentially is a version byte + the sha256 hash of the blob commitment (e.g. + /// `BLOB_COMMITMENT_VERSION_KZG + sha256(commitment)\[1:\]`. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(bytes = "vec", repeated, tag = "35")] + pub blob_hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// SetCodeAuthorizations represents the authorizations of a transaction to set code to an EOA (Externally Owned Accounts) + /// as defined in EIP-7702. The list will contain all the authorizations as they were specified in the + /// transaction itself regardless of their validity. If you need to determined if a given authorization was + /// correctly applied on chain's state, refer to \[SetCodeAuthorization.discarded\] field that records + /// if the authorization was discarded or not by the chain due to invalidity. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_SET_CODE` which is possible only + /// if Prague fork is active on the chain. + #[prost(message, repeated, tag = "36")] + pub set_code_authorizations: ::prost::alloc::vec::Vec, } /// Nested message and enum types in `TransactionTrace`. pub mod transaction_trace { @@ -272,15 +511,38 @@ pub mod transaction_trace { pub enum Type { /// All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. TrxTypeLegacy = 0, - /// Field that specifies an access list of contract/storage_keys that is going to be used + /// Transaction that specicy an access list of contract/storage_keys that is going to be used /// in this transaction. /// /// Added in Berlin fork (EIP-2930). TrxTypeAccessList = 1, - /// Transaction that specifies an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the + /// Transaction that specifis an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the /// max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are /// executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. TrxTypeDynamicFee = 2, + /// Transaction which contain a large amount of data that cannot be accessed by EVM execution, but whose commitment + /// can be accessed. The format is intended to be fully compatible with the format that will be used in full sharding. + /// + /// Transaction that defines an access list just like TRX_TYPE_ACCESS_LIST and enables dynamic fee just like + /// TRX_TYPE_DYNAMIC_FEE but in addition defines the fields 'max_fee_per_data_gas' of type 'uint256' and the fields + /// 'blob_versioned_hashes' which represents a list of hash outputs from 'kzg_to_versioned_hash'. + /// + /// Activated in Cancun fork (EIP-4844) + TrxTypeBlob = 3, + /// Transaction that sets code to an EOA (Externally Owned Accounts) + /// + /// Activated in Prague (EIP-7702) + TrxTypeSetCode = 4, + /// Arbitrum-specific transactions + TrxTypeArbitrumDeposit = 100, + TrxTypeArbitrumUnsigned = 101, + TrxTypeArbitrumContract = 102, + TrxTypeArbitrumRetry = 104, + TrxTypeArbitrumSubmitRetryable = 105, + TrxTypeArbitrumInternal = 106, + TrxTypeArbitrumLegacy = 120, + /// OPTIMISM-specific transactions + TrxTypeOptimismDeposit = 126, } impl Type { /// String value of the enum field names used in the ProtoBuf definition. @@ -289,9 +551,21 @@ pub mod transaction_trace { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Type::TrxTypeLegacy => "TRX_TYPE_LEGACY", - Type::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", - Type::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", + Self::TrxTypeLegacy => "TRX_TYPE_LEGACY", + Self::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", + Self::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", + Self::TrxTypeBlob => "TRX_TYPE_BLOB", + Self::TrxTypeSetCode => "TRX_TYPE_SET_CODE", + Self::TrxTypeArbitrumDeposit => "TRX_TYPE_ARBITRUM_DEPOSIT", + Self::TrxTypeArbitrumUnsigned => "TRX_TYPE_ARBITRUM_UNSIGNED", + Self::TrxTypeArbitrumContract => "TRX_TYPE_ARBITRUM_CONTRACT", + Self::TrxTypeArbitrumRetry => "TRX_TYPE_ARBITRUM_RETRY", + Self::TrxTypeArbitrumSubmitRetryable => { + "TRX_TYPE_ARBITRUM_SUBMIT_RETRYABLE" + } + Self::TrxTypeArbitrumInternal => "TRX_TYPE_ARBITRUM_INTERNAL", + Self::TrxTypeArbitrumLegacy => "TRX_TYPE_ARBITRUM_LEGACY", + Self::TrxTypeOptimismDeposit => "TRX_TYPE_OPTIMISM_DEPOSIT", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -300,6 +574,18 @@ pub mod transaction_trace { "TRX_TYPE_LEGACY" => Some(Self::TrxTypeLegacy), "TRX_TYPE_ACCESS_LIST" => Some(Self::TrxTypeAccessList), "TRX_TYPE_DYNAMIC_FEE" => Some(Self::TrxTypeDynamicFee), + "TRX_TYPE_BLOB" => Some(Self::TrxTypeBlob), + "TRX_TYPE_SET_CODE" => Some(Self::TrxTypeSetCode), + "TRX_TYPE_ARBITRUM_DEPOSIT" => Some(Self::TrxTypeArbitrumDeposit), + "TRX_TYPE_ARBITRUM_UNSIGNED" => Some(Self::TrxTypeArbitrumUnsigned), + "TRX_TYPE_ARBITRUM_CONTRACT" => Some(Self::TrxTypeArbitrumContract), + "TRX_TYPE_ARBITRUM_RETRY" => Some(Self::TrxTypeArbitrumRetry), + "TRX_TYPE_ARBITRUM_SUBMIT_RETRYABLE" => { + Some(Self::TrxTypeArbitrumSubmitRetryable) + } + "TRX_TYPE_ARBITRUM_INTERNAL" => Some(Self::TrxTypeArbitrumInternal), + "TRX_TYPE_ARBITRUM_LEGACY" => Some(Self::TrxTypeArbitrumLegacy), + "TRX_TYPE_OPTIMISM_DEPOSIT" => Some(Self::TrxTypeOptimismDeposit), _ => None, } } @@ -307,36 +593,84 @@ pub mod transaction_trace { } /// AccessTuple represents a list of storage keys for a given contract's address and is used /// for AccessList construction. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AccessTuple { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", repeated, tag = "2")] pub storage_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -/// TransactionTraceWithBlockRef -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionTraceWithBlockRef { - #[prost(message, optional, tag = "1")] - pub trace: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block_ref: ::core::option::Option, +/// SetCodeAuthorization represents the authorization of a transaction to set code of an EOA (Externally Owned Account) +/// as defined in EIP-7702. +/// +/// The 'authority' field is the address that is authorizing the delegation mechanism. The 'authority' value is computed +/// from the signature contained in the message using the computation +/// `authority = ecrecover(keccak(MAGIC || rlp(\[chain_id, address, nonce\])), y_parity, r, s)` +/// where `MAGIC` is `0x5`, `||` is the bytes concatenation operator, `ecrecover` is the Ethereum signature recovery +/// and `y_parity` is the recovery ID value denoted `v` in the message below. Checking the go-ethereum implementation +/// at might prove easier to "read". +/// +/// We do extract the 'authority' value from the signature in the message and store it in the 'authority' field for +/// convenience so you don't need to perform the computation yourself. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SetCodeAuthorization { + /// Discarded determines if this authorization was skipped due to being invalid. As EIP-7702 states, + /// if the authorization is invalid (invalid signature, nonce mismatch, etc.) it must be simply + /// discarded and the transaction is processed as if the authorization was not present in the + /// authorization list. + /// + /// This boolean records if the authorization was discarded or not by the chain due to invalidity. + #[prost(bool, tag = "1")] + pub discarded: bool, + /// ChainID is the chain ID of the chain where the transaction was executed, used + /// to recover the authority from the signature. + #[prost(bytes = "vec", tag = "2")] + pub chain_id: ::prost::alloc::vec::Vec, + /// Address contains the address this account is delegating to. This address usually + /// contain code that this account essentially "delegates" to. + /// + /// Note: This was missing when EIP-7702 was first activated on Holesky, Sepolia, BSC Chapel, + /// BSC Mainnet and Arbitrum Sepolia but was ready for Ethereum Mainnet hard fork. We will backfill + /// those missing values in the near future at which point we will remove this note. + #[prost(bytes = "vec", tag = "8")] + pub address: ::prost::alloc::vec::Vec, + /// Nonce is the nonce of the account that is authorizing delegation mechanism, EIP-7702 rules + /// states that nonce should be verified using this rule: + /// + /// * Verify the nonce of authority is equal to nonce. In case authority does not exist in the trie, + /// verify that nonce is equal to 0. + /// + /// Read SetCodeAuthorization to know how to recover the `authority` value. + #[prost(uint64, tag = "3")] + pub nonce: u64, + /// V is the recovery ID value for the signature Y point. While it's defined as a + /// `uint32`, it's actually bounded by a `uint8` data type withing the Ethereum protocol. + #[prost(uint32, tag = "4")] + pub v: u32, + /// R is the signature's X point on the elliptic curve (32 bytes). + #[prost(bytes = "vec", tag = "5")] + pub r: ::prost::alloc::vec::Vec, + /// S is the signature's Y point on the elliptic curve (32 bytes). + #[prost(bytes = "vec", tag = "6")] + pub s: ::prost::alloc::vec::Vec, + /// Authority is the address of the account that is authorizing delegation mechanism, it + /// is computed from the signature contained in the message and stored for convenience. + /// + /// If the authority cannot be recovered from the signature, this field will be empty and + /// the `discarded` field will be set to `true`. + #[prost(bytes = "vec", optional, tag = "7")] + pub authority: ::core::option::Option<::prost::alloc::vec::Vec>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionReceipt { /// State root is an intermediate state_root hash, computed in-between transactions to make /// **sure** you could build a proof and point to state in the middle of a block. Geth client - /// uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles - /// hardforks, see (read the EIPs first): - /// - - /// - - /// - - /// - /// Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are - /// stored in the same field `status_code`` can be computed based on such a hack of the `state_root` + /// uses ```PostState + root + PostStateOrStatus`` while Parity used ```status_code, root...\`` this piles + /// hard forks, see (read the EIPs first): + /// + /// * + /// + /// Moreover, the notion of ```Outcome`` in parity, which segregates the two concepts, which are stored in the same field ```status_code\`` can be computed based on such a hack of the `state_root` /// field, following `EIP-658`. /// /// Before Byzantinium hard fork, this field is always empty. @@ -348,9 +682,25 @@ pub struct TransactionReceipt { pub logs_bloom: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "4")] pub logs: ::prost::alloc::vec::Vec, + /// BlobGasUsed is the amount of blob gas that has been used within this transaction. At time + /// of writing, this is equal to `self.blob_gas_fee_cap * len(self.blob_hashes)`. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(uint64, optional, tag = "5")] + pub blob_gas_used: ::core::option::Option, + /// BlobGasPrice is the amount to pay per blob item in the transaction. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(message, optional, tag = "6")] + pub blob_gas_price: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Log { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -359,8 +709,10 @@ pub struct Log { #[prost(bytes = "vec", tag = "3")] pub data: ::prost::alloc::vec::Vec, /// Index is the index of the log relative to the transaction. This index - /// is always populated regardless of the state revertion of the the call + /// is always populated regardless of the state reversion of the the call /// that emitted this log. + /// + /// Only available in DetailLevel: EXTENDED #[prost(uint32, tag = "4")] pub index: u32, /// BlockIndex represents the index of the log relative to the Block. @@ -369,8 +721,9 @@ pub struct Log { /// that emitted the log has been reverted by the chain. /// /// Currently, there is two locations where a Log can be obtained: - /// - block.transaction_traces\[].receipt.logs[\] - /// - block.transaction_traces\[].calls[].logs[\] + /// + /// * block.transaction_traces\[\].receipt.logs\[\] + /// * block.transaction_traces\[\].calls\[\].logs\[\] /// /// In the `receipt` case, the logs will be populated only when the call /// that emitted them has not been reverted by the chain and when in this @@ -380,10 +733,11 @@ pub struct Log { /// the `blockIndex` value will always be 0. #[prost(uint32, tag = "6")] pub block_index: u32, + /// The block's global ordinal when the log was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "7")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Call { #[prost(uint32, tag = "1")] @@ -398,6 +752,25 @@ pub struct Call { pub caller: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "6")] pub address: ::prost::alloc::vec::Vec, + /// AddressDelegatesTo contains the address from which the actual code to execute will be loaded + /// as defined per EIP-7702 rules. If the Call's address value resolves to a code + /// that delegates to another address, this field will be populated with the address + /// that the call is delegated to. It will be empty in all other situations. + /// + /// Assumes that a 'SetCode' transaction set address `0xA` to delegates to address `0xB`, + /// then when a call is made to `0xA`, the Call object would have: + /// + /// * caller = + /// * address = 0xA + /// * address_delegates_to = 0xB + /// + /// Again, it's important to emphasize that this field relates to EIP-7702, if the call is + /// a DELEGATE or CALLCODE type, this field will not be populated and will remain empty. + /// + /// It will be populated only if EIP-7702 is active on the chain (Prague fork) and if the + /// 'address' of the call was pointing to another address at time of execution. + #[prost(bytes = "vec", optional, tag = "34")] + pub address_delegates_to: ::core::option::Option<::prost::alloc::vec::Vec>, #[prost(message, optional, tag = "7")] pub value: ::core::option::Option, #[prost(uint64, tag = "8")] @@ -406,8 +779,27 @@ pub struct Call { pub gas_consumed: u64, #[prost(bytes = "vec", tag = "13")] pub return_data: ::prost::alloc::vec::Vec, + /// Known Issues + /// + /// * Version 3: + /// When call is `CREATE` or `CREATE2`, this field is not populated. A couple of suggestions: + /// + /// 1. You can get the contract's code in the `code_changes` field. + /// 1. In the root `CREATE` call, you can directly use the `TransactionTrace`'s input field. + /// Fixed in `Version 4`, see for information about block versions. #[prost(bytes = "vec", tag = "14")] pub input: ::prost::alloc::vec::Vec, + /// Indicates whether the call executed code. + /// + /// Known Issues + /// + /// * Version 3: + /// This may be incorrectly set to `false` for accounts with code handling native value transfers, + /// as well as for certain precompiles with no input. + /// The value is initially set based on `call.type != CREATE && len(call.input) > 0` + /// and later adjusted if the tracer detects an account without code. + /// + /// Fixed in `Version 4`, see for information about block versions. #[prost(bool, tag = "15")] pub executed_code: bool, #[prost(bool, tag = "16")] @@ -418,6 +810,12 @@ pub struct Call { ::prost::alloc::string::String, ::prost::alloc::string::String, >, + /// Known Issues + /// + /// * Version 3: + /// The data might be not be in order. + /// + /// Fixed in `Version 4`, see for information about block versions. #[prost(message, repeated, tag = "21")] pub storage_changes: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "22")] @@ -428,12 +826,21 @@ pub struct Call { pub logs: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "26")] pub code_changes: ::prost::alloc::vec::Vec, + /// Known Issues + /// + /// * Version 3: + /// Some gas changes are not correctly tracked: + /// 1. Gas refunded due to data returned to the chain (occurs at the end of a transaction, before buyback). + /// 1. Initial gas allocation (0 -> GasLimit) at the start of a call. + /// 1. Final gas deduction (LeftOver -> 0) at the end of a call (if applicable). + /// Fixed in `Version 4`, see for information about block versions. #[prost(message, repeated, tag = "28")] pub gas_changes: ::prost::alloc::vec::Vec, /// In Ethereum, a call can be either: - /// - Successfull, execution passes without any problem encountered - /// - Failed, execution failed, and remaining gas should be consumed - /// - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded + /// + /// * Successful, execution passes without any problem encountered + /// * Failed, execution failed, and remaining gas should be consumed + /// * Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded /// /// When a call is either `failed` or `reverted`, the `status_failed` field /// below is set to `true`. If the status is `reverted`, then both `status_failed` @@ -446,7 +853,7 @@ pub struct Call { /// see above for details about those flags. #[prost(string, tag = "11")] pub failure_reason: ::prost::alloc::string::String, - /// This field represents wheter or not the state changes performed + /// This field represents whether or not the state changes performed /// by this call were correctly recorded by the blockchain. /// /// On Ethereum, a transaction can record state changes even if some @@ -456,28 +863,59 @@ pub struct Call { /// has a status of `SUCCESS`, the chain might have reverted all the state /// changes it performed. /// - /// ```text - /// Trx 1 - /// Call #1 - /// Call #2 - /// Call #3 - /// |--- Failure here - /// Call #4 + /// ```text,text + /// Trx 1 + /// Call #1 + /// Call #2 + /// Call #3 + /// |--- Failure here + /// Call #4 /// ``` /// /// In the transaction above, while Call #2 and Call #3 would have the - /// status `EXECUTED` + /// status `EXECUTED`. + /// + /// If you check all calls and check only `state_reverted` flag, you might be missing + /// some balance changes and nonce changes. This is because when a full transaction fails + /// in ethereum (e.g. `calls.all(x.state_reverted == true)`), there is still the transaction + /// fee that are recorded to the chain. + /// + /// Refer to \[TransactionTrace#status\] field for more details about the handling you must + /// perform. #[prost(bool, tag = "30")] pub state_reverted: bool, + /// Known Issues + /// + /// * Version 3: + /// + /// 1. The block's global ordinal when the call started executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. + /// 1. The transaction root call `begin_ordial` is always `0` (also in the GENESIS block), which can cause issues + /// when sorting by this field. To ensure proper execution order, set it as follows: + /// `trx.Calls\[0\].BeginOrdinal = trx.BeginOrdinal`. + /// Fixed in `Version 4`, see for information about block versions. #[prost(uint64, tag = "31")] pub begin_ordinal: u64, + /// Known Issues + /// + /// * Version 3: + /// + /// 1. The block's global ordinal when the call finished executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. + /// 1. The root call of the GENESIS block is always `0`. To fix it, you can set it as follows: + /// `rx.Calls\[0\].EndOrdinal = max.Uint64`. + /// Fixed in `Version 4`, see for information about block versions. #[prost(uint64, tag = "32")] pub end_ordinal: u64, + /// Known Issues + /// + /// * Version 4: + /// AccountCreations are NOT SUPPORTED anymore. DO NOT rely on them. + #[deprecated] #[prost(message, repeated, tag = "33")] pub account_creations: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StorageChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -487,30 +925,50 @@ pub struct StorageChange { pub old_value: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "4")] pub new_value: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the storage change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "5")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BalanceChange { + /// Address is the address of the account that has changed balance. #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, + /// OldValue is the balance of the address before the change. This value + /// can be **nil/null/None** if there was no previous balance for the address. + /// It is safe in those case(s) to consider the balance as being 0. + /// + /// If you consume this from a Substreams, you can safely use: + /// + /// ```text,ignore + /// let old_value = old_value.unwrap_or_default(); + /// ``` #[prost(message, optional, tag = "2")] pub old_value: ::core::option::Option, + /// NewValue is the balance of the address after the change. This value + /// can be **nil/null/None** if there was no previous balance for the address + /// after the change. It is safe in those case(s) to consider the balance as being + /// 0. + /// + /// If you consume this from a Substreams, you can safely use: + /// + /// ```text,ignore + /// let new_value = new_value.unwrap_or_default(); + /// ``` #[prost(message, optional, tag = "3")] pub new_value: ::core::option::Option, + /// Reason is the reason why the balance has changed. This is useful to determine + /// why the balance has changed and what is the context of the change. #[prost(enumeration = "balance_change::Reason", tag = "4")] pub reason: i32, + /// The block's global ordinal when the balance change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "5")] pub ordinal: u64, } /// Nested message and enum types in `BalanceChange`. pub mod balance_change { - /// Obtain all balanche change reasons under deep mind repository: - /// - /// ```shell - /// ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - /// ``` #[derive( Clone, Copy, @@ -541,6 +999,14 @@ pub mod balance_change { CallBalanceOverride = 12, /// Used on chain(s) where some Ether burning happens Burn = 15, + Withdrawal = 16, + /// Rewards for Blob processing on BNB chain added in Tycho hard-fork, refers + /// to BNB documentation to check the timestamp at which it was activated. + RewardBlobFee = 17, + /// This reason is used only on Optimism chain. + IncreaseMint = 18, + /// This reason is used only on Optimism chain. + Revert = 19, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -549,22 +1015,26 @@ pub mod balance_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", - Reason::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", - Reason::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", - Reason::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", - Reason::Transfer => "REASON_TRANSFER", - Reason::GenesisBalance => "REASON_GENESIS_BALANCE", - Reason::GasBuy => "REASON_GAS_BUY", - Reason::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", - Reason::RewardFeeReset => "REASON_REWARD_FEE_RESET", - Reason::GasRefund => "REASON_GAS_REFUND", - Reason::TouchAccount => "REASON_TOUCH_ACCOUNT", - Reason::SuicideRefund => "REASON_SUICIDE_REFUND", - Reason::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", - Reason::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", - Reason::Burn => "REASON_BURN", + Self::Unknown => "REASON_UNKNOWN", + Self::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", + Self::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", + Self::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", + Self::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", + Self::Transfer => "REASON_TRANSFER", + Self::GenesisBalance => "REASON_GENESIS_BALANCE", + Self::GasBuy => "REASON_GAS_BUY", + Self::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", + Self::RewardFeeReset => "REASON_REWARD_FEE_RESET", + Self::GasRefund => "REASON_GAS_REFUND", + Self::TouchAccount => "REASON_TOUCH_ACCOUNT", + Self::SuicideRefund => "REASON_SUICIDE_REFUND", + Self::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", + Self::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", + Self::Burn => "REASON_BURN", + Self::Withdrawal => "REASON_WITHDRAWAL", + Self::RewardBlobFee => "REASON_REWARD_BLOB_FEE", + Self::IncreaseMint => "REASON_INCREASE_MINT", + Self::Revert => "REASON_REVERT", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -586,13 +1056,16 @@ pub mod balance_change { "REASON_SUICIDE_WITHDRAW" => Some(Self::SuicideWithdraw), "REASON_CALL_BALANCE_OVERRIDE" => Some(Self::CallBalanceOverride), "REASON_BURN" => Some(Self::Burn), + "REASON_WITHDRAWAL" => Some(Self::Withdrawal), + "REASON_REWARD_BLOB_FEE" => Some(Self::RewardBlobFee), + "REASON_INCREASE_MINT" => Some(Self::IncreaseMint), + "REASON_REVERT" => Some(Self::Revert), _ => None, } } } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct NonceChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -600,19 +1073,21 @@ pub struct NonceChange { pub old_value: u64, #[prost(uint64, tag = "3")] pub new_value: u64, + /// The block's global ordinal when the nonce change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "4")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AccountCreation { #[prost(bytes = "vec", tag = "1")] pub account: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the account creation was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "2")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CodeChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -624,6 +1099,8 @@ pub struct CodeChange { pub new_hash: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "5")] pub new_code: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the code change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "6")] pub ordinal: u64, } @@ -633,8 +1110,7 @@ pub struct CodeChange { /// /// Hence, we only index some of them, those that are costy like all the calls /// one, log events, return data, etc. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GasChange { #[prost(uint64, tag = "1")] pub old_value: u64, @@ -642,16 +1118,13 @@ pub struct GasChange { pub new_value: u64, #[prost(enumeration = "gas_change::Reason", tag = "3")] pub reason: i32, + /// The block's global ordinal when the gas change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "4")] pub ordinal: u64, } /// Nested message and enum types in `GasChange`. pub mod gas_change { - /// Obtain all gas change reasons under deep mind repository: - /// - /// ```shell - /// ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - /// ``` #[derive( Clone, Copy, @@ -666,27 +1139,88 @@ pub mod gas_change { #[repr(i32)] pub enum Reason { Unknown = 0, + /// REASON_CALL is the amount of gas that will be charged for a 'CALL' opcode executed by the EVM Call = 1, + /// REASON_CALL_CODE is the amount of gas that will be charged for a 'CALLCODE' opcode executed by the EVM CallCode = 2, + /// REASON_CALL_DATA_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM CallDataCopy = 3, + /// REASON_CODE_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM CodeCopy = 4, + /// REASON_CODE_STORAGE is the amount of gas that will be charged for code storage CodeStorage = 5, + /// REASON_CONTRACT_CREATION is the amount of gas that will be charged for a 'CREATE' opcode executed by the EVM and for the gas + /// burned for a CREATE, today controlled by EIP150 rules ContractCreation = 6, + /// REASON_CONTRACT_CREATION2 is the amount of gas that will be charged for a 'CREATE2' opcode executed by the EVM and for the gas + /// burned for a CREATE2, today controlled by EIP150 rules ContractCreation2 = 7, + /// REASON_DELEGATE_CALL is the amount of gas that will be charged for a 'DELEGATECALL' opcode executed by the EVM DelegateCall = 8, + /// REASON_EVENT_LOG is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM EventLog = 9, + /// REASON_EXT_CODE_COPY is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM ExtCodeCopy = 10, + /// REASON_FAILED_EXECUTION is the burning of the remaining gas when the execution failed without a revert FailedExecution = 11, + /// REASON_INTRINSIC_GAS is the amount of gas that will be charged for the intrinsic cost of the transaction, there is + /// always exactly one of those per transaction IntrinsicGas = 12, + /// GasChangePrecompiledContract is the amount of gas that will be charged for a precompiled contract execution PrecompiledContract = 13, + /// REASON_REFUND_AFTER_EXECUTION is the amount of gas that will be refunded to the caller after the execution of the call, + /// if there is left over at the end of execution RefundAfterExecution = 14, + /// REASON_RETURN is the amount of gas that will be charged for a 'RETURN' opcode executed by the EVM Return = 15, + /// REASON_RETURN_DATA_COPY is the amount of gas that will be charged for a 'RETURNDATACOPY' opcode executed by the EVM ReturnDataCopy = 16, + /// REASON_REVERT is the amount of gas that will be charged for a 'REVERT' opcode executed by the EVM Revert = 17, + /// REASON_SELF_DESTRUCT is the amount of gas that will be charged for a 'SELFDESTRUCT' opcode executed by the EVM SelfDestruct = 18, + /// REASON_STATIC_CALL is the amount of gas that will be charged for a 'STATICALL' opcode executed by the EVM StaticCall = 19, + /// REASON_STATE_COLD_ACCESS is the amount of gas that will be charged for a cold storage access as controlled by EIP2929 rules + /// /// Added in Berlin fork (Geth 1.10+) StateColdAccess = 20, + /// REASON_TX_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call + /// + /// Added as new tracing reason in Geth, available only on some chains + TxInitialBalance = 21, + /// REASON_TX_REFUNDS is the sum of all refunds which happened during the tx execution (e.g. storage slot being cleared) + /// this generates an increase in gas. There is only one such gas change per transaction. + /// + /// Added as new tracing reason in Geth, available only on some chains + TxRefunds = 22, + /// REASON_TX_LEFT_OVER_RETURNED is the amount of gas left over at the end of transaction's execution that will be returned + /// to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + /// left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. + /// There is at most one of such gas change per transaction. + /// + /// Added as new tracing reason in Geth, available only on some chains + TxLeftOverReturned = 23, + /// REASON_CALL_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call. There is only + /// one such gas change per call. + /// + /// Added as new tracing reason in Geth, available only on some chains + CallInitialBalance = 24, + /// REASON_CALL_LEFT_OVER_RETURNED is the amount of gas left over that will be returned to the caller, this change will always + /// be a negative change as we "drain" left over gas towards 0. If there was no gas left at the end of execution, no such even + /// will be emitted. + CallLeftOverReturned = 25, + /// REASON_WITNESS_CONTRACT_INIT flags the event of adding to the witness during the contract creation initialization step. + WitnessContractInit = 26, + /// REASON_WITNESS_CONTRACT_CREATION flags the event of adding to the witness during the contract creation finalization step. + WitnessContractCreation = 27, + /// REASON_WITNESS_CODE_CHUNK flags the event of adding one or more contract code chunks to the witness. + WitnessCodeChunk = 28, + /// REASON_WITNESS_CONTRACT_COLLISION_CHECK flags the event of adding to the witness when checking for contract address collision. + WitnessContractCollisionCheck = 29, + /// REASON_TX_DATA_FLOOR is the amount of extra gas the transaction has to pay to reach the minimum gas requirement for the + /// transaction data. This change will always be a negative change. + TxDataFloor = 30, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -695,27 +1229,39 @@ pub mod gas_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::Call => "REASON_CALL", - Reason::CallCode => "REASON_CALL_CODE", - Reason::CallDataCopy => "REASON_CALL_DATA_COPY", - Reason::CodeCopy => "REASON_CODE_COPY", - Reason::CodeStorage => "REASON_CODE_STORAGE", - Reason::ContractCreation => "REASON_CONTRACT_CREATION", - Reason::ContractCreation2 => "REASON_CONTRACT_CREATION2", - Reason::DelegateCall => "REASON_DELEGATE_CALL", - Reason::EventLog => "REASON_EVENT_LOG", - Reason::ExtCodeCopy => "REASON_EXT_CODE_COPY", - Reason::FailedExecution => "REASON_FAILED_EXECUTION", - Reason::IntrinsicGas => "REASON_INTRINSIC_GAS", - Reason::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", - Reason::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", - Reason::Return => "REASON_RETURN", - Reason::ReturnDataCopy => "REASON_RETURN_DATA_COPY", - Reason::Revert => "REASON_REVERT", - Reason::SelfDestruct => "REASON_SELF_DESTRUCT", - Reason::StaticCall => "REASON_STATIC_CALL", - Reason::StateColdAccess => "REASON_STATE_COLD_ACCESS", + Self::Unknown => "REASON_UNKNOWN", + Self::Call => "REASON_CALL", + Self::CallCode => "REASON_CALL_CODE", + Self::CallDataCopy => "REASON_CALL_DATA_COPY", + Self::CodeCopy => "REASON_CODE_COPY", + Self::CodeStorage => "REASON_CODE_STORAGE", + Self::ContractCreation => "REASON_CONTRACT_CREATION", + Self::ContractCreation2 => "REASON_CONTRACT_CREATION2", + Self::DelegateCall => "REASON_DELEGATE_CALL", + Self::EventLog => "REASON_EVENT_LOG", + Self::ExtCodeCopy => "REASON_EXT_CODE_COPY", + Self::FailedExecution => "REASON_FAILED_EXECUTION", + Self::IntrinsicGas => "REASON_INTRINSIC_GAS", + Self::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", + Self::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", + Self::Return => "REASON_RETURN", + Self::ReturnDataCopy => "REASON_RETURN_DATA_COPY", + Self::Revert => "REASON_REVERT", + Self::SelfDestruct => "REASON_SELF_DESTRUCT", + Self::StaticCall => "REASON_STATIC_CALL", + Self::StateColdAccess => "REASON_STATE_COLD_ACCESS", + Self::TxInitialBalance => "REASON_TX_INITIAL_BALANCE", + Self::TxRefunds => "REASON_TX_REFUNDS", + Self::TxLeftOverReturned => "REASON_TX_LEFT_OVER_RETURNED", + Self::CallInitialBalance => "REASON_CALL_INITIAL_BALANCE", + Self::CallLeftOverReturned => "REASON_CALL_LEFT_OVER_RETURNED", + Self::WitnessContractInit => "REASON_WITNESS_CONTRACT_INIT", + Self::WitnessContractCreation => "REASON_WITNESS_CONTRACT_CREATION", + Self::WitnessCodeChunk => "REASON_WITNESS_CODE_CHUNK", + Self::WitnessContractCollisionCheck => { + "REASON_WITNESS_CONTRACT_COLLISION_CHECK" + } + Self::TxDataFloor => "REASON_TX_DATA_FLOOR", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -742,11 +1288,84 @@ pub mod gas_change { "REASON_SELF_DESTRUCT" => Some(Self::SelfDestruct), "REASON_STATIC_CALL" => Some(Self::StaticCall), "REASON_STATE_COLD_ACCESS" => Some(Self::StateColdAccess), + "REASON_TX_INITIAL_BALANCE" => Some(Self::TxInitialBalance), + "REASON_TX_REFUNDS" => Some(Self::TxRefunds), + "REASON_TX_LEFT_OVER_RETURNED" => Some(Self::TxLeftOverReturned), + "REASON_CALL_INITIAL_BALANCE" => Some(Self::CallInitialBalance), + "REASON_CALL_LEFT_OVER_RETURNED" => Some(Self::CallLeftOverReturned), + "REASON_WITNESS_CONTRACT_INIT" => Some(Self::WitnessContractInit), + "REASON_WITNESS_CONTRACT_CREATION" => Some(Self::WitnessContractCreation), + "REASON_WITNESS_CODE_CHUNK" => Some(Self::WitnessCodeChunk), + "REASON_WITNESS_CONTRACT_COLLISION_CHECK" => { + Some(Self::WitnessContractCollisionCheck) + } + "REASON_TX_DATA_FLOOR" => Some(Self::TxDataFloor), _ => None, } } } } +/// HeaderOnlyBlock is used to optimally unpack the \[Block\] structure (note the +/// corresponding message number for the `header` field) while consuming less +/// memory, when only the `header` is desired. +/// +/// WARN: this is a client-side optimization pattern and should be moved in the +/// consuming code. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HeaderOnlyBlock { + #[prost(message, optional, tag = "5")] + pub header: ::core::option::Option, +} +/// BlockWithRefs is a lightweight block, with traces and transactions +/// purged from the `block` within, and only. It is used in transports +/// to pass block data around. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockWithRefs { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub block: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub transaction_trace_refs: ::core::option::Option, + #[prost(bool, tag = "4")] + pub irreversible: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionTraceWithBlockRef { + #[prost(message, optional, tag = "1")] + pub trace: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub block_ref: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TransactionRefs { + #[prost(bytes = "vec", repeated, tag = "1")] + pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BlockRef { + #[prost(bytes = "vec", tag = "1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "2")] + pub number: u64, +} +/// Withdrawal represents a validator withdrawal from the beacon chain to the EVM. +/// Introduced in EIP-4895 (Shanghai hard fork). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Withdrawal { + /// Index is the monotonically increasing identifier of the withdrawal + #[prost(uint64, tag = "1")] + pub index: u64, + /// ValidatorIndex is the index of the validator that is withdrawing + #[prost(uint64, tag = "2")] + pub validator_index: u64, + /// Address is the Ethereum address receiving the withdrawn funds + #[prost(bytes = "vec", tag = "3")] + pub address: ::prost::alloc::vec::Vec, + /// Amount is the value of the withdrawal in gwei (1 gwei = 1e9 wei) + #[prost(uint64, tag = "4")] + pub amount: u64, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum TransactionTraceStatus { @@ -762,10 +1381,10 @@ impl TransactionTraceStatus { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - TransactionTraceStatus::Unknown => "UNKNOWN", - TransactionTraceStatus::Succeeded => "SUCCEEDED", - TransactionTraceStatus::Failed => "FAILED", - TransactionTraceStatus::Reverted => "REVERTED", + Self::Unknown => "UNKNOWN", + Self::Succeeded => "SUCCEEDED", + Self::Failed => "FAILED", + Self::Reverted => "REVERTED", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -798,12 +1417,12 @@ impl CallType { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - CallType::Unspecified => "UNSPECIFIED", - CallType::Call => "CALL", - CallType::Callcode => "CALLCODE", - CallType::Delegate => "DELEGATE", - CallType::Static => "STATIC", - CallType::Create => "CREATE", + Self::Unspecified => "UNSPECIFIED", + Self::Call => "CALL", + Self::Callcode => "CALLCODE", + Self::Delegate => "DELEGATE", + Self::Static => "STATIC", + Self::Create => "CREATE", } } /// Creates an enum from field names used in the ProtoBuf definition. diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index d88bf2b22d7..4311fed3c48 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -2,15 +2,18 @@ use super::runtime_adapter::UnresolvedContractCall; use crate::trigger::{ EthereumBlockData, EthereumCallData, EthereumEventData, EthereumTransactionData, }; +use anyhow::anyhow; +use async_trait::async_trait; +use graph::abi; +use graph::prelude::alloy; +use graph::prelude::alloy::consensus::TxReceipt; +use graph::prelude::alloy::network::ReceiptResponse; +use graph::prelude::alloy::rpc::types::{Log, TransactionReceipt}; use graph::{ - prelude::{ - ethabi, - web3::types::{Log, TransactionReceipt, H256}, - BigInt, - }, + prelude::BigInt, runtime::{ - asc_get, asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, - DeterministicHostError, FromAscObj, HostExportError, IndexForAscTypeId, ToAscObj, + AscHeap, AscIndexId, AscPtr, AscType, DeterministicHostError, FromAscObj, HostExportError, + IndexForAscTypeId, ToAscObj, asc_get, asc_new, asc_new_or_null, gas::GasCounter, }, }; use graph_runtime_derive::AscType; @@ -20,7 +23,7 @@ use graph_runtime_wasm::asc_abi::class::{ }; use semver::Version; -type AscH256 = Uint8Array; +type AscB256 = Uint8Array; type AscH2048 = Uint8Array; pub struct AscLogParamArray(Array>); @@ -37,15 +40,18 @@ impl AscType for AscLogParamArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for &[abi::DynSolParam] { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscLogParamArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::with_capacity(self.len()); + for x in *self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscLogParamArray(Array::new(&content, heap, gas).await?)) } } @@ -53,7 +59,7 @@ impl AscIndexId for AscLogParamArray { const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayEventParam; } -pub struct AscTopicArray(Array>); +pub struct AscTopicArray(Array>); impl AscType for AscTopicArray { fn to_asc_bytes(&self) -> Result, DeterministicHostError> { @@ -68,22 +74,23 @@ impl AscType for AscTopicArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for &[alloy::primitives::B256] { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let topics = self - .iter() - .map(|topic| asc_new(heap, topic, gas)) - .collect::, _>>()?; - Ok(AscTopicArray(Array::new(&topics, heap, gas)?)) + let mut topics = Vec::with_capacity(self.len()); + for topic in *self { + topics.push(asc_new(heap, topic, gas).await?); + } + Ok(AscTopicArray(Array::new(&topics, heap, gas).await?)) } } impl AscIndexId for AscTopicArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayH256; + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayB256; } pub struct AscLogArray(Array>); @@ -101,17 +108,19 @@ impl AscType for AscLogArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for &[Log] { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let logs = self - .iter() - .map(|log| asc_new(heap, &log, gas)) - .collect::, _>>()?; - Ok(AscLogArray(Array::new(&logs, heap, gas)?)) + let mut logs = Vec::with_capacity(self.len()); + for log in *self { + logs.push(asc_new(heap, log, gas).await?); + } + + Ok(AscLogArray(Array::new(&logs, heap, gas).await?)) } } @@ -121,6 +130,7 @@ impl AscIndexId for AscLogArray { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub struct AscUnresolvedContractCall_0_0_4 { pub contract_name: AscPtr, pub contract_address: AscPtr, @@ -179,13 +189,13 @@ impl FromAscObj for UnresolvedContractCall { #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumBlock { - pub hash: AscPtr, - pub parent_hash: AscPtr, - pub uncles_hash: AscPtr, + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, pub author: AscPtr, - pub state_root: AscPtr, - pub transactions_root: AscPtr, - pub receipts_root: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, pub number: AscPtr, pub gas_used: AscPtr, pub gas_limit: AscPtr, @@ -201,14 +211,15 @@ impl AscIndexId for AscEthereumBlock { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumBlock_0_0_6 { - pub hash: AscPtr, - pub parent_hash: AscPtr, - pub uncles_hash: AscPtr, + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, pub author: AscPtr, - pub state_root: AscPtr, - pub transactions_root: AscPtr, - pub receipts_root: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, pub number: AscPtr, pub gas_used: AscPtr, pub gas_limit: AscPtr, @@ -225,8 +236,9 @@ impl AscIndexId for AscEthereumBlock_0_0_6 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_1 { - pub hash: AscPtr, + pub hash: AscPtr, pub index: AscPtr, pub from: AscPtr, pub to: AscPtr, @@ -241,8 +253,9 @@ impl AscIndexId for AscEthereumTransaction_0_0_1 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_2 { - pub hash: AscPtr, + pub hash: AscPtr, pub index: AscPtr, pub from: AscPtr, pub to: AscPtr, @@ -258,8 +271,9 @@ impl AscIndexId for AscEthereumTransaction_0_0_2 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_6 { - pub hash: AscPtr, + pub hash: AscPtr, pub index: AscPtr, pub from: AscPtr, pub to: AscPtr, @@ -308,9 +322,9 @@ pub(crate) struct AscEthereumLog { pub address: AscPtr, pub topics: AscPtr, pub data: AscPtr, - pub block_hash: AscPtr, - pub block_number: AscPtr, - pub transaction_hash: AscPtr, + pub block_hash: AscPtr, + pub block_number: AscPtr, + pub transaction_hash: AscPtr, pub transaction_index: AscPtr, pub log_index: AscPtr, pub transaction_log_index: AscPtr, @@ -325,16 +339,16 @@ impl AscIndexId for AscEthereumLog { #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumTransactionReceipt { - pub transaction_hash: AscPtr, + pub transaction_hash: AscPtr, pub transaction_index: AscPtr, - pub block_hash: AscPtr, + pub block_hash: AscPtr, pub block_number: AscPtr, pub cumulative_gas_used: AscPtr, pub gas_used: AscPtr, pub contract_address: AscPtr, pub logs: AscPtr, pub status: AscPtr, - pub root: AscPtr, + pub root: AscPtr, pub logs_bloom: AscPtr, } @@ -346,6 +360,7 @@ impl AscIndexId for AscEthereumTransactionReceipt { /// `receipt` field. #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumEvent_0_0_7 where T: AscType, @@ -392,6 +407,7 @@ impl AscIndexId for AscEthereumCall { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumCall_0_0_3 where T: AscType, @@ -413,181 +429,173 @@ where const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumCall; } -impl ToAscObj for EthereumBlockData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumBlockData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { + let size = asc_new_or_null_u256(heap, self.size(), gas).await?; + Ok(AscEthereumBlock { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas).await?, + parent_hash: asc_new(heap, self.parent_hash(), gas).await?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas).await?, + author: asc_new(heap, self.author(), gas).await?, + state_root: asc_new(heap, self.state_root(), gas).await?, + transactions_root: asc_new(heap, self.transactions_root(), gas).await?, + receipts_root: asc_new(heap, self.receipts_root(), gas).await?, + number: asc_new(heap, &BigInt::from(self.number()), gas).await?, + gas_used: asc_new(heap, &BigInt::from(self.gas_used()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + timestamp: asc_new(heap, &BigInt::from(self.timestamp()), gas).await?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas).await?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, - )?, - size: self - .size - .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) - .unwrap_or(Ok(AscPtr::null()))?, + ) + .await?, + size, }) } } -impl ToAscObj for EthereumBlockData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumBlockData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { + let size = asc_new_or_null_u256(heap, self.size(), gas).await?; + let base_fee_per_block = asc_new_or_null_u64(heap, self.base_fee_per_gas(), gas).await?; + Ok(AscEthereumBlock_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas).await?, + parent_hash: asc_new(heap, self.parent_hash(), gas).await?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas).await?, + author: asc_new(heap, self.author(), gas).await?, + state_root: asc_new(heap, self.state_root(), gas).await?, + transactions_root: asc_new(heap, self.transactions_root(), gas).await?, + receipts_root: asc_new(heap, self.receipts_root(), gas).await?, + number: asc_new(heap, &BigInt::from(self.number()), gas).await?, + gas_used: asc_new(heap, &BigInt::from(self.gas_used()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + timestamp: asc_new(heap, &BigInt::from(self.timestamp()), gas).await?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas).await?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, - )?, - size: self - .size - .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - base_fee_per_block: self - .base_fee_per_gas - .map(|base_fee| asc_new(heap, &BigInt::from_unsigned_u256(&base_fee), gas)) - .unwrap_or(Ok(AscPtr::null()))?, + ) + .await?, + size, + base_fee_per_block, }) } } -impl ToAscObj for EthereumTransactionData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumTransactionData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_1 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, - to: self - .to - .map(|to| asc_new(heap, &to, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + hash: asc_new(heap, &self.hash(), gas).await?, + index: asc_new(heap, &BigInt::from(self.index()), gas).await?, + from: asc_new(heap, &self.from(), gas).await?, + to: asc_new_or_null(heap, &self.to(), gas).await?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + gas_price: asc_new(heap, &BigInt::from(self.gas_price()), gas).await?, }) } } -impl ToAscObj for EthereumTransactionData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumTransactionData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_2 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, - to: self - .to - .map(|to| asc_new(heap, &to, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, + hash: asc_new(heap, &self.hash(), gas).await?, + index: asc_new(heap, &BigInt::from(self.index()), gas).await?, + from: asc_new(heap, &self.from(), gas).await?, + to: asc_new_or_null(heap, &self.to(), gas).await?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + gas_price: asc_new(heap, &BigInt::from(self.gas_price()), gas).await?, + input: asc_new(heap, self.input(), gas).await?, }) } } -impl ToAscObj for EthereumTransactionData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumTransactionData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, - to: self - .to - .map(|to| asc_new(heap, &to, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, - nonce: asc_new(heap, &BigInt::from_unsigned_u256(&self.nonce), gas)?, + hash: asc_new(heap, &self.hash(), gas).await?, + index: asc_new(heap, &BigInt::from(self.index()), gas).await?, + from: asc_new(heap, &self.from(), gas).await?, + to: asc_new_or_null(heap, &self.to(), gas).await?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + gas_price: asc_new(heap, &BigInt::from(self.gas_price()), gas).await?, + input: asc_new(heap, self.input(), gas).await?, + nonce: asc_new(heap, &BigInt::from(self.nonce()), gas).await?, }) } } -impl ToAscObj> for EthereumEventData +#[async_trait] +impl<'a, T, B> ToAscObj> for EthereumEventData<'a> where - T: AscType + AscIndexId, - B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + T: AscType + AscIndexId + Send, + B: AscType + AscIndexId + Send, + EthereumTransactionData<'a>: ToAscObj, + EthereumBlockData<'a>: ToAscObj, { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result, HostExportError> { Ok(AscEthereumEvent { - address: asc_new(heap, &self.address, gas)?, - log_index: asc_new(heap, &BigInt::from_unsigned_u256(&self.log_index), gas)?, - transaction_log_index: asc_new( - heap, - &BigInt::from_unsigned_u256(&self.transaction_log_index), - gas, - )?, - log_type: self - .log_type - .clone() - .map(|log_type| asc_new(heap, &log_type, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - block: asc_new::(heap, &self.block, gas)?, - transaction: asc_new::(heap, &self.transaction, gas)?, - params: asc_new(heap, &self.params, gas)?, + address: asc_new(heap, self.address(), gas).await?, + log_index: asc_new(heap, &BigInt::from(self.log_index()), gas).await?, + transaction_log_index: asc_new(heap, &BigInt::from(self.transaction_log_index()), gas) + .await?, + log_type: asc_new_or_null(heap, &self.log_type().as_ref(), gas).await?, + block: asc_new::(heap, &self.block, gas).await?, + transaction: asc_new::(heap, &self.transaction, gas) + .await?, + params: asc_new(heap, &self.params, gas).await?, }) } } -impl ToAscObj> - for (EthereumEventData, Option<&TransactionReceipt>) +#[async_trait] +impl<'a, T, B, Inner> ToAscObj> + for (EthereumEventData<'a>, Option<&TransactionReceipt>) where - T: AscType + AscIndexId, - B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + T: AscType + AscIndexId + Send, + B: AscType + AscIndexId + Send, + EthereumTransactionData<'a>: ToAscObj, + EthereumBlockData<'a>: ToAscObj, + Inner: Send + Sync, + TransactionReceipt: ToAscObj, { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -601,9 +609,9 @@ where block, transaction, params, - } = event_data.to_asc_obj(heap, gas)?; + } = event_data.to_asc_obj(heap, gas).await?; let receipt = if let Some(receipt_data) = optional_receipt { - asc_new(heap, receipt_data, gas)? + asc_new(heap, receipt_data, gas).await? } else { AscPtr::null() }; @@ -620,117 +628,115 @@ where } } +async fn asc_new_or_null_u256( + heap: &mut H, + value: &Option, + gas: &GasCounter, +) -> Result, HostExportError> { + match value { + Some(value) => asc_new(heap, &BigInt::from_unsigned_u256(value), gas).await, + None => Ok(AscPtr::null()), + } +} + +async fn asc_new_or_null_u64( + heap: &mut H, + value: &Option, + gas: &GasCounter, +) -> Result, HostExportError> { + match value { + Some(value) => asc_new(heap, &BigInt::from(*value), gas).await, + None => Ok(AscPtr::null()), + } +} + +#[async_trait] impl ToAscObj for Log { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumLog { - address: asc_new(heap, &self.address, gas)?, - topics: asc_new(heap, &self.topics, gas)?, - data: asc_new(heap, self.data.0.as_slice(), gas)?, - block_hash: self - .block_hash - .map(|block_hash| asc_new(heap, &block_hash, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - block_number: self - .block_number - .map(|block_number| asc_new(heap, &BigInt::from(block_number), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - transaction_hash: self - .transaction_hash - .map(|txn_hash| asc_new(heap, &txn_hash, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - transaction_index: self - .transaction_index - .map(|txn_index| asc_new(heap, &BigInt::from(txn_index), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - log_index: self - .log_index - .map(|log_index| asc_new(heap, &BigInt::from_unsigned_u256(&log_index), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - transaction_log_index: self - .transaction_log_index - .map(|index| asc_new(heap, &BigInt::from_unsigned_u256(&index), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - log_type: self - .log_type - .as_ref() - .map(|log_type| asc_new(heap, &log_type, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - removed: self - .removed - .map(|removed| asc_new(heap, &AscWrapped { inner: removed }, gas)) - .unwrap_or(Ok(AscPtr::null()))?, + address: asc_new(heap, &self.address(), gas).await?, + topics: asc_new(heap, &self.topics(), gas).await?, + data: asc_new(heap, self.data().data.as_ref(), gas).await?, + block_hash: asc_new_or_null(heap, &self.block_hash, gas).await?, + block_number: asc_new_or_null_u64(heap, &self.block_number, gas).await?, + transaction_hash: asc_new_or_null(heap, &self.transaction_hash, gas).await?, + transaction_index: asc_new_or_null_u64(heap, &self.transaction_index, gas).await?, + log_index: asc_new_or_null_u64(heap, &self.log_index, gas).await?, + transaction_log_index: AscPtr::null(), // Non-standard field, not available in alloy + log_type: AscPtr::null(), // Non-standard field, not available in alloy + removed: asc_new( + heap, + &AscWrapped { + inner: self.removed, + }, + gas, + ) + .await?, }) } } -impl ToAscObj for &TransactionReceipt { - fn to_asc_obj( +#[async_trait] +impl ToAscObj + for TransactionReceipt> +{ + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { + let transaction_index = self + .transaction_index + .ok_or(HostExportError::Unknown(anyhow!( + "Transaction index is missing" + )))?; + let status = match self.inner.status_or_post_state().as_eip658() { + Some(success) => asc_new(heap, &BigInt::from(success as u64), gas).await?, + None => AscPtr::null(), // Pre-EIP-658 (pre-Byzantium) receipt + }; Ok(AscEthereumTransactionReceipt { - transaction_hash: asc_new(heap, &self.transaction_hash, gas)?, - transaction_index: asc_new(heap, &BigInt::from(self.transaction_index), gas)?, - block_hash: self - .block_hash - .map(|block_hash| asc_new(heap, &block_hash, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - block_number: self - .block_number - .map(|block_number| asc_new(heap, &BigInt::from(block_number), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - cumulative_gas_used: asc_new( - heap, - &BigInt::from_unsigned_u256(&self.cumulative_gas_used), - gas, - )?, - gas_used: self - .gas_used - .map(|gas_used| asc_new(heap, &BigInt::from_unsigned_u256(&gas_used), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - contract_address: self - .contract_address - .map(|contract_address| asc_new(heap, &contract_address, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - logs: asc_new(heap, &self.logs, gas)?, - status: self - .status - .map(|status| asc_new(heap, &BigInt::from(status), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - root: self - .root - .map(|root| asc_new(heap, &root, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - logs_bloom: asc_new(heap, self.logs_bloom.as_bytes(), gas)?, + transaction_hash: asc_new(heap, &self.transaction_hash, gas).await?, + transaction_index: asc_new(heap, &BigInt::from(transaction_index), gas).await?, + block_hash: asc_new_or_null(heap, &self.block_hash, gas).await?, + block_number: asc_new_or_null_u64(heap, &self.block_number, gas).await?, + cumulative_gas_used: asc_new(heap, &BigInt::from(self.cumulative_gas_used()), gas) + .await?, + gas_used: asc_new(heap, &BigInt::from(self.gas_used), gas).await?, + contract_address: asc_new_or_null(heap, &self.contract_address, gas).await?, + logs: asc_new(heap, &self.logs(), gas).await?, + status, + root: asc_new_or_null(heap, &self.state_root(), gas).await?, + logs_bloom: asc_new(heap, self.inner.bloom().as_slice(), gas).await?, }) } } -impl ToAscObj for EthereumCallData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumCallData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumCall { - address: asc_new(heap, &self.to, gas)?, - block: asc_new(heap, &self.block, gas)?, - transaction: asc_new(heap, &self.transaction, gas)?, - inputs: asc_new(heap, &self.inputs, gas)?, - outputs: asc_new(heap, &self.outputs, gas)?, + address: asc_new(heap, self.to(), gas).await?, + block: asc_new(heap, &self.block, gas).await?, + transaction: asc_new(heap, &self.transaction, gas).await?, + inputs: asc_new(heap, &self.inputs, gas).await?, + outputs: asc_new(heap, &self.outputs, gas).await?, }) } } -impl ToAscObj> - for EthereumCallData +#[async_trait] +impl<'a> ToAscObj> + for EthereumCallData<'a> { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -739,20 +745,21 @@ impl ToAscObj { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, - block: asc_new(heap, &self.block, gas)?, - transaction: asc_new(heap, &self.transaction, gas)?, - inputs: asc_new(heap, &self.inputs, gas)?, - outputs: asc_new(heap, &self.outputs, gas)?, + to: asc_new(heap, self.to(), gas).await?, + from: asc_new(heap, self.from(), gas).await?, + block: asc_new(heap, &self.block, gas).await?, + transaction: asc_new(heap, &self.transaction, gas).await?, + inputs: asc_new(heap, &self.inputs, gas).await?, + outputs: asc_new(heap, &self.outputs, gas).await?, }) } } -impl ToAscObj> - for EthereumCallData +#[async_trait] +impl<'a> ToAscObj> + for EthereumCallData<'a> { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -761,25 +768,26 @@ impl ToAscObj { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, - block: asc_new(heap, &self.block, gas)?, - transaction: asc_new(heap, &self.transaction, gas)?, - inputs: asc_new(heap, &self.inputs, gas)?, - outputs: asc_new(heap, &self.outputs, gas)?, + to: asc_new(heap, self.to(), gas).await?, + from: asc_new(heap, self.from(), gas).await?, + block: asc_new(heap, &self.block, gas).await?, + transaction: asc_new(heap, &self.transaction, gas).await?, + inputs: asc_new(heap, &self.inputs, gas).await?, + outputs: asc_new(heap, &self.outputs, gas).await?, }) } } -impl ToAscObj for ethabi::LogParam { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for abi::DynSolParam { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscLogParam { - name: asc_new(heap, self.name.as_str(), gas)?, - value: asc_new(heap, &self.value, gas)?, + name: asc_new(heap, self.name.as_str(), gas).await?, + value: asc_new(heap, &self.value, gas).await?, }) } } diff --git a/chain/ethereum/src/runtime/runtime_adapter.rs b/chain/ethereum/src/runtime/runtime_adapter.rs index 3ba3d3235df..a5d68a761a7 100644 --- a/chain/ethereum/src/runtime/runtime_adapter.rs +++ b/chain/ethereum/src/runtime/runtime_adapter.rs @@ -1,29 +1,34 @@ use std::{sync::Arc, time::Instant}; -use crate::data_source::MappingABI; +use crate::adapter::EthereumRpcError; use crate::{ - capabilities::NodeCapabilities, network::EthereumNetworkAdapters, Chain, DataSource, - EthereumAdapter, EthereumAdapterTrait, EthereumContractCall, EthereumContractCallError, - ENV_VARS, + Chain, ContractCallError, ENV_VARS, EthereumAdapter, EthereumAdapterTrait, + capabilities::NodeCapabilities, network::EthereumNetworkAdapters, }; -use anyhow::{Context, Error}; +use anyhow::{Context, Error, anyhow}; use blockchain::HostFn; +use graph::abi; +use graph::abi::DynSolValueExt; use graph::blockchain::ChainIdentifier; use graph::components::subgraph::HostMetrics; +use graph::data::store::ethereum::call; +use graph::data::store::scalar::BigInt; +use graph::data::subgraph::{API_VERSION_0_0_4, API_VERSION_0_0_9}; +use graph::data_source; +use graph::data_source::common::{ContractCall, MappingABI}; use graph::runtime::gas::Gas; use graph::runtime::{AscIndexId, IndexForAscTypeId}; +use graph::slog::debug; use graph::{ blockchain::{self, BlockPtr, HostFnCtx}, cheap_clone::CheapClone, - prelude::{ - ethabi::{self, Address, Token}, - EthereumCallCache, Future01CompatExt, - }, - runtime::{asc_get, asc_new, AscPtr, HostExportError}, - semver::Version, - slog::{info, trace, Logger}, + futures03::FutureExt, + prelude::{EthereumCallCache, alloy::primitives::Address}, + runtime::{AscPtr, HostExportError, asc_get, asc_new}, + slog::Logger, }; -use graph_runtime_wasm::asc_abi::class::{AscEnumArray, EthereumValueKind}; +use graph_runtime_wasm::asc_abi::class::{AscBigInt, AscEnumArray, AscWrapped, EthereumValueKind}; +use itertools::Itertools; use super::abi::{AscUnresolvedContractCall, AscUnresolvedContractCall_0_0_4}; @@ -47,56 +52,143 @@ const ETH_CALL_GAS: u32 = 50_000_000; // [1] - https://www.sciencedirect.com/science/article/abs/pii/S0166531620300900 pub const ETHEREUM_CALL: Gas = Gas::new(5_000_000_000); +// TODO: Determine the appropriate gas cost for `ETH_GET_BALANCE`, initially aligned with `ETHEREUM_CALL`. +pub const ETH_GET_BALANCE: Gas = Gas::new(5_000_000_000); + +// TODO: Determine the appropriate gas cost for `ETH_HAS_CODE`, initially aligned with `ETHEREUM_CALL`. +pub const ETH_HAS_CODE: Gas = Gas::new(5_000_000_000); + pub struct RuntimeAdapter { pub eth_adapters: Arc, pub call_cache: Arc, pub chain_identifier: Arc, } +pub fn eth_call_gas(chain_identifier: &ChainIdentifier) -> Option { + // Check if the current network version is in the eth_call_no_gas list + let should_skip_gas = ENV_VARS + .eth_call_no_gas + .contains(&chain_identifier.net_version); + + if should_skip_gas { + None + } else { + Some(ETH_CALL_GAS) + } +} + impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, ds: &DataSource) -> Result, Error> { - let abis = ds.mapping.abis.clone(); - let call_cache = self.call_cache.cheap_clone(); - let eth_adapters = self.eth_adapters.cheap_clone(); - let archive = ds.mapping.requires_archive()?; - - // Check if the current network version is in the eth_call_no_gas list - let should_skip_gas = ENV_VARS - .eth_call_no_gas - .contains(&self.chain_identifier.net_version); - - let eth_call_gas = if should_skip_gas { - None - } else { - Some(ETH_CALL_GAS) - }; + fn host_fns(&self, ds: &data_source::DataSource) -> Result, Error> { + fn create_host_fns( + abis: Arc>>, // Use Arc to ensure `'static` lifetimes. + archive: bool, + call_cache: Arc, + eth_adapters: Arc, + eth_call_gas: Option, + ) -> Vec { + vec![ + HostFn { + name: "ethereum.call", + func: Arc::new({ + let eth_adapters = eth_adapters.clone(); + let call_cache = call_cache.clone(); + let abis = abis.clone(); + move |ctx, wasm_ptr| { + let eth_adapters = eth_adapters.cheap_clone(); + let call_cache = call_cache.cheap_clone(); + let abis = abis.cheap_clone(); + async move { + let eth_adapter = + eth_adapters.call_or_cheapest(Some(&NodeCapabilities { + archive, + traces: false, + }))?; + ethereum_call( + ð_adapter, + call_cache.clone(), + ctx, + wasm_ptr, + &abis, + eth_call_gas, + ) + .await + .map(|ptr| ptr.wasm_ptr()) + } + .boxed() + } + }), + }, + HostFn { + name: "ethereum.getBalance", + func: Arc::new({ + let eth_adapters = eth_adapters.clone(); + move |ctx, wasm_ptr| { + let eth_adapters = eth_adapters.cheap_clone(); + async move { + let eth_adapter = + eth_adapters.unverified_cheapest_with(&NodeCapabilities { + archive, + traces: false, + })?; + eth_get_balance(ð_adapter, ctx, wasm_ptr) + .await + .map(|ptr| ptr.wasm_ptr()) + } + .boxed() + } + }), + }, + HostFn { + name: "ethereum.hasCode", + func: Arc::new({ + move |ctx, wasm_ptr| { + let eth_adapters = eth_adapters.cheap_clone(); + async move { + let eth_adapter = + eth_adapters.unverified_cheapest_with(&NodeCapabilities { + archive, + traces: false, + })?; + eth_has_code(ð_adapter, ctx, wasm_ptr) + .await + .map(|ptr| ptr.wasm_ptr()) + } + .boxed() + } + }), + }, + ] + } - let ethereum_call = HostFn { - name: "ethereum.call", - func: Arc::new(move |ctx, wasm_ptr| { - // Ethereum calls should prioritise call-only adapters if one is available. - let eth_adapter = eth_adapters.call_or_cheapest(Some(&NodeCapabilities { - archive, - traces: false, - }))?; - ethereum_call( - ð_adapter, - call_cache.cheap_clone(), - ctx, - wasm_ptr, - &abis, - eth_call_gas, - ) - .map(|ptr| ptr.wasm_ptr()) - }), + let host_fns = match ds { + data_source::DataSource::Onchain(onchain_ds) => { + let abis = Arc::new(onchain_ds.mapping.abis.clone()); + let archive = onchain_ds.mapping.requires_archive()?; + let call_cache = self.call_cache.cheap_clone(); + let eth_adapters = self.eth_adapters.cheap_clone(); + let eth_call_gas = eth_call_gas(&self.chain_identifier); + + create_host_fns(abis, archive, call_cache, eth_adapters, eth_call_gas) + } + data_source::DataSource::Subgraph(subgraph_ds) => { + let abis = Arc::new(subgraph_ds.mapping.abis.clone()); + let archive = subgraph_ds.mapping.requires_archive()?; + let call_cache = self.call_cache.cheap_clone(); + let eth_adapters = self.eth_adapters.cheap_clone(); + let eth_call_gas = eth_call_gas(&self.chain_identifier); + + create_host_fns(abis, archive, call_cache, eth_adapters, eth_call_gas) + } + data_source::DataSource::Offchain(_) => vec![], + data_source::DataSource::Amp(_) => vec![], }; - Ok(vec![ethereum_call]) + Ok(host_fns) } } /// function ethereum.call(call: SmartContractCall): Array | null -fn ethereum_call( +async fn ethereum_call( eth_adapter: &EthereumAdapter, call_cache: Arc, ctx: HostFnCtx<'_>, @@ -110,7 +202,7 @@ fn ethereum_call( // For apiVersion >= 0.0.4 the call passed from the mapping includes the // function signature; subgraphs using an apiVersion < 0.0.4 don't pass // the signature along with the call. - let call: UnresolvedContractCall = if ctx.heap.api_version() >= Version::new(0, 0, 4) { + let call: UnresolvedContractCall = if ctx.heap.api_version() >= &API_VERSION_0_0_4 { asc_get::<_, AscUnresolvedContractCall_0_0_4, _>(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)? } else { asc_get::<_, AscUnresolvedContractCall, _>(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)? @@ -125,15 +217,86 @@ fn ethereum_call( abis, eth_call_gas, ctx.metrics.cheap_clone(), - )?; + ) + .await?; match result { - Some(tokens) => Ok(asc_new(ctx.heap, tokens.as_slice(), &ctx.gas)?), + Some(tokens) => Ok(asc_new(ctx.heap, tokens.as_slice(), &ctx.gas).await?), None => Ok(AscPtr::null()), } } +async fn eth_get_balance( + eth_adapter: &EthereumAdapter, + ctx: HostFnCtx<'_>, + wasm_ptr: u32, +) -> Result, HostExportError> { + ctx.gas + .consume_host_fn_with_metrics(ETH_GET_BALANCE, "eth_get_balance")?; + + if ctx.heap.api_version() < &API_VERSION_0_0_9 { + return Err(HostExportError::Deterministic(anyhow!( + "ethereum.getBalance call is not supported before API version 0.0.9" + ))); + } + + let logger = &ctx.logger; + let block_ptr = &ctx.block_ptr; + + let address: Address = asc_get(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)?; + + let result = eth_adapter + .get_balance(logger, address, block_ptr.clone()) + .await; + + match result { + Ok(v) => { + let bigint = BigInt::from_unsigned_u256(&v); + Ok(asc_new(ctx.heap, &bigint, &ctx.gas).await?) + } + // Retry on any kind of error + Err(EthereumRpcError::AlloyError(e)) => Err(HostExportError::PossibleReorg(e.into())), + Err(EthereumRpcError::Timeout) => Err(HostExportError::PossibleReorg( + EthereumRpcError::Timeout.into(), + )), + } +} + +async fn eth_has_code( + eth_adapter: &EthereumAdapter, + ctx: HostFnCtx<'_>, + wasm_ptr: u32, +) -> Result>, HostExportError> { + ctx.gas + .consume_host_fn_with_metrics(ETH_HAS_CODE, "eth_has_code")?; + + if ctx.heap.api_version() < &API_VERSION_0_0_9 { + return Err(HostExportError::Deterministic(anyhow!( + "ethereum.hasCode call is not supported before API version 0.0.9" + ))); + } + + let logger = &ctx.logger; + let block_ptr = &ctx.block_ptr; + + let address: Address = asc_get(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)?; + + let result = eth_adapter + .get_code(logger, address, block_ptr.clone()) + .await + .map(|v| !v.0.is_empty()); + + match result { + Ok(v) => Ok(asc_new(ctx.heap, &AscWrapped { inner: v }, &ctx.gas).await?), + // Retry on any kind of error + Err(EthereumRpcError::AlloyError(e)) => Err(HostExportError::PossibleReorg(e.into())), + Err(EthereumRpcError::Timeout) => Err(HostExportError::PossibleReorg( + EthereumRpcError::Timeout.into(), + )), + } +} + /// Returns `Ok(None)` if the call was reverted. -fn eth_call( +async fn eth_call( eth_adapter: &EthereumAdapter, call_cache: Arc, logger: &Logger, @@ -142,11 +305,11 @@ fn eth_call( abis: &[Arc], eth_call_gas: Option, metrics: Arc, -) -> Result>, HostExportError> { +) -> Result>, HostExportError> { let start_time = Instant::now(); // Obtain the path to the contract ABI - let contract = abis + let abi = abis .iter() .find(|abi| abi.name == unresolved_call.contract_name) .with_context(|| { @@ -156,51 +319,18 @@ fn eth_call( unresolved_call.contract_name ) }) - .map_err(HostExportError::Deterministic)? - .contract - .clone(); - - let function = match unresolved_call.function_signature { - // Behavior for apiVersion < 0.0.4: look up function by name; for overloaded - // functions this always picks the same overloaded variant, which is incorrect - // and may lead to encoding/decoding errors - None => contract - .function(unresolved_call.function_name.as_str()) - .with_context(|| { - format!( - "Unknown function \"{}::{}\" called from WASM runtime", - unresolved_call.contract_name, unresolved_call.function_name - ) - }) - .map_err(HostExportError::Deterministic)?, - - // Behavior for apiVersion >= 0.0.04: look up function by signature of - // the form `functionName(uint256,string) returns (bytes32,string)`; this - // correctly picks the correct variant of an overloaded function - Some(ref function_signature) => contract - .functions_by_name(unresolved_call.function_name.as_str()) - .with_context(|| { - format!( - "Unknown function \"{}::{}\" called from WASM runtime", - unresolved_call.contract_name, unresolved_call.function_name - ) - }) - .map_err(HostExportError::Deterministic)? - .iter() - .find(|f| function_signature == &f.signature()) - .with_context(|| { - format!( - "Unknown function \"{}::{}\" with signature `{}` \ - called from WASM runtime", - unresolved_call.contract_name, - unresolved_call.function_name, - function_signature, - ) - }) - .map_err(HostExportError::Deterministic)?, - }; + .map_err(HostExportError::Deterministic)?; + + let function = abi + .function( + &unresolved_call.contract_name, + &unresolved_call.function_name, + unresolved_call.function_signature.as_deref(), + ) + .map_err(HostExportError::Deterministic)?; - let call = EthereumContractCall { + let call = ContractCall { + contract_name: unresolved_call.contract_name.clone(), address: unresolved_call.contract_address, block_ptr: block_ptr.cheap_clone(), function: function.clone(), @@ -211,55 +341,70 @@ fn eth_call( // Run Ethereum call in tokio runtime let logger1 = logger.clone(); let call_cache = call_cache.clone(); - let result = match graph::block_on( - eth_adapter.contract_call(&logger1, call, call_cache).compat() - ) { - Ok(tokens) => Ok(Some(tokens)), - Err(EthereumContractCallError::Revert(reason)) => { - info!(logger, "Contract call reverted"; "reason" => reason); - Ok(None) - } + let (result, source) = match eth_adapter.contract_call(&logger1, &call, call_cache).await { + Ok((result, source)) => (Ok(result), source), + Err(e) => (Err(e), call::Source::Rpc), + }; + let result = match result { + Ok(res) => Ok(res), - // Any error reported by the Ethereum node could be due to the block no longer being on - // the main chain. This is very unespecific but we don't want to risk failing a - // subgraph due to a transient error such as a reorg. - Err(EthereumContractCallError::Web3Error(e)) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + // Any error reported by the Ethereum node could be due to the block no longer being on + // the main chain. This is very unespecific but we don't want to risk failing a + // subgraph due to a transient error such as a reorg. + Err(ContractCallError::AlloyError(e)) => { + Err(HostExportError::PossibleReorg(anyhow::anyhow!( "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", unresolved_call.function_name, unresolved_call.contract_name, e - ))), + ))) + } - // Also retry on timeouts. - Err(EthereumContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( - "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", - unresolved_call.function_name, - unresolved_call.contract_name, - ))), + // Also retry on timeouts. + Err(ContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", + unresolved_call.function_name, + unresolved_call.contract_name, + ))), - Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( - "Failed to call function \"{}\" of contract \"{}\": {}", - unresolved_call.function_name, - unresolved_call.contract_name, - e - ))), - }; + Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( + "Failed to call function \"{}\" of contract \"{}\": {}", + unresolved_call.function_name, + unresolved_call.contract_name, + e + ))), + }; let elapsed = start_time.elapsed(); - metrics.observe_eth_call_execution_time( - elapsed.as_secs_f64(), - &unresolved_call.contract_name, - &unresolved_call.contract_address.to_string(), - &unresolved_call.function_name, - ); + if source.observe() { + metrics.observe_eth_call_execution_time( + elapsed.as_secs_f64(), + &unresolved_call.contract_name, + &unresolved_call.function_name, + ); + } - trace!(logger, "Contract call finished"; - "address" => &unresolved_call.contract_address.to_string(), - "contract" => &unresolved_call.contract_name, - "function" => &unresolved_call.function_name, - "function_signature" => &unresolved_call.function_signature, - "time" => format!("{}ms", elapsed.as_millis())); + let args_as_string = format!("[{}]", values_to_string(&unresolved_call.function_args)); + + let result_as_string = match &result { + Ok(Some(values)) => format!("({})", values_to_string(values)), + Ok(None) => "none".to_owned(), + Err(_err) => "error".to_owned(), + }; + + debug!( + logger, "Contract call finished"; + "address" => format!("0x{:x}", &unresolved_call.contract_address), + "contract" => &unresolved_call.contract_name, + "signature" => &unresolved_call.function_signature, + "args" => args_as_string, + "time_ms" => format!("{}ms", elapsed.as_millis()), + "result" => result_as_string, + "block_hash" => block_ptr.hash_hex(), + "block_number" => block_ptr.block_number(), + "source" => source.to_string(), + ); result } @@ -270,9 +415,18 @@ pub struct UnresolvedContractCall { pub contract_address: Address, pub function_name: String, pub function_signature: Option, - pub function_args: Vec, + pub function_args: Vec, } impl AscIndexId for AscUnresolvedContractCall { const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::SmartContractCall; } + +#[inline] +fn values_to_string(values: &[abi::DynSolValue]) -> String { + values + .iter() + .map(|x| x.to_string()) + .collect_vec() + .join(", ") +} diff --git a/chain/ethereum/src/tests.rs b/chain/ethereum/src/tests.rs index 455a7c07432..a5f0a17bff0 100644 --- a/chain/ethereum/src/tests.rs +++ b/chain/ethereum/src/tests.rs @@ -1,12 +1,17 @@ use std::sync::Arc; use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr}, + blockchain::{BlockPtr, Trigger, block_stream::BlockWithTriggers}, prelude::{ - web3::types::{Address, Bytes, Log, H160, H256, U64}, EthereumCall, LightEthereumBlock, + alloy::{ + self, + primitives::{Address, B256, Bytes, LogData}, + rpc::types::{Block, Log}, + }, + rand::{self, Rng}, }, - slog::{self, o, Logger}, + slog::{self, Logger, o}, }; use crate::{ @@ -14,54 +19,86 @@ use crate::{ trigger::{EthereumBlockTriggerType, EthereumTrigger, LogRef}, }; +pub trait Random { + fn random() -> Self; +} + +impl Random for B256 { + fn random() -> Self { + let mut rng = rand::rng(); + let mut bytes = [0u8; 32]; + rng.fill(&mut bytes); + Self::from(bytes) + } +} + +impl Random for Address { + fn random() -> Self { + let mut rng = rand::rng(); + let mut bytes = [0u8; 20]; + rng.fill(&mut bytes); + Self::from(bytes) + } +} + +fn create_log(tx_index: u64, log_index: u64) -> Arc { + let log = Log { + inner: alloy::primitives::Log { + address: Address::default(), + data: LogData::new_unchecked(vec![], Bytes::from(vec![])), + }, + block_hash: Some(B256::ZERO), + block_number: Some(0), + block_timestamp: Some(0), + transaction_hash: Some(B256::ZERO), + transaction_index: Some(tx_index), + log_index: Some(log_index), + removed: false, + }; + + Arc::new(log) +} + #[test] fn test_trigger_ordering() { let block1 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 1u64)), + BlockPtr::from((B256::random(), 1u64)), EthereumBlockTriggerType::End, ); let block2 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 0u64)), + BlockPtr::from((B256::random(), 0u64)), EthereumBlockTriggerType::WithCallTo(Address::random()), ); - let mut call1 = EthereumCall::default(); - call1.transaction_index = 1; + let call1 = EthereumCall { + transaction_index: 1, + ..Default::default() + }; let call1 = EthereumTrigger::Call(Arc::new(call1)); - let mut call2 = EthereumCall::default(); - call2.transaction_index = 2; - call2.input = Bytes(vec![0]); + let call2 = EthereumCall { + transaction_index: 2, + input: Bytes::from(vec![0]), + ..Default::default() + }; let call2 = EthereumTrigger::Call(Arc::new(call2)); - let mut call3 = EthereumCall::default(); - call3.transaction_index = 3; + let call3 = EthereumCall { + transaction_index: 3, + ..Default::default() + }; let call3 = EthereumTrigger::Call(Arc::new(call3)); // Call with the same tx index as call2 - let mut call4 = EthereumCall::default(); - call4.transaction_index = 2; // different than call2 so they don't get mistaken as the same - call4.input = Bytes(vec![1]); + let call4 = EthereumCall { + transaction_index: 2, + input: Bytes::from(vec![1]), + ..Default::default() + }; let call4 = EthereumTrigger::Call(Arc::new(call4)); - fn create_log(tx_index: u64, log_index: u64) -> Arc { - Arc::new(Log { - address: H160::default(), - topics: vec![], - data: Bytes::default(), - block_hash: Some(H256::zero()), - block_number: Some(U64::zero()), - transaction_hash: Some(H256::zero()), - transaction_index: Some(tx_index.into()), - log_index: Some(log_index.into()), - transaction_log_index: Some(log_index.into()), - log_type: Some("".into()), - removed: Some(false), - }) - } - // Event with transaction_index 1 and log_index 0; // should be the first element after sorting let log1 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 0), None)); @@ -92,13 +129,9 @@ fn test_trigger_ordering() { let logger = Logger::root(slog::Discard, o!()); - let mut b: LightEthereumBlock = Default::default(); + let b = Block::default(); - // This is necessary because inside of BlockWithTriggers::new - // there's a log for both fields. So just using Default above - // gives None on them. - b.number = Some(Default::default()); - b.hash = Some(Default::default()); + let b = LightEthereumBlock::new(graph::components::ethereum::AnyBlock::from(b)); // Test that `BlockWithTriggers` sorts the triggers. let block_with_triggers = BlockWithTriggers::::new( @@ -107,60 +140,54 @@ fn test_trigger_ordering() { &logger, ); - assert_eq!( - block_with_triggers.trigger_data, - vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] - ); + let expected = vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] + .into_iter() + .map(Trigger::Chain) + .collect::>(); + + assert_eq!(block_with_triggers.trigger_data, expected); } #[test] fn test_trigger_dedup() { let block1 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 1u64)), + BlockPtr::from((B256::random(), 1u64)), EthereumBlockTriggerType::End, ); let block2 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 0u64)), + BlockPtr::from((B256::random(), 0u64)), EthereumBlockTriggerType::WithCallTo(Address::random()), ); // duplicate block2 let block3 = block2.clone(); - let mut call1 = EthereumCall::default(); - call1.transaction_index = 1; + let call1 = EthereumCall { + transaction_index: 1, + ..Default::default() + }; let call1 = EthereumTrigger::Call(Arc::new(call1)); - let mut call2 = EthereumCall::default(); - call2.transaction_index = 2; + let call2 = EthereumCall { + transaction_index: 2, + ..Default::default() + }; let call2 = EthereumTrigger::Call(Arc::new(call2)); - let mut call3 = EthereumCall::default(); - call3.transaction_index = 3; + let call3 = EthereumCall { + transaction_index: 3, + ..Default::default() + }; let call3 = EthereumTrigger::Call(Arc::new(call3)); // duplicate call2 - let mut call4 = EthereumCall::default(); - call4.transaction_index = 2; + let call4 = EthereumCall { + transaction_index: 2, + ..Default::default() + }; let call4 = EthereumTrigger::Call(Arc::new(call4)); - fn create_log(tx_index: u64, log_index: u64) -> Arc { - Arc::new(Log { - address: H160::default(), - topics: vec![], - data: Bytes::default(), - block_hash: Some(H256::zero()), - block_number: Some(U64::zero()), - transaction_hash: Some(H256::zero()), - transaction_index: Some(tx_index.into()), - log_index: Some(log_index.into()), - transaction_log_index: Some(log_index.into()), - log_type: Some("".into()), - removed: Some(false), - }) - } - let log1 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 0), None)); let log2 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 1), None)); let log3 = EthereumTrigger::Log(LogRef::FullLog(create_log(2, 5), None)); @@ -188,13 +215,11 @@ fn test_trigger_dedup() { let logger = Logger::root(slog::Discard, o!()); - let mut b: LightEthereumBlock = Default::default(); + #[allow(unused_variables)] + let b = Block::default(); - // This is necessary because inside of BlockWithTriggers::new - // there's a log for both fields. So just using Default above - // gives None on them. - b.number = Some(Default::default()); - b.hash = Some(Default::default()); + #[allow(unreachable_code)] + let b = LightEthereumBlock::new(graph::components::ethereum::AnyBlock::from(b)); // Test that `BlockWithTriggers` sorts the triggers. let block_with_triggers = BlockWithTriggers::::new( @@ -203,8 +228,10 @@ fn test_trigger_dedup() { &logger, ); - assert_eq!( - block_with_triggers.trigger_data, - vec![log1, log2, call1, log3, call2, call3, block2, block1] - ); + let expected = vec![log1, log2, call1, log3, call2, call3, block2, block1] + .into_iter() + .map(Trigger::Chain) + .collect::>(); + + assert_eq!(block_with_triggers.trigger_data, expected); } diff --git a/chain/ethereum/src/transport.rs b/chain/ethereum/src/transport.rs index f19b45dc3df..5b5414c4a5d 100644 --- a/chain/ethereum/src/transport.rs +++ b/chain/ethereum/src/transport.rs @@ -1,140 +1,305 @@ -use graph::endpoint::{EndpointMetrics, Provider, RequestLabels}; -use jsonrpc_core::types::Call; -use jsonrpc_core::Value; - -use web3::transports::{http, ipc, ws}; -use web3::RequestId; - +use alloy::transports::{TransportError, TransportErrorKind, TransportFut}; +use graph::components::ethereum::json_patch; +use graph::components::network_provider::ProviderName; +use graph::endpoint::{ConnectionType, EndpointMetrics, RequestLabels}; +use graph::prelude::alloy::rpc::json_rpc::{RequestPacket, ResponsePacket}; +use graph::prelude::alloy::transports::{ipc::IpcConnect, ws::WsConnect}; use graph::prelude::*; use graph::url::Url; -use std::future::Future; +use serde_json::Value; +use std::sync::Arc; +use std::task::{Context, Poll}; +use tower::Service; + +/// Compression method for RPC requests. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + #[default] + None, + Gzip, + Brotli, + Deflate, +} + +impl std::fmt::Display for Compression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Compression::None => write!(f, "none"), + Compression::Gzip => write!(f, "gzip"), + Compression::Brotli => write!(f, "brotli"), + Compression::Deflate => write!(f, "deflate"), + } + } +} -/// Abstraction over the different web3 transports. +/// Abstraction over different transport types for Alloy providers. #[derive(Clone, Debug)] pub enum Transport { - RPC { - client: http::Http, - metrics: Arc, - provider: Provider, - }, - IPC(ipc::Ipc), - WS(ws::WebSocket), + RPC(alloy::rpc::client::RpcClient), + IPC(IpcConnect), + WS(WsConnect), } impl Transport { /// Creates an IPC transport. + /// + /// Accepts both a raw file path (`/tmp/geth.ipc`) and an `ipc://` URL + /// (`ipc:///tmp/geth.ipc`). When a URL is provided the file path is + /// extracted so that the underlying IPC connector receives a plain path. #[cfg(unix)] pub async fn new_ipc(ipc: &str) -> Self { - ipc::Ipc::new(ipc) - .await - .map(Transport::IPC) - .expect("Failed to connect to Ethereum IPC") + let path = Url::parse(ipc) + .ok() + .map(|u| u.path().to_string()) + .unwrap_or_else(|| ipc.to_string()); + let transport = IpcConnect::new(path); + + Transport::IPC(transport) + } + + #[cfg(not(unix))] + pub async fn new_ipc(_ipc: &str) -> Self { + panic!("IPC connections are not supported on non-Unix platforms") } /// Creates a WebSocket transport. pub async fn new_ws(ws: &str) -> Self { - ws::WebSocket::new(ws) - .await - .map(Transport::WS) - .expect("Failed to connect to Ethereum WS") + let transport = WsConnect::new(ws.to_string()); + + Transport::WS(transport) } /// Creates a JSON-RPC over HTTP transport. /// - /// Note: JSON-RPC over HTTP doesn't always support subscribing to new - /// blocks (one such example is Infura's HTTP endpoint). + /// Set `no_eip2718` to true for chains that don't return the `type` field + /// in transaction receipts (pre-EIP-2718 chains). Use provider feature `no_eip2718`. pub fn new_rpc( rpc: Url, - headers: ::http::HeaderMap, + headers: graph::http::HeaderMap, metrics: Arc, provider: impl AsRef, + no_eip2718: bool, + compression: Compression, ) -> Self { - // Unwrap: This only fails if something is wrong with the system's TLS config. - let client = reqwest::Client::builder() - .default_headers(headers) - .build() - .unwrap(); - - Transport::RPC { - client: http::Http::with_client(client, rpc), - metrics, - provider: provider.as_ref().into(), + let mut client_builder = reqwest::Client::builder().default_headers(headers); + + match compression { + Compression::None => {} + Compression::Gzip => { + client_builder = client_builder.gzip(true); + } + Compression::Brotli => { + client_builder = client_builder.brotli(true); + } + Compression::Deflate => { + client_builder = client_builder.deflate(true); + } } + + let client = client_builder.build().expect("Failed to build HTTP client"); + + let patching_transport = PatchingHttp::new(client, rpc, no_eip2718); + let metrics_transport = + MetricsHttp::new(patching_transport, metrics, provider.as_ref().into()); + let rpc_client = alloy::rpc::client::RpcClient::new(metrics_transport, false); + + Transport::RPC(rpc_client) } } -impl web3::Transport for Transport { - type Out = Pin> + Send + 'static>>; +/// Custom HTTP transport wrapper that collects metrics +#[derive(Clone)] +pub struct MetricsHttp { + inner: PatchingHttp, + metrics: Arc, + provider: ProviderName, +} - fn prepare(&self, method: &str, params: Vec) -> (RequestId, Call) { - match self { - Transport::RPC { - client, - metrics: _, - provider: _, - } => client.prepare(method, params), - Transport::IPC(ipc) => ipc.prepare(method, params), - Transport::WS(ws) => ws.prepare(method, params), +impl MetricsHttp { + pub fn new(inner: PatchingHttp, metrics: Arc, provider: ProviderName) -> Self { + Self { + inner, + metrics, + provider, } } +} + +// Implement tower::Service trait for MetricsHttp to intercept RPC calls +impl Service for MetricsHttp { + type Response = ResponsePacket; + type Error = TransportError; + type Future = TransportFut<'static>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } - fn send(&self, id: RequestId, request: Call) -> Self::Out { - match self { - Transport::RPC { - client, - metrics, + fn call(&mut self, request: RequestPacket) -> Self::Future { + let metrics = self.metrics.clone(); + let provider = self.provider.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + // Extract method name from request + let method = match &request { + RequestPacket::Single(req) => req.method().to_string(), + RequestPacket::Batch(reqs) => reqs + .first() + .map(|r| r.method().to_string()) + .unwrap_or_else(|| "batch".to_string()), + }; + + let labels = RequestLabels { provider, - } => { - let metrics = metrics.cheap_clone(); - let client = client.clone(); - let method = match request { - Call::MethodCall(ref m) => m.method.as_str(), - _ => "unknown", - }; - - let labels = RequestLabels { - provider: provider.clone(), - req_type: method.into(), - conn_type: graph::endpoint::ConnectionType::Rpc, - }; - let out = async move { - let labels = labels; - let out = client.send(id, request).await; - match out { - Ok(_) => metrics.success(&labels), - Err(_) => metrics.failure(&labels), - } - - out - }; - - Box::pin(out) + req_type: method.into(), + conn_type: ConnectionType::Rpc, + }; + + // Call inner transport and track metrics + let result = inner.call(request).await; + + match &result { + Ok(_) => metrics.success(&labels), + Err(_) => metrics.failure(&labels), } - Transport::IPC(ipc) => Box::pin(ipc.send(id, request)), - Transport::WS(ws) => Box::pin(ws.send(id, request)), - } + + result + }) } } -impl web3::BatchTransport for Transport { - type Batch = Box< - dyn Future>, web3::error::Error>> - + Send - + Unpin, - >; - - fn send_batch(&self, requests: T) -> Self::Batch - where - T: IntoIterator, - { - match self { - Transport::RPC { - client, - metrics: _, - provider: _, - } => Box::new(client.send_batch(requests)), - Transport::IPC(ipc) => Box::new(ipc.send_batch(requests)), - Transport::WS(ws) => Box::new(ws.send_batch(requests)), +/// HTTP transport that patches receipts for chains that don't support EIP-2718 (typed transactions). +/// When `no_eip2718` is set, adds missing `type` field to receipts. +#[derive(Clone)] +pub struct PatchingHttp { + client: reqwest::Client, + url: Url, + no_eip2718: bool, +} + +impl PatchingHttp { + pub fn new(client: reqwest::Client, url: Url, no_eip2718: bool) -> Self { + Self { + client, + url, + no_eip2718, + } + } + + fn is_receipt_method(method: &str) -> bool { + method == "eth_getTransactionReceipt" || method == "eth_getBlockReceipts" + } + + fn patch_rpc_response(response: &mut Value) -> bool { + response + .get_mut("result") + .map(json_patch::patch_receipts) + .unwrap_or(false) + } + + fn patch_response(body: &[u8]) -> Option> { + let mut json: Value = serde_json::from_slice(body).ok()?; + + let patched = match &mut json { + Value::Object(_) => Self::patch_rpc_response(&mut json), + Value::Array(batch) => { + let mut patched = false; + for r in batch { + patched |= Self::patch_rpc_response(r); + } + patched + } + _ => false, + }; + + if patched { + serde_json::to_vec(&json).ok() + } else { + None } } } + +impl Service for PatchingHttp { + type Response = ResponsePacket; + type Error = TransportError; + type Future = TransportFut<'static>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, request: RequestPacket) -> Self::Future { + let client = self.client.clone(); + let url = self.url.clone(); + let no_eip2718 = self.no_eip2718; + + let should_patch = if no_eip2718 { + match &request { + RequestPacket::Single(req) => Self::is_receipt_method(req.method()), + RequestPacket::Batch(reqs) => { + reqs.iter().any(|r| Self::is_receipt_method(r.method())) + } + } + } else { + false + }; + + Box::pin(async move { + let resp = client + .post(url) + .json(&request) + .headers(request.headers()) + .send() + .await + .map_err(TransportErrorKind::custom)?; + + let status = resp.status(); + let body = resp.bytes().await.map_err(TransportErrorKind::custom)?; + + if !status.is_success() { + return Err(TransportErrorKind::http_error( + status.as_u16(), + String::from_utf8_lossy(&body).into_owned(), + )); + } + + if should_patch && let Some(patched) = Self::patch_response(&body) { + return serde_json::from_slice(&patched).map_err(|err| { + TransportError::deser_err(err, String::from_utf8_lossy(&patched)) + }); + } + serde_json::from_slice(&body) + .map_err(|err| TransportError::deser_err(err, String::from_utf8_lossy(&body))) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn patch_response_single() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"status":"0x1"}}"#; + let patched = PatchingHttp::patch_response(body).unwrap(); + let json: Value = serde_json::from_slice(&patched).unwrap(); + assert_eq!(json["result"]["type"], "0x0"); + } + + #[test] + fn patch_response_returns_none_when_type_exists() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"status":"0x1","type":"0x2"}}"#; + assert!(PatchingHttp::patch_response(body).is_none()); + } + + #[test] + fn patch_response_batch() { + let body = br#"[{"jsonrpc":"2.0","id":1,"result":{"status":"0x1"}},{"jsonrpc":"2.0","id":2,"result":{"status":"0x1"}}]"#; + let patched = PatchingHttp::patch_response(body).unwrap(); + let json: Value = serde_json::from_slice(&patched).unwrap(); + assert_eq!(json[0]["result"]["type"], "0x0"); + assert_eq!(json[1]["result"]["type"], "0x0"); + } +} diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index a57f4c8410f..b5d51d9a379 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -1,31 +1,28 @@ +use async_trait::async_trait; +use graph::abi; use graph::blockchain::MappingTriggerTrait; use graph::blockchain::TriggerData; +use graph::components::ethereum::AnyTransaction; +use graph::components::ethereum::AnyTransactionReceiptBare as AlloyTransactionReceipt; use graph::data::subgraph::API_VERSION_0_0_2; use graph::data::subgraph::API_VERSION_0_0_6; use graph::data::subgraph::API_VERSION_0_0_7; -use graph::prelude::ethabi::ethereum_types::H160; -use graph::prelude::ethabi::ethereum_types::H256; -use graph::prelude::ethabi::ethereum_types::U128; -use graph::prelude::ethabi::ethereum_types::U256; -use graph::prelude::ethabi::ethereum_types::U64; -use graph::prelude::ethabi::Address; -use graph::prelude::ethabi::Bytes; -use graph::prelude::ethabi::LogParam; -use graph::prelude::web3::types::Block; -use graph::prelude::web3::types::Log; -use graph::prelude::web3::types::Transaction; -use graph::prelude::web3::types::TransactionReceipt; +use graph::data_source::common::DeclaredCall; use graph::prelude::BlockNumber; use graph::prelude::BlockPtr; +use graph::prelude::LightEthereumBlock; +use graph::prelude::alloy::consensus::Transaction as TransactionTrait; +use graph::prelude::alloy::network::TransactionResponse; +use graph::prelude::alloy::primitives::{Address, B256, U256}; +use graph::prelude::alloy::rpc::types::Log; use graph::prelude::{CheapClone, EthereumCall}; -use graph::runtime::asc_new; -use graph::runtime::gas::GasCounter; use graph::runtime::AscHeap; use graph::runtime::AscPtr; use graph::runtime::HostExportError; +use graph::runtime::asc_new; +use graph::runtime::gas::GasCounter; use graph::semver::Version; use graph_runtime_wasm::module::ToAscPtr; -use std::ops::Deref; use std::{cmp::Ordering, sync::Arc}; use crate::runtime::abi::AscEthereumBlock; @@ -38,23 +35,23 @@ use crate::runtime::abi::AscEthereumTransaction_0_0_1; use crate::runtime::abi::AscEthereumTransaction_0_0_2; use crate::runtime::abi::AscEthereumTransaction_0_0_6; -// ETHDEP: This should be defined in only one place. -type LightEthereumBlock = Block; +static U256_DEFAULT: U256 = U256::ZERO; pub enum MappingTrigger { Log { block: Arc, - transaction: Arc, + transaction: Arc, log: Arc, - params: Vec, - receipt: Option>, + params: Vec, + receipt: Option>, + calls: Vec, }, Call { block: Arc, - transaction: Arc, + transaction: Arc, call: Arc, - inputs: Vec, - outputs: Vec, + inputs: Vec, + outputs: Vec, }, Block { block: Arc, @@ -62,7 +59,7 @@ pub enum MappingTrigger { } impl MappingTriggerTrait for MappingTrigger { - fn error_context(&self) -> std::string::String { + fn error_context(&self) -> String { let transaction_id = match self { MappingTrigger::Log { log, .. } => log.transaction_hash, MappingTrigger::Call { call, .. } => call.transaction_hash, @@ -82,15 +79,15 @@ impl std::fmt::Debug for MappingTrigger { #[derive(Debug)] enum MappingTriggerWithoutBlock { Log { - _transaction: Arc, + _transaction: Arc, _log: Arc, - _params: Vec, + _params: Vec, }, Call { - _transaction: Arc, + _transaction: Arc, _call: Arc, - _inputs: Vec, - _outputs: Vec, + _inputs: Vec, + _outputs: Vec, }, Block, } @@ -102,6 +99,7 @@ impl std::fmt::Debug for MappingTrigger { log, params, receipt: _, + calls: _, } => MappingTriggerWithoutBlock::Log { _transaction: transaction.cheap_clone(), _log: log.cheap_clone(), @@ -126,8 +124,9 @@ impl std::fmt::Debug for MappingTrigger { } } +#[async_trait] impl ToAscPtr for MappingTrigger { - fn to_asc_ptr( + async fn to_asc_ptr( self, heap: &mut H, gas: &GasCounter, @@ -139,18 +138,16 @@ impl ToAscPtr for MappingTrigger { log, params, receipt, + calls: _, } => { let api_version = heap.api_version(); - let ethereum_event_data = EthereumEventData { - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), - address: log.address, - log_index: log.log_index.unwrap_or(U256::zero()), - transaction_log_index: log.log_index.unwrap_or(U256::zero()), - log_type: log.log_type.clone(), - params, - }; - if api_version >= API_VERSION_0_0_7 { + let ethereum_event_data = EthereumEventData::new( + block.as_ref(), + transaction.as_ref(), + log.as_ref(), + ¶ms, + ); + if api_version >= &API_VERSION_0_0_7 { asc_new::< AscEthereumEvent_0_0_7< AscEthereumTransaction_0_0_6, @@ -158,28 +155,31 @@ impl ToAscPtr for MappingTrigger { >, _, _, - >(heap, &(ethereum_event_data, receipt.as_deref()), gas)? + >(heap, &(ethereum_event_data, receipt.as_deref()), gas) + .await? .erase() - } else if api_version >= API_VERSION_0_0_6 { + } else if api_version >= &API_VERSION_0_0_6 { asc_new::< AscEthereumEvent, _, _, - >(heap, ðereum_event_data, gas)? + >(heap, ðereum_event_data, gas) + .await? .erase() - } else if api_version >= API_VERSION_0_0_2 { + } else if api_version >= &API_VERSION_0_0_2 { asc_new::< AscEthereumEvent, _, _, - >(heap, ðereum_event_data, gas)? + >(heap, ðereum_event_data, gas) + .await? .erase() } else { asc_new::< AscEthereumEvent, _, _, - >(heap, ðereum_event_data, gas)? + >(heap, ðereum_event_data, gas).await? .erase() } } @@ -190,87 +190,107 @@ impl ToAscPtr for MappingTrigger { inputs, outputs, } => { - let call = EthereumCallData { - to: call.to, - from: call.from, - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), - inputs, - outputs, - }; - if heap.api_version() >= Version::new(0, 0, 6) { + let call = EthereumCallData::new(&block, &transaction, &call, &inputs, &outputs); + if heap.api_version() >= &Version::new(0, 0, 6) { asc_new::< AscEthereumCall_0_0_3, _, _, - >(heap, &call, gas)? + >(heap, &call, gas) + .await? .erase() - } else if heap.api_version() >= Version::new(0, 0, 3) { + } else if heap.api_version() >= &Version::new(0, 0, 3) { asc_new::< AscEthereumCall_0_0_3, _, _, - >(heap, &call, gas)? + >(heap, &call, gas) + .await? .erase() } else { - asc_new::(heap, &call, gas)?.erase() + asc_new::(heap, &call, gas) + .await? + .erase() } } MappingTrigger::Block { block } => { let block = EthereumBlockData::from(block.as_ref()); - if heap.api_version() >= Version::new(0, 0, 6) { - asc_new::(heap, &block, gas)?.erase() + if heap.api_version() >= &Version::new(0, 0, 6) { + asc_new::(heap, &block, gas) + .await? + .erase() } else { - asc_new::(heap, &block, gas)?.erase() + asc_new::(heap, &block, gas) + .await? + .erase() } } }) } } +#[derive(Clone, Debug)] +pub struct LogPosition { + pub index: usize, + pub receipt: Arc, + pub requires_transaction_receipt: bool, +} + #[derive(Clone, Debug)] pub enum LogRef { - FullLog(Arc, Option>), - LogPosition(usize, Arc), + FullLog(Arc, Option>), + LogPosition(LogPosition), } impl LogRef { pub fn log(&self) -> &Log { match self { LogRef::FullLog(log, _) => log.as_ref(), - LogRef::LogPosition(index, receipt) => receipt.logs.get(*index).unwrap(), + LogRef::LogPosition(pos) => pos.receipt.logs().get(pos.index).unwrap(), } } - pub fn receipt(&self) -> Option<&Arc> { + /// Returns the transaction receipt if it's available and required. + /// + /// For `FullLog` variants, returns the receipt if present. + /// For `LogPosition` variants, only returns the receipt if the + /// `requires_transaction_receipt` flag is true, otherwise returns None + /// even though the receipt is stored internally. + pub fn receipt(&self) -> Option<&Arc> { match self { LogRef::FullLog(_, receipt) => receipt.as_ref(), - LogRef::LogPosition(_, receipt) => Some(receipt), + LogRef::LogPosition(pos) => { + if pos.requires_transaction_receipt { + Some(&pos.receipt) + } else { + None + } + } } } - pub fn log_index(&self) -> Option { + pub fn log_index(&self) -> Option { self.log().log_index } - pub fn transaction_index(&self) -> Option { + pub fn transaction_index(&self) -> Option { self.log().transaction_index } - fn transaction_hash(&self) -> Option { + fn transaction_hash(&self) -> Option { self.log().transaction_hash } - pub fn block_hash(&self) -> Option { + pub fn block_hash(&self) -> Option { self.log().block_hash } - pub fn block_number(&self) -> Option { + pub fn block_number(&self) -> Option { self.log().block_number } - pub fn address(&self) -> &H160 { - &self.log().address + pub fn address(&self) -> &Address { + &self.log().inner.address } } @@ -313,14 +333,14 @@ impl EthereumTrigger { EthereumTrigger::Block(block_ptr, _) => block_ptr.number, EthereumTrigger::Call(call) => call.block_number, EthereumTrigger::Log(log_ref) => { - i32::try_from(log_ref.block_number().unwrap().as_u64()).unwrap() + i32::try_from(log_ref.block_number().unwrap()).unwrap() } } } - pub fn block_hash(&self) -> H256 { + pub fn block_hash(&self) -> B256 { match self { - EthereumTrigger::Block(block_ptr, _) => block_ptr.hash_as_h256(), + EthereumTrigger::Block(block_ptr, _) => block_ptr.hash.as_b256(), EthereumTrigger::Call(call) => call.block_hash, EthereumTrigger::Log(log_ref) => log_ref.block_hash().unwrap(), } @@ -333,7 +353,7 @@ impl EthereumTrigger { Some(address) } EthereumTrigger::Call(call) => Some(&call.to), - EthereumTrigger::Log(log_ref) => Some(&log_ref.address()), + EthereumTrigger::Log(log_ref) => Some(log_ref.address()), // Unfiltered block triggers match any data source address. EthereumTrigger::Block(_, EthereumBlockTriggerType::End) => None, EthereumTrigger::Block(_, EthereumBlockTriggerType::Start) => None, @@ -364,23 +384,21 @@ impl Ord for EthereumTrigger { // Calls vs. events are logged by their tx index; // if they are from the same transaction, events come first (Self::Call(a), Self::Log(b)) - if a.transaction_index == b.transaction_index().unwrap().as_u64() => + if a.transaction_index == b.transaction_index().unwrap() => { Ordering::Greater } (Self::Log(a), Self::Call(b)) - if a.transaction_index().unwrap().as_u64() == b.transaction_index => + if a.transaction_index().unwrap() == b.transaction_index => { Ordering::Less } - (Self::Call(a), Self::Log(b)) => a - .transaction_index - .cmp(&b.transaction_index().unwrap().as_u64()), - (Self::Log(a), Self::Call(b)) => a - .transaction_index() - .unwrap() - .as_u64() - .cmp(&b.transaction_index), + (Self::Call(a), Self::Log(b)) => { + a.transaction_index.cmp(&b.transaction_index().unwrap()) + } + (Self::Log(a), Self::Call(b)) => { + a.transaction_index().unwrap().cmp(&b.transaction_index) + } } } } @@ -411,104 +429,223 @@ impl TriggerData for EthereumTrigger { } fn address_match(&self) -> Option<&[u8]> { - self.address().map(|address| address.as_bytes()) + self.address().map(|address| address.as_slice()) } } /// Ethereum block data. -#[derive(Clone, Debug, Default)] -pub struct EthereumBlockData { - pub hash: H256, - pub parent_hash: H256, - pub uncles_hash: H256, - pub author: H160, - pub state_root: H256, - pub transactions_root: H256, - pub receipts_root: H256, - pub number: U64, - pub gas_used: U256, - pub gas_limit: U256, - pub timestamp: U256, - pub difficulty: U256, - pub total_difficulty: U256, - pub size: Option, - pub base_fee_per_gas: Option, +#[derive(Clone, Debug)] +pub struct EthereumBlockData<'a> { + block: &'a LightEthereumBlock, } -impl<'a, T> From<&'a Block> for EthereumBlockData { - fn from(block: &'a Block) -> EthereumBlockData { - EthereumBlockData { - hash: block.hash.unwrap(), - parent_hash: block.parent_hash, - uncles_hash: block.uncles_hash, - author: block.author, - state_root: block.state_root, - transactions_root: block.transactions_root, - receipts_root: block.receipts_root, - number: block.number.unwrap(), - gas_used: block.gas_used, - gas_limit: block.gas_limit, - timestamp: block.timestamp, - difficulty: block.difficulty, - total_difficulty: block.total_difficulty.unwrap_or_default(), - size: block.size, - base_fee_per_gas: block.base_fee_per_gas, - } +impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData<'a> { + fn from(block: &'a LightEthereumBlock) -> EthereumBlockData<'a> { + EthereumBlockData { block } + } +} + +impl<'a> EthereumBlockData<'a> { + pub fn hash(&self) -> &B256 { + &self.block.inner().header.hash + } + + pub fn parent_hash(&self) -> &B256 { + &self.block.inner().header.parent_hash + } + + pub fn uncles_hash(&self) -> &B256 { + &self.block.inner().header.ommers_hash + } + + pub fn author(&self) -> &Address { + &self.block.inner().header.beneficiary + } + + pub fn state_root(&self) -> &B256 { + &self.block.inner().header.state_root + } + + pub fn transactions_root(&self) -> &B256 { + &self.block.inner().header.transactions_root + } + + pub fn receipts_root(&self) -> &B256 { + &self.block.inner().header.receipts_root + } + + pub fn number(&self) -> u64 { + self.block.number_u64() + } + + pub fn gas_used(&self) -> u64 { + self.block.inner().header.gas_used + } + + pub fn gas_limit(&self) -> u64 { + self.block.inner().header.gas_limit + } + + pub fn timestamp(&self) -> u64 { + self.block.inner().header.timestamp + } + + pub fn difficulty(&self) -> &U256 { + &self.block.inner().header.difficulty + } + + pub fn total_difficulty(&self) -> &U256 { + self.block + .inner() + .header + .total_difficulty + .as_ref() + .unwrap_or(&U256_DEFAULT) + } + + pub fn size(&self) -> &Option { + &self.block.inner().header.size + } + + pub fn base_fee_per_gas(&self) -> &Option { + &self.block.inner().header.base_fee_per_gas } } /// Ethereum transaction data. #[derive(Clone, Debug)] -pub struct EthereumTransactionData { - pub hash: H256, - pub index: U128, - pub from: H160, - pub to: Option, - pub value: U256, - pub gas_limit: U256, - pub gas_price: U256, - pub input: Bytes, - pub nonce: U256, +pub struct EthereumTransactionData<'a> { + tx: &'a AnyTransaction, + base_fee_per_gas: Option, } -impl From<&'_ Transaction> for EthereumTransactionData { - fn from(tx: &Transaction) -> EthereumTransactionData { - // unwrap: this is always `Some` for txns that have been mined - // (see https://github.com/tomusdrw/rust-web3/pull/407) - let from = tx.from.unwrap(); +impl<'a> EthereumTransactionData<'a> { + // We don't implement `From` because it causes confusion with the `from` + // accessor method + fn new(tx: &'a AnyTransaction, base_fee_per_gas: Option) -> EthereumTransactionData<'a> { EthereumTransactionData { - hash: tx.hash, - index: tx.transaction_index.unwrap().as_u64().into(), - from, - to: tx.to, - value: tx.value, - gas_limit: tx.gas, - gas_price: tx.gas_price.unwrap_or(U256::zero()), // EIP-1559 made this optional. - input: tx.input.0.clone(), - nonce: tx.nonce, + tx, + base_fee_per_gas, } } + + pub fn hash(&self) -> B256 { + self.tx.tx_hash() + } + + pub fn index(&self) -> u64 { + self.tx.transaction_index.unwrap() + } + + pub fn from(&self) -> Address { + self.tx.from() + } + + pub fn to(&self) -> Option
{ + self.tx.to() + } + + pub fn value(&self) -> U256 { + self.tx.value() + } + + pub fn gas_limit(&self) -> u64 { + self.tx.gas_limit() + } + + pub fn gas_price(&self) -> u128 { + self.tx.effective_gas_price(self.base_fee_per_gas) + } + + pub fn input(&self) -> &[u8] { + self.tx.input() + } + + pub fn nonce(&self) -> u64 { + self.tx.nonce() + } } /// An Ethereum event logged from a specific contract address and block. #[derive(Debug, Clone)] -pub struct EthereumEventData { - pub address: Address, - pub log_index: U256, - pub transaction_log_index: U256, - pub log_type: Option, - pub block: EthereumBlockData, - pub transaction: EthereumTransactionData, - pub params: Vec, +pub struct EthereumEventData<'a> { + pub block: EthereumBlockData<'a>, + pub transaction: EthereumTransactionData<'a>, + pub params: &'a [abi::DynSolParam], + log: &'a Log, +} + +impl<'a> EthereumEventData<'a> { + pub fn new( + block: &'a LightEthereumBlock, + tx: &'a AnyTransaction, + log: &'a Log, + params: &'a [abi::DynSolParam], + ) -> Self { + EthereumEventData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(tx, block.base_fee_per_gas()), + log, + params, + } + } + + pub fn address(&self) -> &Address { + &self.log.inner.address + } + + pub fn log_index(&self) -> u64 { + self.log.log_index.unwrap_or(0) + } + + pub fn transaction_log_index(&self) -> u64 { + // We purposely use the `log_index` here. Geth does not support + // `transaction_log_index`, and subgraphs that use it only care that + // it identifies the log, the specific value is not important. Still + // this will change the output of subgraphs that use this field. + // + // This was initially changed in commit b95c6953 + self.log.log_index.unwrap_or(0) + } + + pub fn log_type(&self) -> Option { + // This field was present in old rust-web3 Block, but alloy doesn't have it. + None + } } /// An Ethereum call executed within a transaction within a block to a contract address. #[derive(Debug, Clone)] -pub struct EthereumCallData { - pub from: Address, - pub to: Address, - pub block: EthereumBlockData, - pub transaction: EthereumTransactionData, - pub inputs: Vec, - pub outputs: Vec, +pub struct EthereumCallData<'a> { + pub block: EthereumBlockData<'a>, + pub transaction: EthereumTransactionData<'a>, + pub inputs: &'a [abi::DynSolParam], + pub outputs: &'a [abi::DynSolParam], + call: &'a EthereumCall, +} + +impl<'a> EthereumCallData<'a> { + fn new( + block: &'a LightEthereumBlock, + transaction: &'a AnyTransaction, + call: &'a EthereumCall, + inputs: &'a [abi::DynSolParam], + outputs: &'a [abi::DynSolParam], + ) -> EthereumCallData<'a> { + EthereumCallData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(transaction, block.base_fee_per_gas()), + inputs, + outputs, + call, + } + } + + pub fn from(&self) -> &Address { + &self.call.from + } + + pub fn to(&self) -> &Address { + &self.call.to + } } diff --git a/chain/near/Cargo.toml b/chain/near/Cargo.toml index 9f43c90a44e..d21f3755d0e 100644 --- a/chain/near/Cargo.toml +++ b/chain/near/Cargo.toml @@ -4,19 +4,19 @@ version.workspace = true edition.workspace = true [build-dependencies] -tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } [dependencies] -base64 = "0.20" +async-trait = { workspace = true } graph = { path = "../../graph" } prost = { workspace = true } prost-types = { workspace = true } -serde = "1.0" +serde = { workspace = true } anyhow = "1" graph-runtime-wasm = { path = "../../runtime/wasm" } graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] -diesel = { version = "1.4.7", features = ["postgres", "serde_json", "numeric", "r2d2"] } -trigger-filters.path = "../../substreams/trigger-filters" +diesel = { workspace = true } +tokio = { workspace = true } diff --git a/chain/near/build.rs b/chain/near/build.rs index 611f861baf2..feafe4718f9 100644 --- a/chain/near/build.rs +++ b/chain/near/build.rs @@ -1,11 +1,8 @@ fn main() { println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() + tonic_prost_build::configure() .out_dir("src/protobuf") .extern_path(".sf.near.codec.v1", "crate::codec::pbcodec") - .compile( - &["proto/near.proto", "proto/substreams-triggers.proto"], - &["proto"], - ) + .compile_protos(&["proto/near.proto"], &["proto"]) .expect("Failed to compile Firehose NEAR proto(s)"); } diff --git a/chain/near/proto/substreams-triggers.proto b/chain/near/proto/substreams-triggers.proto deleted file mode 100644 index 947052a2566..00000000000 --- a/chain/near/proto/substreams-triggers.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -import "near.proto"; - -package receipts.v1; - -message BlockAndReceipts { - sf.near.codec.v1.Block block = 1; - repeated sf.near.codec.v1.ExecutionOutcomeWithId outcome = 2; - repeated sf.near.codec.v1.Receipt receipt = 3; -} - diff --git a/chain/near/src/adapter.rs b/chain/near/src/adapter.rs index 4d6151aa5ca..31c7dca7c97 100644 --- a/chain/near/src/adapter.rs +++ b/chain/near/src/adapter.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use crate::data_source::PartialAccounts; -use crate::{data_source::DataSource, Chain}; +use crate::{Chain, data_source::DataSource}; use graph::blockchain as bc; use graph::firehose::{BasicReceiptFilter, PrefixSuffixPair}; use graph::itertools::Itertools; @@ -251,14 +251,13 @@ mod test { use std::collections::HashSet; use super::NearBlockFilter; - use crate::adapter::{NearReceiptFilter, TriggerFilter, BASIC_RECEIPT_FILTER_TYPE_URL}; + use crate::adapter::{BASIC_RECEIPT_FILTER_TYPE_URL, TriggerFilter}; use graph::{ blockchain::TriggerFilter as _, firehose::{BasicReceiptFilter, PrefixSuffixPair}, }; use prost::Message; use prost_types::Any; - use trigger_filters::NearFilter; #[test] fn near_trigger_empty_filter() { @@ -340,7 +339,7 @@ mod test { let firehose_filter = decode_filter(filter); assert_eq!(firehose_filter.accounts, vec![String::from("acc1"),],); - let expected_pairs = vec![ + let expected_pairs = [ PrefixSuffixPair { prefix: "acc3".to_string(), suffix: "acc4".to_string(), @@ -357,132 +356,13 @@ mod test { let pairs = firehose_filter.prefix_and_suffix_pairs; assert_eq!(pairs.len(), 3); - assert_eq!( - true, + assert!( expected_pairs.iter().all(|x| pairs.contains(x)), "{:?}", pairs ); } - #[test] - fn test_near_filter_params_serialization() -> anyhow::Result<()> { - struct Case<'a> { - name: &'a str, - input: NearReceiptFilter, - expected: NearFilter<'a>, - } - - let cases = vec![ - Case { - name: "empty", - input: NearReceiptFilter::default(), - expected: NearFilter::default(), - }, - Case { - name: "only full matches", - input: super::NearReceiptFilter { - accounts: HashSet::from_iter(vec!["acc1".into()]), - partial_accounts: HashSet::new(), - }, - expected: NearFilter { - accounts: HashSet::from_iter(vec!["acc1"]), - partial_accounts: HashSet::default(), - }, - }, - Case { - name: "only partial matches", - input: super::NearReceiptFilter { - accounts: HashSet::new(), - partial_accounts: HashSet::from_iter(vec![(Some("acc1".into()), None)]), - }, - expected: NearFilter { - accounts: HashSet::default(), - partial_accounts: HashSet::from_iter(vec![(Some("acc1"), None)]), - }, - }, - Case { - name: "both 1len matches", - input: super::NearReceiptFilter { - accounts: HashSet::from_iter(vec!["acc1".into()]), - partial_accounts: HashSet::from_iter(vec![(Some("s1".into()), None)]), - }, - expected: NearFilter { - accounts: HashSet::from_iter(vec!["acc1"]), - partial_accounts: HashSet::from_iter(vec![(Some("s1"), None)]), - }, - }, - Case { - name: "more partials matches", - input: super::NearReceiptFilter { - accounts: HashSet::from_iter(vec!["acc1".into()]), - partial_accounts: HashSet::from_iter(vec![ - (Some("s1".into()), None), - (None, Some("s3".into())), - (Some("s2".into()), Some("s2".into())), - ]), - }, - expected: NearFilter { - accounts: HashSet::from_iter(vec!["acc1"]), - partial_accounts: HashSet::from_iter(vec![ - (Some("s1"), None), - (None, Some("s3")), - (Some("s2"), Some("s2")), - ]), - }, - }, - Case { - name: "both matches", - input: NearReceiptFilter { - accounts: HashSet::from_iter(vec![ - "acc1".into(), - "=12-30786jhasdgmasd".into(), - "^&%^&^$".into(), - "acc3".into(), - ]), - partial_accounts: HashSet::from_iter(vec![ - (Some("1.2.2.3.45.5".into()), None), - (None, Some("kjysdfoiua6sd".into())), - (Some("120938pokasd".into()), Some("102938poai[sd]".into())), - ]), - }, - expected: NearFilter { - accounts: HashSet::from_iter(vec![ - "acc1", - "=12-30786jhasdgmasd", - "^&%^&^$", - "acc3", - ]), - partial_accounts: HashSet::from_iter(vec![ - (Some("1.2.2.3.45.5"), None), - (None, Some("kjysdfoiua6sd")), - (Some("120938pokasd"), Some("102938poai[sd]")), - ]), - }, - }, - ]; - - for case in cases.into_iter() { - let tf = TriggerFilter { - block_filter: NearBlockFilter::default(), - receipt_filter: case.input, - }; - let param = tf.to_module_params(); - let filter = NearFilter::try_from(param.as_str()).expect(&format!( - "case: {}, the filter to parse params correctly", - case.name - )); - - assert_eq!( - filter, case.expected, - "case {},param:\n{}", - case.name, param - ); - } - - Ok(()) - } - fn decode_filter(firehose_filter: Vec) -> BasicReceiptFilter { let firehose_filter = firehose_filter[0].clone(); assert_eq!( diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index e8925c89366..5698301b5e8 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -1,38 +1,37 @@ -use anyhow::anyhow; +use async_trait::async_trait; use graph::blockchain::client::ChainClient; use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; -use graph::blockchain::substreams_block_stream::SubstreamsBlockStream; use graph::blockchain::{ - BasicBlockchainBuilder, BlockIngestor, BlockchainBuilder, BlockchainKind, NoopRuntimeAdapter, + BlockIngestor, BlockchainKind, NoopDecoderHook, NoopRuntimeAdapter, TriggerFilterWrapper, }; use graph::cheap_clone::CheapClone; -use graph::components::store::DeploymentCursorTracker; +use graph::components::network_provider::ChainName; +use graph::components::store::{ChainHeadStore, DeploymentCursorTracker, SourceableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::env::EnvVars; -use graph::firehose::FirehoseEndpoint; -use graph::prelude::{MetricsRegistry, TryFutureExt}; -use graph::schema::InputSchema; -use graph::substreams::{Clock, Package}; +use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints}; +use graph::futures03::TryFutureExt; +use graph::prelude::MetricsRegistry; use graph::{ anyhow::Result, blockchain::{ + BlockHash, BlockPtr, Blockchain, EmptyNodeCapabilities, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, block_stream::{ BlockStreamEvent, BlockWithTriggers, FirehoseError, FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, }, firehose_block_stream::FirehoseBlockStream, - BlockHash, BlockPtr, Blockchain, EmptyNodeCapabilities, IngestorError, - RuntimeAdapter as RuntimeAdapterTrait, }, components::store::DeploymentLocator, firehose::{self as firehose, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, + prelude::{BlockNumber, Error, Logger, LoggerFactory, o}, }; use prost::Message; +use std::collections::BTreeSet; use std::sync::Arc; use crate::adapter::TriggerFilter; -use crate::codec::substreams_triggers::BlockAndReceipts; +use crate::codec::Block; use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; use crate::trigger::{self, NearTrigger}; use crate::{ @@ -40,72 +39,13 @@ use crate::{ data_source::{DataSource, UnresolvedDataSource}, }; use graph::blockchain::block_stream::{ - BlockStream, BlockStreamBuilder, BlockStreamMapper, FirehoseCursor, + BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, }; -const NEAR_FILTER_MODULE_NAME: &str = "near_filter"; -const SUBSTREAMS_TRIGGER_FILTER_BYTES: &[u8; 510162] = include_bytes!( - "../../../substreams/substreams-trigger-filter/substreams-trigger-filter-v0.1.0.spkg" -); - pub struct NearStreamBuilder {} #[async_trait] impl BlockStreamBuilder for NearStreamBuilder { - async fn build_substreams( - &self, - chain: &Chain, - _schema: InputSchema, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - subgraph_current_block: Option, - filter: Arc<::TriggerFilter>, - ) -> Result>> { - let mapper = Arc::new(FirehoseMapper { - adapter: Arc::new(TriggersAdapter {}), - filter, - }); - let mut package = - Package::decode(SUBSTREAMS_TRIGGER_FILTER_BYTES.to_vec().as_ref()).unwrap(); - match package.modules.as_mut() { - Some(modules) => modules - .modules - .iter_mut() - .find(|module| module.name == NEAR_FILTER_MODULE_NAME) - .map(|module| { - graph::substreams::patch_module_params( - mapper.filter.to_module_params(), - module, - ); - module - }), - None => None, - }; - - let logger = chain - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "SubstreamsBlockStream")); - let start_block = subgraph_current_block - .as_ref() - .map(|b| b.number) - .unwrap_or_default(); - - Ok(Box::new(SubstreamsBlockStream::new( - deployment.hash, - chain.chain_client(), - subgraph_current_block, - block_cursor.as_ref().clone(), - mapper, - package.modules.clone(), - NEAR_FILTER_MODULE_NAME.to_string(), - vec![start_block], - vec![], - logger, - chain.metrics_registry.clone(), - ))) - } - async fn build_firehose( &self, chain: &Chain, @@ -148,8 +88,9 @@ impl BlockStreamBuilder for NearStreamBuilder { _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec>, _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { todo!() @@ -158,12 +99,11 @@ impl BlockStreamBuilder for NearStreamBuilder { pub struct Chain { logger_factory: LoggerFactory, - name: String, + name: ChainName, client: Arc>, - chain_store: Arc, + chain_head_store: Arc, metrics_registry: Arc, block_stream_builder: Arc>, - prefer_substreams: bool, } impl std::fmt::Debug for Chain { @@ -172,16 +112,21 @@ impl std::fmt::Debug for Chain { } } -impl BlockchainBuilder for BasicBlockchainBuilder { - fn build(self, config: &Arc) -> Chain { +impl Chain { + pub fn new( + logger_factory: LoggerFactory, + name: ChainName, + chain_head_store: Arc, + firehose_endpoints: FirehoseEndpoints, + metrics_registry: Arc, + ) -> Self { Chain { - logger_factory: self.logger_factory, - name: self.name, - chain_store: self.chain_store, - client: Arc::new(ChainClient::new_firehose(self.firehose_endpoints)), - metrics_registry: self.metrics_registry, + logger_factory, + name, + chain_head_store, + client: Arc::new(ChainClient::new_firehose(firehose_endpoints)), + metrics_registry, block_stream_builder: Arc::new(NearStreamBuilder {}), - prefer_substreams: config.prefer_substreams_block_streams, } } } @@ -209,6 +154,8 @@ impl Blockchain for Chain { type NodeCapabilities = EmptyNodeCapabilities; + type DecoderHook = NoopDecoderHook; + fn triggers_adapter( &self, _loc: &DeploymentLocator, @@ -224,23 +171,10 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + _source_subgraph_stores: Vec>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { - if self.prefer_substreams { - return self - .block_stream_builder - .build_substreams( - self, - store.input_schema(), - deployment, - store.firehose_cursor(), - store.block_ptr(), - filter, - ) - .await; - } - self.block_stream_builder .build_firehose( self, @@ -248,7 +182,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, store.block_ptr(), - filter, + filter.chain_filter.clone(), unified_api_version, ) .await @@ -263,11 +197,13 @@ impl Blockchain for Chain { _logger: &Logger, _cursor: FirehoseCursor, ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") + unimplemented!( + "This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called." + ) } - fn chain_store(&self) -> Arc { - self.chain_store.clone() + async fn chain_head_ptr(&self) -> Result, Error> { + self.chain_head_store.cheap_clone().chain_head_ptr().await } async fn block_pointer_from_number( @@ -275,7 +211,7 @@ impl Blockchain for Chain { logger: &Logger, number: BlockNumber, ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint()?; + let firehose_endpoint = self.client.firehose_endpoint().await?; firehose_endpoint .block_ptr_for_number::(logger, number) @@ -283,17 +219,19 @@ impl Blockchain for Chain { .await } - fn runtime_adapter(&self) -> Arc> { - Arc::new(NoopRuntimeAdapter::default()) + async fn runtime( + &self, + ) -> anyhow::Result<(Arc>, Self::DecoderHook)> { + Ok((Arc::new(NoopRuntimeAdapter::default()), NoopDecoderHook)) } fn chain_client(&self) -> Arc> { self.client.clone() } - fn block_ingestor(&self) -> anyhow::Result> { + async fn block_ingestor(&self) -> anyhow::Result> { let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), + self.chain_head_store.cheap_clone(), self.chain_client(), self.logger_factory .component_logger("NearFirehoseBlockIngestor", None), @@ -312,10 +250,22 @@ impl TriggersAdapterTrait for TriggersAdapter { _from: BlockNumber, _to: BlockNumber, _filter: &TriggerFilter, - ) -> Result>, Error> { + ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn load_block_ptrs_by_numbers( + &self, + _logger: Logger, + _block_numbers: BTreeSet, + ) -> Result> { + unimplemented!() + } + + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + async fn triggers_in_block( &self, logger: &Logger, @@ -386,6 +336,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _ptr: BlockPtr, _offset: BlockNumber, + _root: Option, ) -> Result, Error> { panic!("Should never be called since FirehoseBlockStream cannot resolve it") } @@ -408,10 +359,18 @@ pub struct FirehoseMapper { #[async_trait] impl BlockStreamMapper for FirehoseMapper { - fn decode_block(&self, output: Option<&[u8]>) -> Result, Error> { + fn decode_block( + &self, + output: Option<&[u8]>, + ) -> Result, BlockStreamError> { let block = match output { Some(block) => codec::Block::decode(block)?, - None => anyhow::bail!("near mapper is expected to always have a block"), + None => { + return Err(anyhow::anyhow!( + "near mapper is expected to always have a block" + )) + .map_err(BlockStreamError::from); + } }; Ok(Some(block)) @@ -421,46 +380,11 @@ impl BlockStreamMapper for FirehoseMapper { &self, logger: &Logger, block: codec::Block, - ) -> Result, Error> { + ) -> Result, BlockStreamError> { self.adapter .triggers_in_block(logger, block, self.filter.as_ref()) .await - } - - async fn handle_substreams_block( - &self, - _logger: &Logger, - _clock: Clock, - cursor: FirehoseCursor, - message: Vec, - ) -> Result, Error> { - let BlockAndReceipts { - block, - outcome, - receipt, - } = BlockAndReceipts::decode(message.as_ref())?; - let block = block.ok_or_else(|| anyhow!("near block is mandatory on substreams"))?; - let arc_block = Arc::new(block.clone()); - - let trigger_data = outcome - .into_iter() - .zip(receipt.into_iter()) - .map(|(outcome, receipt)| { - NearTrigger::Receipt(Arc::new(trigger::ReceiptWithOutcome { - outcome, - receipt, - block: arc_block.clone(), - })) - }) - .collect(); - - Ok(BlockStreamEvent::ProcessBlock( - BlockWithTriggers { - block, - trigger_data, - }, - cursor, - )) + .map_err(BlockStreamError::from) } } @@ -475,7 +399,7 @@ impl FirehoseMapperTrait for FirehoseMapper { logger: &Logger, response: &firehose::Response, ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + let step = ForkStep::try_from(response.step).unwrap_or_else(|_| { panic!( "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", response.step @@ -517,7 +441,9 @@ impl FirehoseMapperTrait for FirehoseMapper { } StepFinal => { - panic!("irreversible step is not handled and should not be requested in the Firehose request") + panic!( + "irreversible step is not handled and should not be requested in the Firehose request" + ) } StepUnset => { @@ -555,22 +481,23 @@ mod test { use std::{collections::HashSet, sync::Arc, vec}; use graph::{ - blockchain::{block_stream::BlockWithTriggers, DataSource as _, TriggersAdapter as _}, - prelude::{tokio, Link}, + blockchain::{DataSource as _, TriggersAdapter as _, block_stream::BlockWithTriggers}, + data::subgraph::LATEST_VERSION, + prelude::Link, semver::Version, - slog::{self, o, Logger}, + slog::{self, Logger, o}, }; use crate::{ + Chain, adapter::{NearReceiptFilter, TriggerFilter}, codec::{ - self, execution_outcome, receipt, Block, BlockHeader, DataReceiver, ExecutionOutcome, - ExecutionOutcomeWithId, IndexerExecutionOutcomeWithReceipt, IndexerShard, - ReceiptAction, SuccessValueExecutionStatus, + self, Block, BlockHeader, DataReceiver, ExecutionOutcome, ExecutionOutcomeWithId, + IndexerExecutionOutcomeWithReceipt, IndexerShard, ReceiptAction, + SuccessValueExecutionStatus, execution_outcome, receipt, }, - data_source::{DataSource, Mapping, PartialAccounts, ReceiptHandler, NEAR_KIND}, + data_source::{DataSource, Mapping, NEAR_KIND, PartialAccounts, ReceiptHandler}, trigger::{NearTrigger, ReceiptWithOutcome}, - Chain, }; use super::TriggersAdapter; @@ -578,7 +505,7 @@ mod test { #[test] fn validate_empty() { let ds = new_data_source(None, None); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 1, "{:?}", ds); assert_eq!(errs[0].to_string(), "subgraph source address is required"); } @@ -586,7 +513,7 @@ mod test { #[test] fn validate_empty_account_none_partial() { let ds = new_data_source(None, Some(PartialAccounts::default())); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 1, "{:?}", ds); assert_eq!(errs[0].to_string(), "subgraph source address is required"); } @@ -600,7 +527,7 @@ mod test { suffixes: vec!["x.near".to_string()], }), ); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 0, "{:?}", ds); } @@ -614,18 +541,17 @@ mod test { }), ); let errs: Vec = ds - .validate() + .validate(LATEST_VERSION) .into_iter() .map(|err| err.to_string()) .collect(); assert_eq!(errs.len(), 2, "{:?}", ds); - let expected_errors = vec![ + let expected_errors = [ "partial account prefixes can't have empty values".to_string(), "partial account suffixes can't have empty values".to_string(), ]; - assert_eq!( - true, + assert!( expected_errors.iter().all(|err| errs.contains(err)), "{:?}", errs @@ -635,7 +561,7 @@ mod test { #[test] fn validate_empty_partials() { let ds = new_data_source(Some("x.near".to_string()), None); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 0, "{:?}", ds); } @@ -711,8 +637,7 @@ mod test { case.name, receipt.partial_accounts, ); - assert_eq!( - true, + assert!( case.expected .iter() .all(|x| receipt.partial_accounts.contains(x)), @@ -880,7 +805,7 @@ mod test { } } - #[tokio::test] + #[graph::test] async fn test_trigger_filter_empty() { let account1: String = "account1".into(); @@ -898,7 +823,7 @@ mod test { assert_eq!(block_with_triggers.trigger_count(), 0); } - #[tokio::test] + #[graph::test] async fn test_trigger_filter_every_block() { let account1: String = "account1".into(); @@ -924,7 +849,7 @@ mod test { assert_eq!(height, vec![1]); } - #[tokio::test] + #[graph::test] async fn test_trigger_filter_every_receipt() { let account1: String = "account1".into(); @@ -956,14 +881,14 @@ mod test { .trigger_data .clone() .into_iter() - .filter_map(|x| match x { - crate::trigger::NearTrigger::Block(b) => b.header.clone().map(|x| x.height), + .filter_map(|x| match x.as_chain() { + Some(crate::trigger::NearTrigger::Block(b)) => b.header.clone().map(|x| x.height), _ => None, }) .collect() } - fn new_success_block(height: u64, receiver_id: &String) -> codec::Block { + fn new_success_block(height: u64, receiver_id: &str) -> codec::Block { codec::Block { header: Some(BlockHeader { height, @@ -975,12 +900,12 @@ mod test { receipt: Some(crate::codec::Receipt { receipt: Some(receipt::Receipt::Action(ReceiptAction { output_data_receivers: vec![DataReceiver { - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }], ..Default::default() })), - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }), execution_outcome: Some(ExecutionOutcomeWithId { @@ -1030,7 +955,7 @@ mod test { } } - fn new_receipt_with_outcome(receiver_id: &String, block: Arc) -> ReceiptWithOutcome { + fn new_receipt_with_outcome(receiver_id: &str, block: Arc) -> ReceiptWithOutcome { ReceiptWithOutcome { outcome: ExecutionOutcomeWithId { outcome: Some(ExecutionOutcome { @@ -1045,12 +970,12 @@ mod test { receipt: codec::Receipt { receipt: Some(receipt::Receipt::Action(ReceiptAction { output_data_receivers: vec![DataReceiver { - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }], ..Default::default() })), - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }, block, diff --git a/chain/near/src/codec.rs b/chain/near/src/codec.rs index bfabce649c4..569e8cf5243 100644 --- a/chain/near/src/codec.rs +++ b/chain/near/src/codec.rs @@ -2,23 +2,18 @@ #[path = "protobuf/sf.near.codec.v1.rs"] pub mod pbcodec; -#[rustfmt::skip] -#[path = "protobuf/receipts.v1.rs"] -pub mod substreams_triggers; - use graph::{ - blockchain::Block as BlockchainBlock, - blockchain::{BlockPtr, BlockTime}, - prelude::{hex, web3::types::H256, BlockNumber}, + blockchain::{Block as BlockchainBlock, BlockPtr, BlockTime}, + prelude::{BlockNumber, alloy::primitives::B256, hex}, }; use std::convert::TryFrom; use std::fmt::LowerHex; pub use pbcodec::*; -impl From<&CryptoHash> for H256 { +impl From<&CryptoHash> for B256 { fn from(input: &CryptoHash) -> Self { - H256::from_slice(&input.bytes) + B256::from_slice(&input.bytes) } } @@ -31,7 +26,7 @@ impl LowerHex for &CryptoHash { impl BlockHeader { pub fn parent_ptr(&self) -> Option { match (self.prev_hash.as_ref(), self.prev_height) { - (Some(hash), number) => Some(BlockPtr::from((H256::from(hash), number))), + (Some(hash), number) => Some(BlockPtr::from((B256::from(hash), number))), _ => None, } } @@ -39,7 +34,7 @@ impl BlockHeader { impl<'a> From<&'a BlockHeader> for BlockPtr { fn from(b: &'a BlockHeader) -> BlockPtr { - BlockPtr::from((H256::from(b.hash.as_ref().unwrap()), b.height)) + BlockPtr::from((B256::from(b.hash.as_ref().unwrap()), b.height)) } } @@ -77,8 +72,7 @@ impl BlockchainBlock for Block { } fn timestamp(&self) -> BlockTime { - let ts = i64::try_from(self.header().timestamp).unwrap(); - BlockTime::since_epoch(ts, 0) + block_time_from_header(self.header()) } } @@ -108,8 +102,7 @@ impl BlockchainBlock for HeaderOnlyBlock { } fn timestamp(&self) -> BlockTime { - let ts = i64::try_from(self.header().timestamp).unwrap(); - BlockTime::since_epoch(ts, 0) + block_time_from_header(self.header()) } } @@ -122,3 +115,25 @@ impl execution_outcome::Status { } } } + +fn block_time_from_header(header: &BlockHeader) -> BlockTime { + // The timstamp is in ns since the epoch + let ts = i64::try_from(header.timestamp_nanosec).unwrap(); + let secs = ts / 1_000_000_000; + let ns: u32 = (ts % 1_000_000_000) as u32; + BlockTime::since_epoch(secs, ns) +} + +#[test] +fn timestamp_conversion() { + // 2020-07-21T21:50:10Z in ns + let ts = 1_595_368_210_762_782_796; + let header = BlockHeader { + timestamp_nanosec: ts, + ..Default::default() + }; + assert_eq!( + 1595368210, + block_time_from_header(&header).as_secs_since_epoch() + ); +} diff --git a/chain/near/src/data_source.rs b/chain/near/src/data_source.rs index 86f5a784322..01484905a58 100644 --- a/chain/near/src/data_source.rs +++ b/chain/near/src/data_source.rs @@ -1,15 +1,15 @@ +use async_trait::async_trait; use graph::anyhow::Context; use graph::blockchain::{Block, TriggerWithHandler}; +use graph::components::link_resolver::LinkResolverContext; use graph::components::store::StoredDynamicDataSource; -use graph::data::subgraph::DataSourceContext; +use graph::components::subgraph::InstanceDSTemplateInfo; +use graph::data::subgraph::{DataSourceContext, DeploymentHash}; use graph::prelude::SubgraphManifestValidationError; use graph::{ - anyhow::{anyhow, Error}, + anyhow::{Error, anyhow}, blockchain::{self, Blockchain}, - prelude::{ - async_trait, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, - LinkResolver, Logger, - }, + prelude::{BlockNumber, CheapClone, Deserialize, Link, LinkResolver, Logger}, semver, }; use std::collections::HashSet; @@ -35,7 +35,10 @@ pub struct DataSource { } impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { + fn from_template_info( + _info: InstanceDSTemplateInfo, + _template: &graph::data_source::DataSourceTemplate, + ) -> Result { Err(anyhow!("Near subgraphs do not support templates")) // How this might be implemented if/when Near gets support for templates: @@ -225,7 +228,7 @@ impl blockchain::DataSource for DataSource { todo!() } - fn validate(&self) -> Vec { + fn validate(&self, _: &semver::Version) -> Vec { let mut errors = Vec::new(); if self.kind != NEAR_KIND { @@ -329,9 +332,11 @@ pub struct UnresolvedDataSource { impl blockchain::UnresolvedDataSource for UnresolvedDataSource { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, _manifest_idx: u32, + _spec_version: &semver::Version, ) -> Result { let UnresolvedDataSource { kind, @@ -342,7 +347,7 @@ impl blockchain::UnresolvedDataSource for UnresolvedDataSource { context, } = self; - let mapping = mapping.resolve(resolver, logger).await.with_context(|| { + let mapping = mapping.resolve(deployment_hash, resolver, logger).await.with_context(|| { format!( "failed to resolve data source {} with source_account {:?} and source_start_block {}", name, source.account, source.start_block @@ -368,9 +373,11 @@ pub type DataSourceTemplate = BaseDataSourceTemplate; impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, _manifest_idx: u32, + _spec_version: &semver::Version, ) -> Result { let UnresolvedDataSourceTemplate { kind, @@ -380,7 +387,7 @@ impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTem } = self; let mapping = mapping - .resolve(resolver, logger) + .resolve(deployment_hash, resolver, logger) .await .with_context(|| format!("failed to resolve data source template {}", name))?; @@ -431,6 +438,7 @@ pub struct UnresolvedMapping { impl UnresolvedMapping { pub async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, ) -> Result { @@ -446,7 +454,7 @@ impl UnresolvedMapping { let api_version = semver::Version::parse(&api_version)?; let module_bytes = resolver - .cat(logger, &link) + .cat(&LinkResolverContext::new(deployment_hash, logger), &link) .await .with_context(|| format!("failed to resolve mapping {}", link.link))?; diff --git a/chain/near/src/protobuf/receipts.v1.rs b/chain/near/src/protobuf/receipts.v1.rs deleted file mode 100644 index 5b648d84e90..00000000000 --- a/chain/near/src/protobuf/receipts.v1.rs +++ /dev/null @@ -1,10 +0,0 @@ -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockAndReceipts { - #[prost(message, optional, tag = "1")] - pub block: ::core::option::Option, - #[prost(message, repeated, tag = "2")] - pub outcome: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "3")] - pub receipt: ::prost::alloc::vec::Vec, -} diff --git a/chain/near/src/runtime/abi.rs b/chain/near/src/runtime/abi.rs index 252a4ffa49f..b74d8f5f7ff 100644 --- a/chain/near/src/runtime/abi.rs +++ b/chain/near/src/runtime/abi.rs @@ -1,124 +1,137 @@ use crate::codec; use crate::trigger::ReceiptWithOutcome; +use async_trait::async_trait; use graph::anyhow::anyhow; use graph::runtime::gas::GasCounter; -use graph::runtime::{asc_new, AscHeap, AscPtr, DeterministicHostError, HostExportError, ToAscObj}; +use graph::runtime::{AscHeap, AscPtr, DeterministicHostError, HostExportError, ToAscObj, asc_new}; use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, EnumPayload, Uint8Array}; pub(crate) use super::generated::*; +#[async_trait] impl ToAscObj for codec::Block { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscBlock { - author: asc_new(heap, &self.author, gas)?, - header: asc_new(heap, self.header(), gas)?, - chunks: asc_new(heap, &self.chunk_headers, gas)?, + author: asc_new(heap, &self.author, gas).await?, + header: asc_new(heap, self.header(), gas).await?, + chunks: asc_new(heap, &self.chunk_headers, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::BlockHeader { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let chunk_mask = Array::new(self.chunk_mask.as_ref(), heap, gas)?; + let chunk_mask = Array::new(self.chunk_mask.as_ref(), heap, gas).await?; Ok(AscBlockHeader { height: self.height, prev_height: self.prev_height, - epoch_id: asc_new(heap, self.epoch_id.as_ref().unwrap(), gas)?, - next_epoch_id: asc_new(heap, self.next_epoch_id.as_ref().unwrap(), gas)?, - hash: asc_new(heap, self.hash.as_ref().unwrap(), gas)?, - prev_hash: asc_new(heap, self.prev_hash.as_ref().unwrap(), gas)?, - prev_state_root: asc_new(heap, self.prev_state_root.as_ref().unwrap(), gas)?, - chunk_receipts_root: asc_new(heap, self.chunk_receipts_root.as_ref().unwrap(), gas)?, - chunk_headers_root: asc_new(heap, self.chunk_headers_root.as_ref().unwrap(), gas)?, - chunk_tx_root: asc_new(heap, self.chunk_tx_root.as_ref().unwrap(), gas)?, - outcome_root: asc_new(heap, self.outcome_root.as_ref().unwrap(), gas)?, + epoch_id: asc_new(heap, self.epoch_id.as_ref().unwrap(), gas).await?, + next_epoch_id: asc_new(heap, self.next_epoch_id.as_ref().unwrap(), gas).await?, + hash: asc_new(heap, self.hash.as_ref().unwrap(), gas).await?, + prev_hash: asc_new(heap, self.prev_hash.as_ref().unwrap(), gas).await?, + prev_state_root: asc_new(heap, self.prev_state_root.as_ref().unwrap(), gas).await?, + chunk_receipts_root: asc_new(heap, self.chunk_receipts_root.as_ref().unwrap(), gas) + .await?, + chunk_headers_root: asc_new(heap, self.chunk_headers_root.as_ref().unwrap(), gas) + .await?, + chunk_tx_root: asc_new(heap, self.chunk_tx_root.as_ref().unwrap(), gas).await?, + outcome_root: asc_new(heap, self.outcome_root.as_ref().unwrap(), gas).await?, chunks_included: self.chunks_included, - challenges_root: asc_new(heap, self.challenges_root.as_ref().unwrap(), gas)?, + challenges_root: asc_new(heap, self.challenges_root.as_ref().unwrap(), gas).await?, timestamp_nanosec: self.timestamp_nanosec, - random_value: asc_new(heap, self.random_value.as_ref().unwrap(), gas)?, - validator_proposals: asc_new(heap, &self.validator_proposals, gas)?, - chunk_mask: AscPtr::alloc_obj(chunk_mask, heap, gas)?, - gas_price: asc_new(heap, self.gas_price.as_ref().unwrap(), gas)?, + random_value: asc_new(heap, self.random_value.as_ref().unwrap(), gas).await?, + validator_proposals: asc_new(heap, &self.validator_proposals, gas).await?, + chunk_mask: AscPtr::alloc_obj(chunk_mask, heap, gas).await?, + gas_price: asc_new(heap, self.gas_price.as_ref().unwrap(), gas).await?, block_ordinal: self.block_ordinal, - total_supply: asc_new(heap, self.total_supply.as_ref().unwrap(), gas)?, - challenges_result: asc_new(heap, &self.challenges_result, gas)?, - last_final_block: asc_new(heap, self.last_final_block.as_ref().unwrap(), gas)?, - last_ds_final_block: asc_new(heap, self.last_ds_final_block.as_ref().unwrap(), gas)?, - next_bp_hash: asc_new(heap, self.next_bp_hash.as_ref().unwrap(), gas)?, - block_merkle_root: asc_new(heap, self.block_merkle_root.as_ref().unwrap(), gas)?, - epoch_sync_data_hash: asc_new(heap, self.epoch_sync_data_hash.as_slice(), gas)?, - approvals: asc_new(heap, &self.approvals, gas)?, - signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas)?, + total_supply: asc_new(heap, self.total_supply.as_ref().unwrap(), gas).await?, + challenges_result: asc_new(heap, &self.challenges_result, gas).await?, + last_final_block: asc_new(heap, self.last_final_block.as_ref().unwrap(), gas).await?, + last_ds_final_block: asc_new(heap, self.last_ds_final_block.as_ref().unwrap(), gas) + .await?, + next_bp_hash: asc_new(heap, self.next_bp_hash.as_ref().unwrap(), gas).await?, + block_merkle_root: asc_new(heap, self.block_merkle_root.as_ref().unwrap(), gas).await?, + epoch_sync_data_hash: asc_new(heap, self.epoch_sync_data_hash.as_slice(), gas).await?, + approvals: asc_new(heap, &self.approvals, gas).await?, + signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas).await?, latest_protocol_version: self.latest_protocol_version, }) } } +#[async_trait] impl ToAscObj for codec::ChunkHeader { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscChunkHeader { - chunk_hash: asc_new(heap, self.chunk_hash.as_slice(), gas)?, - signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas)?, - prev_block_hash: asc_new(heap, self.prev_block_hash.as_slice(), gas)?, - prev_state_root: asc_new(heap, self.prev_state_root.as_slice(), gas)?, - encoded_merkle_root: asc_new(heap, self.encoded_merkle_root.as_slice(), gas)?, + chunk_hash: asc_new(heap, self.chunk_hash.as_slice(), gas).await?, + signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas).await?, + prev_block_hash: asc_new(heap, self.prev_block_hash.as_slice(), gas).await?, + prev_state_root: asc_new(heap, self.prev_state_root.as_slice(), gas).await?, + encoded_merkle_root: asc_new(heap, self.encoded_merkle_root.as_slice(), gas).await?, encoded_length: self.encoded_length, height_created: self.height_created, height_included: self.height_included, shard_id: self.shard_id, gas_used: self.gas_used, gas_limit: self.gas_limit, - balance_burnt: asc_new(heap, self.balance_burnt.as_ref().unwrap(), gas)?, - outgoing_receipts_root: asc_new(heap, self.outgoing_receipts_root.as_slice(), gas)?, - tx_root: asc_new(heap, self.tx_root.as_slice(), gas)?, - validator_proposals: asc_new(heap, &self.validator_proposals, gas)?, + balance_burnt: asc_new(heap, self.balance_burnt.as_ref().unwrap(), gas).await?, + outgoing_receipts_root: asc_new(heap, self.outgoing_receipts_root.as_slice(), gas) + .await?, + tx_root: asc_new(heap, self.tx_root.as_slice(), gas).await?, + validator_proposals: asc_new(heap, &self.validator_proposals, gas).await?, _padding: 0, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscChunkHeaderArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscChunkHeaderArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for ReceiptWithOutcome { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscReceiptWithOutcome { - outcome: asc_new(heap, &self.outcome, gas)?, - receipt: asc_new(heap, &self.receipt, gas)?, - block: asc_new(heap, self.block.as_ref(), gas)?, + outcome: asc_new(heap, &self.outcome, gas).await?, + receipt: asc_new(heap, &self.receipt, gas).await?, + block: asc_new(heap, self.block.as_ref(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::Receipt { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -133,21 +146,23 @@ impl ToAscObj for codec::Receipt { }; Ok(AscActionReceipt { - id: asc_new(heap, &self.receipt_id.as_ref().unwrap(), gas)?, - predecessor_id: asc_new(heap, &self.predecessor_id, gas)?, - receiver_id: asc_new(heap, &self.receiver_id, gas)?, - signer_id: asc_new(heap, &action.signer_id, gas)?, - signer_public_key: asc_new(heap, action.signer_public_key.as_ref().unwrap(), gas)?, - gas_price: asc_new(heap, action.gas_price.as_ref().unwrap(), gas)?, - output_data_receivers: asc_new(heap, &action.output_data_receivers, gas)?, - input_data_ids: asc_new(heap, &action.input_data_ids, gas)?, - actions: asc_new(heap, &action.actions, gas)?, + id: asc_new(heap, &self.receipt_id.as_ref().unwrap(), gas).await?, + predecessor_id: asc_new(heap, &self.predecessor_id, gas).await?, + receiver_id: asc_new(heap, &self.receiver_id, gas).await?, + signer_id: asc_new(heap, &action.signer_id, gas).await?, + signer_public_key: asc_new(heap, action.signer_public_key.as_ref().unwrap(), gas) + .await?, + gas_price: asc_new(heap, action.gas_price.as_ref().unwrap(), gas).await?, + output_data_receivers: asc_new(heap, &action.output_data_receivers, gas).await?, + input_data_ids: asc_new(heap, &action.input_data_ids, gas).await?, + actions: asc_new(heap, &action.actions, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::Action { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -155,35 +170,35 @@ impl ToAscObj for codec::Action { let (kind, payload) = match self.action.as_ref().unwrap() { codec::action::Action::CreateAccount(action) => ( AscActionKind::CreateAccount, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::DeployContract(action) => ( AscActionKind::DeployContract, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::FunctionCall(action) => ( AscActionKind::FunctionCall, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::Transfer(action) => ( AscActionKind::Transfer, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::Stake(action) => ( AscActionKind::Stake, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::AddKey(action) => ( AscActionKind::AddKey, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::DeleteKey(action) => ( AscActionKind::DeleteKey, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::DeleteAccount(action) => ( AscActionKind::DeleteAccount, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), }; @@ -195,20 +210,24 @@ impl ToAscObj for codec::Action { } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscActionEnumArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscActionEnumArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::CreateAccountAction { - fn to_asc_obj( + async fn to_asc_obj( &self, _heap: &mut H, _gas: &GasCounter, @@ -217,88 +236,95 @@ impl ToAscObj for codec::CreateAccountAction { } } +#[async_trait] impl ToAscObj for codec::DeployContractAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscDeployContractAction { - code: asc_new(heap, self.code.as_slice(), gas)?, + code: asc_new(heap, self.code.as_slice(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::FunctionCallAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscFunctionCallAction { - method_name: asc_new(heap, &self.method_name, gas)?, - args: asc_new(heap, self.args.as_slice(), gas)?, + method_name: asc_new(heap, &self.method_name, gas).await?, + args: asc_new(heap, self.args.as_slice(), gas).await?, gas: self.gas, - deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas)?, + deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas).await?, _padding: 0, }) } } +#[async_trait] impl ToAscObj for codec::TransferAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscTransferAction { - deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas)?, + deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::StakeAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscStakeAction { - stake: asc_new(heap, self.stake.as_ref().unwrap(), gas)?, - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + stake: asc_new(heap, self.stake.as_ref().unwrap(), gas).await?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::AddKeyAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscAddKeyAction { - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, - access_key: asc_new(heap, self.access_key.as_ref().unwrap(), gas)?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, + access_key: asc_new(heap, self.access_key.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::AccessKey { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscAccessKey { nonce: self.nonce, - permission: asc_new(heap, self.permission.as_ref().unwrap(), gas)?, + permission: asc_new(heap, self.permission.as_ref().unwrap(), gas).await?, _padding: 0, }) } } +#[async_trait] impl ToAscObj for codec::AccessKeyPermission { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -306,11 +332,11 @@ impl ToAscObj for codec::AccessKeyPermission { let (kind, payload) = match self.permission.as_ref().unwrap() { codec::access_key_permission::Permission::FunctionCall(permission) => ( AscAccessKeyPermissionKind::FunctionCall, - asc_new(heap, permission, gas)?.to_payload(), + asc_new(heap, permission, gas).await?.to_payload(), ), codec::access_key_permission::Permission::FullAccess(permission) => ( AscAccessKeyPermissionKind::FullAccess, - asc_new(heap, permission, gas)?.to_payload(), + asc_new(heap, permission, gas).await?.to_payload(), ), }; @@ -322,8 +348,9 @@ impl ToAscObj for codec::AccessKeyPermission { } } +#[async_trait] impl ToAscObj for codec::FunctionCallPermission { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -331,17 +358,18 @@ impl ToAscObj for codec::FunctionCallPermission { Ok(AscFunctionCallPermission { // The `allowance` field is one of the few fields that can actually be None for real allowance: match self.allowance.as_ref() { - Some(allowance) => asc_new(heap, allowance, gas)?, + Some(allowance) => asc_new(heap, allowance, gas).await?, None => AscPtr::null(), }, - receiver_id: asc_new(heap, &self.receiver_id, gas)?, - method_names: asc_new(heap, &self.method_names, gas)?, + receiver_id: asc_new(heap, &self.receiver_id, gas).await?, + method_names: asc_new(heap, &self.method_names, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::FullAccessPermission { - fn to_asc_obj( + async fn to_asc_obj( &self, _heap: &mut H, _gas: &GasCounter, @@ -350,57 +378,64 @@ impl ToAscObj for codec::FullAccessPermission { } } +#[async_trait] impl ToAscObj for codec::DeleteKeyAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscDeleteKeyAction { - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::DeleteAccountAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscDeleteAccountAction { - beneficiary_id: asc_new(heap, &self.beneficiary_id, gas)?, + beneficiary_id: asc_new(heap, &self.beneficiary_id, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::DataReceiver { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscDataReceiver { - data_id: asc_new(heap, self.data_id.as_ref().unwrap(), gas)?, - receiver_id: asc_new(heap, &self.receiver_id, gas)?, + data_id: asc_new(heap, self.data_id.as_ref().unwrap(), gas).await?, + receiver_id: asc_new(heap, &self.receiver_id, gas).await?, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscDataReceiverArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscDataReceiverArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::ExecutionOutcomeWithId { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -408,21 +443,22 @@ impl ToAscObj for codec::ExecutionOutcomeWithId { let outcome = self.outcome.as_ref().unwrap(); Ok(AscExecutionOutcome { - proof: asc_new(heap, &self.proof.as_ref().unwrap().path, gas)?, - block_hash: asc_new(heap, self.block_hash.as_ref().unwrap(), gas)?, - id: asc_new(heap, self.id.as_ref().unwrap(), gas)?, - logs: asc_new(heap, &outcome.logs, gas)?, - receipt_ids: asc_new(heap, &outcome.receipt_ids, gas)?, + proof: asc_new(heap, &self.proof.as_ref().unwrap().path, gas).await?, + block_hash: asc_new(heap, self.block_hash.as_ref().unwrap(), gas).await?, + id: asc_new(heap, self.id.as_ref().unwrap(), gas).await?, + logs: asc_new(heap, &outcome.logs, gas).await?, + receipt_ids: asc_new(heap, &outcome.receipt_ids, gas).await?, gas_burnt: outcome.gas_burnt, - tokens_burnt: asc_new(heap, outcome.tokens_burnt.as_ref().unwrap(), gas)?, - executor_id: asc_new(heap, &outcome.executor_id, gas)?, - status: asc_new(heap, outcome.status.as_ref().unwrap(), gas)?, + tokens_burnt: asc_new(heap, outcome.tokens_burnt.as_ref().unwrap(), gas).await?, + executor_id: asc_new(heap, &outcome.executor_id, gas).await?, + status: asc_new(heap, outcome.status.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::execution_outcome::Status { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -433,12 +469,14 @@ impl ToAscObj for codec::execution_outcome::Status { ( AscSuccessStatusKind::Value, - asc_new(heap, bytes.as_slice(), gas)?.to_payload(), + asc_new(heap, bytes.as_slice(), gas).await?.to_payload(), ) } codec::execution_outcome::Status::SuccessReceiptId(receipt_id) => ( AscSuccessStatusKind::ReceiptId, - asc_new(heap, receipt_id.id.as_ref().unwrap(), gas)?.to_payload(), + asc_new(heap, receipt_id.id.as_ref().unwrap(), gas) + .await? + .to_payload(), ), codec::execution_outcome::Status::Failure(_) => { return Err(DeterministicHostError::from(anyhow!( @@ -462,14 +500,15 @@ impl ToAscObj for codec::execution_outcome::Status { } } +#[async_trait] impl ToAscObj for codec::MerklePathItem { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscMerklePathItem { - hash: asc_new(heap, self.hash.as_ref().unwrap(), gas)?, + hash: asc_new(heap, self.hash.as_ref().unwrap(), gas).await?, direction: match self.direction { 0 => AscDirection::Left, 1 => AscDirection::Right, @@ -478,27 +517,33 @@ impl ToAscObj for codec::MerklePathItem { "Invalid direction value {}", x )) - .into()) + .into()); } }, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscMerklePathItemArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscMerklePathItemArray( + Array::new(&content, heap, gas).await?, + )) } } +#[async_trait] impl ToAscObj for codec::Signature { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -512,28 +557,32 @@ impl ToAscObj for codec::Signature { "Invalid signature type {}", value, )) - .into()) + .into()); } }, - bytes: asc_new(heap, self.bytes.as_slice(), gas)?, + bytes: asc_new(heap, self.bytes.as_slice(), gas).await?, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscSignatureArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscSignatureArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::PublicKey { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -547,89 +596,106 @@ impl ToAscObj for codec::PublicKey { "Invalid public key type {}", value, )) - .into()) + .into()); } }, - bytes: asc_new(heap, self.bytes.as_slice(), gas)?, + bytes: asc_new(heap, self.bytes.as_slice(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::ValidatorStake { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscValidatorStake { - account_id: asc_new(heap, &self.account_id, gas)?, - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, - stake: asc_new(heap, self.stake.as_ref().unwrap(), gas)?, + account_id: asc_new(heap, &self.account_id, gas).await?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, + stake: asc_new(heap, self.stake.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscValidatorStakeArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscValidatorStakeArray( + Array::new(&content, heap, gas).await?, + )) } } +#[async_trait] impl ToAscObj for codec::SlashedValidator { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscSlashedValidator { - account_id: asc_new(heap, &self.account_id, gas)?, + account_id: asc_new(heap, &self.account_id, gas).await?, is_double_sign: self.is_double_sign, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscSlashedValidatorArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscSlashedValidatorArray( + Array::new(&content, heap, gas).await?, + )) } } +#[async_trait] impl ToAscObj for codec::CryptoHash { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - self.bytes.to_asc_obj(heap, gas) + self.bytes.to_asc_obj(heap, gas).await } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscCryptoHashArray(Array::new(&content, heap, gas)?)) + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscCryptoHashArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::BigInt { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, @@ -637,6 +703,6 @@ impl ToAscObj for codec::BigInt { // Bytes are reversed to align with BigInt bytes endianess let reversed: Vec = self.bytes.iter().rev().copied().collect(); - reversed.to_asc_obj(heap, gas) + reversed.to_asc_obj(heap, gas).await } } diff --git a/chain/near/src/runtime/generated.rs b/chain/near/src/runtime/generated.rs index 153eb8b5ab5..d8fe2937f43 100644 --- a/chain/near/src/runtime/generated.rs +++ b/chain/near/src/runtime/generated.rs @@ -227,20 +227,15 @@ impl AscIndexId for AscSignature { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscAccessKeyPermissionKind { + #[default] FunctionCall, FullAccess, } impl AscValue for AscAccessKeyPermissionKind {} -impl Default for AscAccessKeyPermissionKind { - fn default() -> Self { - Self::FunctionCall - } -} - #[repr(C)] #[derive(AscType)] pub(crate) struct AscFunctionCallPermission { @@ -293,8 +288,9 @@ impl AscIndexId for AscDataReceiver { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscActionKind { + #[default] CreateAccount, DeployContract, FunctionCall, @@ -307,12 +303,6 @@ pub(crate) enum AscActionKind { impl AscValue for AscActionKind {} -impl Default for AscActionKind { - fn default() -> Self { - Self::CreateAccount - } -} - #[repr(C)] #[derive(AscType)] pub(crate) struct AscCreateAccountAction {} @@ -424,20 +414,15 @@ impl AscIndexId for AscActionReceipt { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscSuccessStatusKind { + #[default] Value, ReceiptId, } impl AscValue for AscSuccessStatusKind {} -impl Default for AscSuccessStatusKind { - fn default() -> Self { - Self::Value - } -} - pub struct AscSuccessStatusEnum(pub(crate) AscEnum); impl AscType for AscSuccessStatusEnum { @@ -458,20 +443,15 @@ impl AscIndexId for AscSuccessStatusEnum { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscDirection { + #[default] Left, Right, } impl AscValue for AscDirection {} -impl Default for AscDirection { - fn default() -> Self { - Self::Left - } -} - #[repr(C)] #[derive(AscType)] pub(crate) struct AscMerklePathItem { diff --git a/chain/near/src/trigger.rs b/chain/near/src/trigger.rs index 22cd8b1eb96..bf9794d2b20 100644 --- a/chain/near/src/trigger.rs +++ b/chain/near/src/trigger.rs @@ -1,12 +1,13 @@ +use async_trait::async_trait; use graph::blockchain::Block; use graph::blockchain::MappingTriggerTrait; use graph::blockchain::TriggerData; -use graph::cheap_clone::CheapClone; -use graph::prelude::hex; -use graph::prelude::web3::types::H256; +use graph::derive::CheapClone; use graph::prelude::BlockNumber; +use graph::prelude::alloy::primitives::B256; +use graph::prelude::hex; use graph::runtime::HostExportError; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr}; +use graph::runtime::{AscHeap, AscPtr, asc_new, gas::GasCounter}; use graph_runtime_wasm::module::ToAscPtr; use std::{cmp::Ordering, sync::Arc}; @@ -15,6 +16,7 @@ use crate::codec; // Logging the block is too verbose, so this strips the block from the trigger for Debug. impl std::fmt::Debug for NearTrigger { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[allow(unused)] #[derive(Debug)] pub enum MappingTriggerWithoutBlock<'a> { Block, @@ -37,34 +39,26 @@ impl std::fmt::Debug for NearTrigger { } } +#[async_trait] impl ToAscPtr for NearTrigger { - fn to_asc_ptr( + async fn to_asc_ptr( self, heap: &mut H, gas: &GasCounter, ) -> Result, HostExportError> { Ok(match self { - NearTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - NearTrigger::Receipt(receipt) => asc_new(heap, receipt.as_ref(), gas)?.erase(), + NearTrigger::Block(block) => asc_new(heap, block.as_ref(), gas).await?.erase(), + NearTrigger::Receipt(receipt) => asc_new(heap, receipt.as_ref(), gas).await?.erase(), }) } } -#[derive(Clone)] +#[derive(Clone, CheapClone)] pub enum NearTrigger { Block(Arc), Receipt(Arc), } -impl CheapClone for NearTrigger { - fn cheap_clone(&self) -> NearTrigger { - match self { - NearTrigger::Block(block) => NearTrigger::Block(block.cheap_clone()), - NearTrigger::Receipt(receipt) => NearTrigger::Receipt(receipt.cheap_clone()), - } - } -} - impl PartialEq for NearTrigger { fn eq(&self, other: &Self) -> bool { match (self, other) { @@ -86,10 +80,10 @@ impl NearTrigger { } } - pub fn block_hash(&self) -> H256 { + pub fn block_hash(&self) -> B256 { match self { - NearTrigger::Block(block) => block.ptr().hash_as_h256(), - NearTrigger::Receipt(receipt) => receipt.block.ptr().hash_as_h256(), + NearTrigger::Block(block) => block.ptr().hash.as_b256(), + NearTrigger::Receipt(receipt) => receipt.block.ptr().hash.as_b256(), } } @@ -158,30 +152,30 @@ pub struct ReceiptWithOutcome { #[cfg(test)] mod tests { - use std::convert::TryFrom; - use super::*; use graph::{ anyhow::anyhow, components::metrics::gas::GasMetrics, data::subgraph::API_VERSION_0_0_5, - prelude::{hex, BigInt}, - runtime::{gas::GasCounter, DeterministicHostError, HostExportError}, + prelude::{BigInt, hex}, + runtime::{DeterministicHostError, HostExportError, gas::GasCounter}, util::mem::init_slice, }; - #[test] - fn block_trigger_to_asc_ptr() { + #[graph::test] + async fn block_trigger_to_asc_ptr() { let mut heap = BytesHeap::new(API_VERSION_0_0_5); let trigger = NearTrigger::Block(Arc::new(block())); - let result = trigger.to_asc_ptr(&mut heap, &GasCounter::new(GasMetrics::mock())); + let result = trigger + .to_asc_ptr(&mut heap, &GasCounter::new(GasMetrics::mock())) + .await; assert!(result.is_ok()); } - #[test] - fn receipt_trigger_to_asc_ptr() { + #[graph::test] + async fn receipt_trigger_to_asc_ptr() { let mut heap = BytesHeap::new(API_VERSION_0_0_5); let trigger = NearTrigger::Receipt(Arc::new(ReceiptWithOutcome { block: Arc::new(block()), @@ -189,7 +183,9 @@ mod tests { receipt: receipt().unwrap(), })); - let result = trigger.to_asc_ptr(&mut heap, &GasCounter::new(GasMetrics::mock())); + let result = trigger + .to_asc_ptr(&mut heap, &GasCounter::new(GasMetrics::mock())) + .await; assert!(result.is_ok()); } @@ -409,8 +405,7 @@ mod tests { } fn big_int(input: u64) -> Option { - let value = - BigInt::try_from(input).unwrap_or_else(|_| panic!("Invalid BigInt value {}", input)); + let value = BigInt::from(input); let bytes = value.to_signed_bytes_le(); Some(codec::BigInt { bytes }) @@ -452,8 +447,9 @@ mod tests { } } + #[async_trait] impl AscHeap for BytesHeap { - fn raw_new( + async fn raw_new( &mut self, bytes: &[u8], _gas: &GasCounter, @@ -505,11 +501,11 @@ mod tests { Ok(init_slice(src, buffer)) } - fn api_version(&self) -> graph::semver::Version { - self.api_version.clone() + fn api_version(&self) -> &graph::semver::Version { + &self.api_version } - fn asc_type_id( + async fn asc_type_id( &mut self, type_id_index: graph::runtime::IndexForAscTypeId, ) -> Result { diff --git a/chain/starknet/Cargo.toml b/chain/starknet/Cargo.toml deleted file mode 100644 index 3a8ebeac8c9..00000000000 --- a/chain/starknet/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "graph-chain-starknet" -version.workspace = true -edition.workspace = true - -[build-dependencies] -tonic-build = { workspace = true } - -[dependencies] -graph = { path = "../../graph" } -hex = { version = "0.4.3", features = ["serde"] } -prost = { workspace = true } -prost-types = { workspace = true } -serde = "1.0" -sha3 = "0.10.8" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/starknet/build.rs b/chain/starknet/build.rs deleted file mode 100644 index 8a67809dfca..00000000000 --- a/chain/starknet/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .out_dir("src/protobuf") - .compile(&["proto/starknet.proto"], &["proto"]) - .expect("Failed to compile Firehose StarkNet proto(s)"); -} diff --git a/chain/starknet/proto/starknet.proto b/chain/starknet/proto/starknet.proto deleted file mode 100644 index 073b8c2c569..00000000000 --- a/chain/starknet/proto/starknet.proto +++ /dev/null @@ -1,37 +0,0 @@ -syntax = "proto3"; - -package zklend.starknet.type.v1; - -option go_package = "github.com/starknet-graph/firehose-starknet/types/pb/zklend/starknet/type/v1;pbacme"; - -// This file only contains the bare minimum types for the POC. It's far from a complete -// representation of a StarkNet network's history as required by the Firehose protocol. As a result, -// any future changes to this schema would require a full re-sync of the StarkNet node. - -message Block { - uint64 height = 1; - bytes hash = 2; - bytes prevHash = 3; - uint64 timestamp = 4; - repeated Transaction transactions = 5; -} - -message Transaction { - TransactionType type = 1; - bytes hash = 2; - repeated Event events = 3; -} - -enum TransactionType { - DEPLOY = 0; - INVOKE_FUNCTION = 1; - DECLARE = 2; - L1_HANDLER = 3; - DEPLOY_ACCOUNT = 4; -} - -message Event { - bytes fromAddr = 1; - repeated bytes keys = 2; - repeated bytes data = 3; -} diff --git a/chain/starknet/src/adapter.rs b/chain/starknet/src/adapter.rs deleted file mode 100644 index e04df8e979c..00000000000 --- a/chain/starknet/src/adapter.rs +++ /dev/null @@ -1,27 +0,0 @@ -use graph::blockchain::{EmptyNodeCapabilities, TriggerFilter as TriggerFilterTrait}; - -use crate::{ - data_source::{DataSource, DataSourceTemplate}, - Chain, -}; - -#[derive(Default, Clone)] -pub struct TriggerFilter; - -impl TriggerFilterTrait for TriggerFilter { - #[allow(unused)] - fn extend_with_template(&mut self, data_source: impl Iterator) { - todo!() - } - - #[allow(unused)] - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) {} - - fn node_capabilities(&self) -> EmptyNodeCapabilities { - todo!() - } - - fn to_firehose_filter(self) -> Vec { - todo!() - } -} diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs deleted file mode 100644 index 060a502d80d..00000000000 --- a/chain/starknet/src/chain.rs +++ /dev/null @@ -1,499 +0,0 @@ -use graph::{ - anyhow::Result, - blockchain::{ - block_stream::{ - BlockStream, BlockStreamBuilder, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, - FirehoseError, FirehoseMapper as FirehoseMapperTrait, - TriggersAdapter as TriggersAdapterTrait, - }, - client::ChainClient, - firehose_block_ingestor::FirehoseBlockIngestor, - firehose_block_stream::FirehoseBlockStream, - BasicBlockchainBuilder, Block, BlockIngestor, BlockPtr, Blockchain, BlockchainBuilder, - BlockchainKind, EmptyNodeCapabilities, IngestorError, NoopRuntimeAdapter, - RuntimeAdapter as RuntimeAdapterTrait, - }, - cheap_clone::CheapClone, - components::store::{DeploymentCursorTracker, DeploymentLocator}, - data::subgraph::UnifiedMappingApiVersion, - env::EnvVars, - firehose::{self, FirehoseEndpoint, ForkStep}, - prelude::{ - async_trait, BlockHash, BlockNumber, ChainStore, Error, Logger, LoggerFactory, - MetricsRegistry, TryFutureExt, - }, - schema::InputSchema, - slog::o, -}; -use prost::Message; -use std::sync::Arc; - -use crate::{ - adapter::TriggerFilter, - codec, - data_source::{ - DataSource, DataSourceTemplate, UnresolvedDataSource, UnresolvedDataSourceTemplate, - }, - trigger::{StarknetBlockTrigger, StarknetEventTrigger, StarknetTrigger}, -}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: String, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, - block_stream_builder: Arc>, -} - -pub struct StarknetStreamBuilder; - -pub struct FirehoseMapper { - adapter: Arc>, - filter: Arc, -} - -pub struct TriggersAdapter; - -impl BlockchainBuilder for BasicBlockchainBuilder { - fn build(self, _config: &Arc) -> Chain { - Chain { - logger_factory: self.logger_factory, - name: self.name, - chain_store: self.chain_store, - client: Arc::new(ChainClient::new_firehose(self.firehose_endpoints)), - metrics_registry: self.metrics_registry, - block_stream_builder: Arc::new(StarknetStreamBuilder {}), - } - } -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: starknet") - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Starknet; - - type Client = (); - type Block = codec::Block; - type DataSource = DataSource; - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = crate::trigger::StarknetTrigger; - - type MappingTrigger = crate::trigger::StarknetTrigger; - - type TriggerFilter = crate::adapter::TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn triggers_adapter( - &self, - _log: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - Ok(Arc::new(TriggersAdapter)) - } - - async fn new_block_stream( - &self, - deployment: DeploymentLocator, - store: impl DeploymentCursorTracker, - start_blocks: Vec, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - self.block_stream_builder - .build_firehose( - self, - deployment, - store.firehose_cursor(), - start_blocks, - store.block_ptr(), - filter, - unified_api_version, - ) - .await - } - - fn is_refetch_block_required(&self) -> bool { - false - } - - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - fn chain_store(&self) -> Arc { - self.chain_store.clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint()?; - - firehose_endpoint - .block_ptr_for_number::(logger, number) - .map_err(Into::into) - .await - } - - fn runtime_adapter(&self) -> Arc> { - Arc::new(NoopRuntimeAdapter::default()) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } - - fn block_ingestor(&self) -> Result> { - let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), - self.chain_client(), - self.logger_factory - .component_logger("StarknetFirehoseBlockIngestor", None), - self.name.clone(), - ); - Ok(Box::new(ingestor)) - } -} - -#[async_trait] -impl BlockStreamBuilder for StarknetStreamBuilder { - async fn build_substreams( - &self, - _chain: &Chain, - _schema: InputSchema, - _deployment: DeploymentLocator, - _block_cursor: FirehoseCursor, - _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, - ) -> Result>> { - unimplemented!() - } - - async fn build_firehose( - &self, - chain: &Chain, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - let adapter = chain - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", chain.name)); - - let logger = chain - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - chain.chain_client(), - subgraph_current_block, - block_cursor, - firehose_mapper, - start_blocks, - logger, - chain.metrics_registry.clone(), - ))) - } - - async fn build_polling( - &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - panic!("StarkNet does not support polling block stream") - } -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - fn trigger_filter(&self) -> &TriggerFilter { - self.filter.as_ref() - } - - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down stuct that would decode only a few fields and ignore all the rest. - let block = codec::Block::decode(any_block.value.as_ref())?; - - use ForkStep::*; - match step { - StepNew => Ok(BlockStreamEvent::ProcessBlock( - self.adapter - .triggers_in_block(logger, block, &self.filter) - .await?, - FirehoseCursor::from(response.cursor.clone()), - )), - - StepUndo => { - let parent_ptr = block - .parent_ptr() - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - StepFinal => { - panic!("irreversible step is not handled and should not be requested in the Firehose request") - } - - StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - /// Returns the [BlockPtr] value for this given block number. This is the block pointer - /// of the longuest according to Firehose view of the blockchain state. - /// - /// This is a thin wrapper around [FirehoseEndpoint#block_ptr_for_number] to make - /// it chain agnostic and callable from chain agnostic [FirehoseBlockStream]. - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - /// Returns the closest final block ptr to the block ptr received. - /// On probablitics chain like Ethereum, final is determined by - /// the confirmations threshold configured for the Firehose stack (currently - /// hard-coded to 200). - /// - /// On some other chain like NEAR, the actual final block number is determined - /// from the block itself since it contains information about which block number - /// is final against the current block. - /// - /// To take an example, assuming we are on Ethereum, the final block pointer - /// for block #10212 would be the determined final block #10012 (10212 - 200 = 10012). - async fn final_block_ptr_for( - &self, - logger: &Logger, - endpoint: &Arc, - block: &codec::Block, - ) -> Result { - // Firehose for Starknet has an hard-coded confirmations for finality sets to 100 block - // behind the current block. The magic value 100 here comes from this hard-coded Firehose - // value. - let final_block_number = match block.number() { - x if x >= 100 => x - 100, - _ => 0, - }; - - self.block_ptr_for_number(logger, endpoint, final_block_number) - .await - } -} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - // Return the block that is `offset` blocks before the block pointed to - // by `ptr` from the local cache. An offset of 0 means the block itself, - // an offset of 1 means the block's parent etc. If the block is not in - // the local cache, return `None` - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - panic!("Should never be called since FirehoseBlockStream cannot resolve it") - } - - // Returns a sequence of blocks in increasing order of block number. - // Each block will include all of its triggers that match the given `filter`. - // The sequence may omit blocks that contain no triggers, - // but all returned blocks must part of a same chain starting at `chain_base`. - // At least one block will be returned, even if it contains no triggers. - // `step_size` is the suggested number blocks to be scanned. - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &crate::adapter::TriggerFilter, - ) -> Result>, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - #[allow(unused)] - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &crate::adapter::TriggerFilter, - ) -> Result, Error> { - let shared_block = Arc::new(block.clone()); - - let mut triggers: Vec<_> = shared_block - .transactions - .iter() - .flat_map(|transaction| -> Vec { - let transaction = Arc::new(transaction.clone()); - transaction - .events - .iter() - .map(|event| { - StarknetTrigger::Event(StarknetEventTrigger { - event: Arc::new(event.clone()), - block: shared_block.clone(), - transaction: transaction.clone(), - }) - }) - .collect() - }) - .collect(); - - triggers.push(StarknetTrigger::Block(StarknetBlockTrigger { - block: shared_block, - })); - - Ok(BlockWithTriggers::new(block, triggers, logger)) - } - - /// Return `true` if the block with the given hash and number is on the - /// main chain, i.e., the chain going back from the current chain head. - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - /// Get pointer to parent of `block`. This is called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - // Panics if `block` is genesis. - // But that's ok since this is only called when reverting `block`. - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use graph::blockchain::DataSource as _; - - use crate::{ - data_source::{ - DataSource, Mapping, MappingBlockHandler, MappingEventHandler, STARKNET_KIND, - }, - felt::Felt, - }; - - #[test] - fn validate_no_handler() { - let ds = new_data_source(None); - - let errs = ds.validate(); - assert_eq!(errs.len(), 1, "{:?}", ds); - assert_eq!( - errs[0].to_string(), - "data source does not define any handler" - ); - } - - #[test] - fn validate_address_without_event_handler() { - let mut ds = new_data_source(Some([1u8; 32].into())); - ds.mapping.block_handler = Some(MappingBlockHandler { - handler: "asdf".into(), - }); - - let errs = ds.validate(); - assert_eq!(errs.len(), 1, "{:?}", ds); - assert_eq!( - errs[0].to_string(), - "data source cannot have source address without event handlers" - ); - } - - #[test] - fn validate_no_address_with_event_handler() { - let mut ds = new_data_source(None); - ds.mapping.event_handlers.push(MappingEventHandler { - handler: "asdf".into(), - event_selector: [2u8; 32].into(), - }); - - let errs = ds.validate(); - assert_eq!(errs.len(), 1, "{:?}", ds); - assert_eq!(errs[0].to_string(), "subgraph source address is required"); - } - - fn new_data_source(address: Option) -> DataSource { - DataSource { - kind: STARKNET_KIND.to_string(), - network: "starknet-mainnet".into(), - name: "asd".to_string(), - source: crate::data_source::Source { - start_block: 10, - end_block: None, - address, - }, - mapping: Mapping { - block_handler: None, - event_handlers: vec![], - runtime: Arc::new(vec![]), - }, - } - } -} diff --git a/chain/starknet/src/codec.rs b/chain/starknet/src/codec.rs deleted file mode 100644 index 4d029c8c01d..00000000000 --- a/chain/starknet/src/codec.rs +++ /dev/null @@ -1,35 +0,0 @@ -#[rustfmt::skip] -#[path = "protobuf/zklend.starknet.r#type.v1.rs"] -mod pbcodec; - -use graph::blockchain::{Block as BlockchainBlock, BlockHash, BlockPtr}; - -pub use pbcodec::*; - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - self.height as i32 - } - - fn ptr(&self) -> BlockPtr { - BlockPtr { - hash: BlockHash(self.hash.clone().into_boxed_slice()), - number: self.height as i32, - } - } - - fn parent_ptr(&self) -> Option { - if self.height == 0 { - None - } else { - Some(BlockPtr { - hash: BlockHash(self.prev_hash.clone().into_boxed_slice()), - number: (self.height - 1) as i32, - }) - } - } - - fn timestamp(&self) -> graph::blockchain::BlockTime { - graph::blockchain::BlockTime::since_epoch(self.timestamp as i64, 0) - } -} diff --git a/chain/starknet/src/data_source.rs b/chain/starknet/src/data_source.rs deleted file mode 100644 index 34daac0f3cc..00000000000 --- a/chain/starknet/src/data_source.rs +++ /dev/null @@ -1,401 +0,0 @@ -use graph::{ - anyhow::{anyhow, Error}, - blockchain::{self, Block as BlockchainBlock, TriggerWithHandler}, - components::{link_resolver::LinkResolver, store::StoredDynamicDataSource}, - data::subgraph::{DataSourceContext, SubgraphManifestValidationError}, - prelude::{async_trait, BlockNumber, DataSourceTemplateInfo, Deserialize, Link, Logger}, - semver, -}; -use sha3::{Digest, Keccak256}; -use std::{collections::HashSet, sync::Arc}; - -use crate::{ - chain::Chain, - codec, - felt::Felt, - trigger::{StarknetEventTrigger, StarknetTrigger}, -}; - -pub const STARKNET_KIND: &str = "starknet"; -const BLOCK_HANDLER_KIND: &str = "block"; -const EVENT_HANDLER_KIND: &str = "event"; - -#[derive(Debug, Clone)] -pub struct DataSource { - pub kind: String, - pub network: String, - pub name: String, - pub source: Source, - pub mapping: Mapping, -} - -#[derive(Debug, Clone)] -pub struct Mapping { - pub block_handler: Option, - pub event_handlers: Vec, - pub runtime: Arc>, -} - -#[derive(Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: String, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, -} - -#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Source { - pub start_block: BlockNumber, - pub end_block: Option, - #[serde(default)] - pub address: Option, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - #[serde(default)] - pub block_handler: Option, - #[serde(default)] - pub event_handlers: Vec, - pub file: Link, -} - -#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -pub struct MappingEventHandler { - pub handler: String, - pub event_selector: Felt, -} - -#[derive(Clone, Deserialize)] -pub struct UnresolvedMappingEventHandler { - pub handler: String, - pub event: String, -} - -#[derive(Debug, Clone)] -pub struct DataSourceTemplate; - -#[derive(Clone, Default, Deserialize)] -pub struct UnresolvedDataSourceTemplate; - -impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!("StarkNet subgraphs do not support templates")) - } - - fn address(&self) -> Option<&[u8]> { - self.source.address.as_ref().map(|addr| addr.as_ref()) - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn end_block(&self) -> Option { - self.source.end_block - } - - fn handler_kinds(&self) -> HashSet<&str> { - let mut kinds = HashSet::new(); - - let Mapping { - block_handler, - event_handlers, - .. - } = &self.mapping; - - if block_handler.is_some() { - kinds.insert(BLOCK_HANDLER_KIND); - } - if !event_handlers.is_empty() { - kinds.insert(EVENT_HANDLER_KIND); - } - - kinds - } - - fn match_and_decode( - &self, - trigger: &StarknetTrigger, - block: &Arc, - _logger: &Logger, - ) -> Result>, Error> { - if self.start_block() > block.number() { - return Ok(None); - } - - let handler = match trigger { - StarknetTrigger::Block(_) => match &self.mapping.block_handler { - Some(handler) => handler.handler.clone(), - None => return Ok(None), - }, - StarknetTrigger::Event(event) => match self.handler_for_event(event) { - Some(handler) => handler.handler, - None => return Ok(None), - }, - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.clone(), - handler, - block.ptr(), - block.timestamp(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - Some(&self.network) - } - - fn context(&self) -> Arc> { - Arc::new(None) - } - - fn creation_block(&self) -> Option { - None - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - } = self; - - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.event_handlers == other.mapping.event_handlers - && mapping.block_handler == other.mapping.block_handler - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - // FIXME (Starknet): Implement me! - todo!() - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - // FIXME (Starknet): Implement me correctly - todo!() - } - - fn validate(&self) -> Vec { - let mut errors = Vec::new(); - - if self.kind != STARKNET_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - STARKNET_KIND, - self.kind - )) - } - - // Validate that there's at least one handler of any kind - if self.mapping.block_handler.is_none() && self.mapping.event_handlers.is_empty() { - errors.push(anyhow!("data source does not define any handler")); - } - - // Validate that `source` address must not be present if there's no event handler - if self.mapping.event_handlers.is_empty() && self.address().is_some() { - errors.push(anyhow!( - "data source cannot have source address without event handlers" - )); - } - - // Validate that `source` address must be present when there's at least 1 event handler - if !self.mapping.event_handlers.is_empty() && self.address().is_none() { - errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); - } - - errors - } - - fn api_version(&self) -> semver::Version { - semver::Version::new(0, 0, 5) - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.clone()) - } -} - -impl DataSource { - /// Returns event trigger if an event.key matches the handler.key and optionally - /// if event.fromAddr matches the source address. Note this only supports the default - /// Starknet behavior of one key per event. - fn handler_for_event(&self, event: &StarknetEventTrigger) -> Option { - let event_key: Felt = Self::pad_to_32_bytes(event.event.keys.first()?)?.into(); - - // Always padding first here seems fine as we expect most sources to define an address - // filter anyways. Alternatively we can use lazy init here, which seems unnecessary. - let event_from_addr: Felt = Self::pad_to_32_bytes(&event.event.from_addr)?.into(); - - return self - .mapping - .event_handlers - .iter() - .find(|handler| { - // No need to compare address if selector doesn't match - if handler.event_selector != event_key { - return false; - } - - match &self.source.address { - Some(addr_filter) => addr_filter == &event_from_addr, - None => true, - } - }) - .cloned(); - } - - /// We need to pad incoming event selectors and addresses to 32 bytes as our data source uses - /// padded 32 bytes. - fn pad_to_32_bytes(slice: &[u8]) -> Option<[u8; 32]> { - if slice.len() > 32 { - None - } else { - let mut buffer = [0u8; 32]; - buffer[(32 - slice.len())..].copy_from_slice(slice); - Some(buffer) - } - } -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let module_bytes = resolver.cat(logger, &self.mapping.file).await?; - - Ok(DataSource { - kind: self.kind, - network: self.network, - name: self.name, - source: self.source, - mapping: Mapping { - block_handler: self.mapping.block_handler, - event_handlers: self - .mapping - .event_handlers - .into_iter() - .map(|handler| { - Ok(MappingEventHandler { - handler: handler.handler, - event_selector: get_selector_from_name(&handler.event)?, - }) - }) - .collect::, Error>>()?, - runtime: Arc::new(module_bytes), - }, - }) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn api_version(&self) -> semver::Version { - todo!() - } - - fn runtime(&self) -> Option>> { - todo!() - } - - fn name(&self) -> &str { - todo!() - } - - fn manifest_idx(&self) -> u32 { - todo!() - } - - fn kind(&self) -> &str { - todo!() - } -} - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - #[allow(unused)] - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - manifest_idx: u32, - ) -> Result { - todo!() - } -} - -// Adapted from: -// https://github.com/xJonathanLEI/starknet-rs/blob/f16271877c9dbf08bc7bf61e4fc72decc13ff73d/starknet-core/src/utils.rs#L110-L121 -fn get_selector_from_name(func_name: &str) -> graph::anyhow::Result { - const DEFAULT_ENTRY_POINT_NAME: &str = "__default__"; - const DEFAULT_L1_ENTRY_POINT_NAME: &str = "__l1_default__"; - - if func_name == DEFAULT_ENTRY_POINT_NAME || func_name == DEFAULT_L1_ENTRY_POINT_NAME { - Ok([0u8; 32].into()) - } else { - let name_bytes = func_name.as_bytes(); - if name_bytes.is_ascii() { - Ok(starknet_keccak(name_bytes).into()) - } else { - Err(anyhow!("the provided name contains non-ASCII characters")) - } - } -} - -// Adapted from: -// https://github.com/xJonathanLEI/starknet-rs/blob/f16271877c9dbf08bc7bf61e4fc72decc13ff73d/starknet-core/src/utils.rs#L98-L108 -fn starknet_keccak(data: &[u8]) -> [u8; 32] { - let mut hasher = Keccak256::new(); - hasher.update(data); - let mut hash = hasher.finalize(); - - // Remove the first 6 bits - hash[0] &= 0b00000011; - - hash.into() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_starknet_keccak() { - let expected_hash: [u8; 32] = - hex::decode("016c11b0b5b808960df26f5bfc471d04c1995b0ffd2055925ad1be28d6baadfd") - .unwrap() - .try_into() - .unwrap(); - - assert_eq!(starknet_keccak("Hello world".as_bytes()), expected_hash); - } -} diff --git a/chain/starknet/src/felt.rs b/chain/starknet/src/felt.rs deleted file mode 100644 index 7c0e6b6496d..00000000000 --- a/chain/starknet/src/felt.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::{ - fmt::{Debug, Formatter}, - str::FromStr, -}; - -use graph::anyhow; -use serde::{de::Visitor, Deserialize}; - -/// Represents the primitive `FieldElement` type used in Starknet. Each `FieldElement` is 252-bit -/// in size. -#[derive(Clone, PartialEq, Eq)] -pub struct Felt([u8; 32]); - -struct FeltVisitor; - -impl Debug for Felt { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "0x{}", hex::encode(self.0)) - } -} - -impl From<[u8; 32]> for Felt { - fn from(value: [u8; 32]) -> Self { - Self(value) - } -} - -impl AsRef<[u8]> for Felt { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl FromStr for Felt { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - let hex_str = s.trim_start_matches("0x"); - if hex_str.len() % 2 == 0 { - Ok(Felt(decode_even_hex_str(hex_str)?)) - } else { - // We need to manually pad it as the `hex` crate does not allow odd hex length - let mut padded_string = String::from("0"); - padded_string.push_str(hex_str); - - Ok(Felt(decode_even_hex_str(&padded_string)?)) - } - } -} - -impl<'de> Deserialize<'de> for Felt { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_any(FeltVisitor) - } -} - -impl<'de> Visitor<'de> for FeltVisitor { - type Value = Felt; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(formatter, "string") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Felt::from_str(v).map_err(|_| { - serde::de::Error::invalid_value(serde::de::Unexpected::Str(v), &"valid Felt value") - }) - } -} - -/// Attempts to decode a even-length hex string into a padded 32-byte array. -pub fn decode_even_hex_str(hex_str: &str) -> anyhow::Result<[u8; 32]> { - let byte_len = hex_str.len() / 2; - if byte_len > 32 { - anyhow::bail!("length exceeds 32 bytes"); - } - - let mut buffer = [0u8; 32]; - hex::decode_to_slice(hex_str, &mut buffer[(32 - byte_len)..])?; - - Ok(buffer) -} diff --git a/chain/starknet/src/lib.rs b/chain/starknet/src/lib.rs deleted file mode 100644 index a2d71dbb626..00000000000 --- a/chain/starknet/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -mod adapter; -mod chain; -pub mod codec; -mod data_source; -mod felt; -mod runtime; -mod trigger; - -pub use crate::chain::{Chain, StarknetStreamBuilder}; -pub use codec::Block; diff --git a/chain/starknet/src/protobuf/zklend.starknet.r#type.v1.rs b/chain/starknet/src/protobuf/zklend.starknet.r#type.v1.rs deleted file mode 100644 index e580ad30143..00000000000 --- a/chain/starknet/src/protobuf/zklend.starknet.r#type.v1.rs +++ /dev/null @@ -1,69 +0,0 @@ -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(uint64, tag = "1")] - pub height: u64, - #[prost(bytes = "vec", tag = "2")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "3")] - pub prev_hash: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "4")] - pub timestamp: u64, - #[prost(message, repeated, tag = "5")] - pub transactions: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transaction { - #[prost(enumeration = "TransactionType", tag = "1")] - pub r#type: i32, - #[prost(bytes = "vec", tag = "2")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "3")] - pub events: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Event { - #[prost(bytes = "vec", tag = "1")] - pub from_addr: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", repeated, tag = "2")] - pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, - #[prost(bytes = "vec", repeated, tag = "3")] - pub data: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum TransactionType { - Deploy = 0, - InvokeFunction = 1, - Declare = 2, - L1Handler = 3, - DeployAccount = 4, -} -impl TransactionType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - TransactionType::Deploy => "DEPLOY", - TransactionType::InvokeFunction => "INVOKE_FUNCTION", - TransactionType::Declare => "DECLARE", - TransactionType::L1Handler => "L1_HANDLER", - TransactionType::DeployAccount => "DEPLOY_ACCOUNT", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "DEPLOY" => Some(Self::Deploy), - "INVOKE_FUNCTION" => Some(Self::InvokeFunction), - "DECLARE" => Some(Self::Declare), - "L1_HANDLER" => Some(Self::L1Handler), - "DEPLOY_ACCOUNT" => Some(Self::DeployAccount), - _ => None, - } - } -} diff --git a/chain/starknet/src/runtime/abi.rs b/chain/starknet/src/runtime/abi.rs deleted file mode 100644 index b306d9eb5f8..00000000000 --- a/chain/starknet/src/runtime/abi.rs +++ /dev/null @@ -1,106 +0,0 @@ -use graph::{ - prelude::BigInt, - runtime::{asc_new, gas::GasCounter, AscHeap, HostExportError, ToAscObj}, -}; -use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, EnumPayload}; - -use crate::{ - codec, - trigger::{StarknetBlockTrigger, StarknetEventTrigger}, -}; - -pub(crate) use super::generated::*; - -impl ToAscObj for codec::Block { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscBlock { - number: asc_new(heap, &BigInt::from(self.height), gas)?, - hash: asc_new(heap, self.hash.as_slice(), gas)?, - prev_hash: asc_new(heap, self.prev_hash.as_slice(), gas)?, - timestamp: asc_new(heap, &BigInt::from(self.timestamp), gas)?, - }) - } -} - -impl ToAscObj for codec::Transaction { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTransaction { - r#type: asc_new( - heap, - &codec::TransactionType::from_i32(self.r#type) - .expect("invalid TransactionType value"), - gas, - )?, - hash: asc_new(heap, self.hash.as_slice(), gas)?, - }) - } -} - -impl ToAscObj for codec::TransactionType { - fn to_asc_obj( - &self, - _heap: &mut H, - _gas: &GasCounter, - ) -> Result { - Ok(AscTransactionTypeEnum(AscEnum { - kind: match self { - codec::TransactionType::Deploy => AscTransactionType::Deploy, - codec::TransactionType::InvokeFunction => AscTransactionType::InvokeFunction, - codec::TransactionType::Declare => AscTransactionType::Declare, - codec::TransactionType::L1Handler => AscTransactionType::L1Handler, - codec::TransactionType::DeployAccount => AscTransactionType::DeployAccount, - }, - _padding: 0, - payload: EnumPayload(0), - })) - } -} - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self - .iter() - .map(|x| asc_new(heap, x.as_slice(), gas)) - .collect(); - - Ok(AscBytesArray(Array::new(&content?, heap, gas)?)) - } -} - -impl ToAscObj for StarknetBlockTrigger { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - self.block.to_asc_obj(heap, gas) - } -} - -impl ToAscObj for StarknetEventTrigger { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscEvent { - from_addr: asc_new(heap, self.event.from_addr.as_slice(), gas)?, - keys: asc_new(heap, &self.event.keys, gas)?, - data: asc_new(heap, &self.event.data, gas)?, - block: asc_new(heap, self.block.as_ref(), gas)?, - transaction: asc_new(heap, self.transaction.as_ref(), gas)?, - }) - } -} diff --git a/chain/starknet/src/runtime/generated.rs b/chain/starknet/src/runtime/generated.rs deleted file mode 100644 index 59932ae576e..00000000000 --- a/chain/starknet/src/runtime/generated.rs +++ /dev/null @@ -1,100 +0,0 @@ -use graph::runtime::{ - AscIndexId, AscPtr, AscType, AscValue, DeterministicHostError, IndexForAscTypeId, -}; -use graph::semver::Version; -use graph_runtime_derive::AscType; -use graph_runtime_wasm::asc_abi::class::{Array, AscBigInt, AscEnum, Uint8Array}; - -pub struct AscBytesArray(pub(crate) Array>); - -impl AscType for AscBytesArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -impl AscIndexId for AscBytesArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::StarknetArrayBytes; -} - -pub struct AscTransactionTypeEnum(pub(crate) AscEnum); - -impl AscType for AscTransactionTypeEnum { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(AscEnum::from_asc_bytes(asc_obj, api_version)?)) - } -} - -impl AscIndexId for AscTransactionTypeEnum { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::StarknetTransactionTypeEnum; -} - -#[repr(C)] -#[derive(AscType)] -pub(crate) struct AscBlock { - pub number: AscPtr, - pub hash: AscPtr, - pub prev_hash: AscPtr, - pub timestamp: AscPtr, -} - -impl AscIndexId for AscBlock { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::StarknetBlock; -} - -#[repr(C)] -#[derive(AscType)] -pub(crate) struct AscTransaction { - pub r#type: AscPtr, - pub hash: AscPtr, -} - -impl AscIndexId for AscTransaction { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::StarknetTransaction; -} - -#[repr(u32)] -#[derive(AscType, Copy, Clone)] -pub(crate) enum AscTransactionType { - Deploy, - InvokeFunction, - Declare, - L1Handler, - DeployAccount, -} - -impl AscValue for AscTransactionType {} - -impl Default for AscTransactionType { - fn default() -> Self { - Self::Deploy - } -} - -#[repr(C)] -#[derive(AscType)] -pub(crate) struct AscEvent { - pub from_addr: AscPtr, - pub keys: AscPtr, - pub data: AscPtr, - pub block: AscPtr, - pub transaction: AscPtr, -} - -impl AscIndexId for AscEvent { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::StarknetEvent; -} diff --git a/chain/starknet/src/runtime/mod.rs b/chain/starknet/src/runtime/mod.rs deleted file mode 100644 index 31e18de7dd8..00000000000 --- a/chain/starknet/src/runtime/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod abi; - -mod generated; diff --git a/chain/starknet/src/trigger.rs b/chain/starknet/src/trigger.rs deleted file mode 100644 index 690c4c5c719..00000000000 --- a/chain/starknet/src/trigger.rs +++ /dev/null @@ -1,105 +0,0 @@ -use graph::{ - blockchain::{MappingTriggerTrait, TriggerData}, - runtime::{asc_new, gas::GasCounter, AscPtr, HostExportError}, -}; -use graph_runtime_wasm::module::ToAscPtr; -use std::{cmp::Ordering, sync::Arc}; - -use crate::codec; - -#[derive(Debug, Clone)] -pub enum StarknetTrigger { - Block(StarknetBlockTrigger), - Event(StarknetEventTrigger), -} - -#[derive(Debug, Clone)] -pub struct StarknetBlockTrigger { - pub(crate) block: Arc, -} - -#[derive(Debug, Clone)] -pub struct StarknetEventTrigger { - pub(crate) event: Arc, - pub(crate) block: Arc, - pub(crate) transaction: Arc, -} - -impl PartialEq for StarknetTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(l), Self::Block(r)) => l.block == r.block, - (Self::Event(l), Self::Event(r)) => { - // Without event index we can't really tell if they're the same - // TODO: implement add event index to trigger data - l.block.hash == r.block.hash - && l.transaction.hash == r.transaction.hash - && l.event == r.event - } - _ => false, - } - } -} - -impl Eq for StarknetTrigger {} - -impl PartialOrd for StarknetTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for StarknetTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - (Self::Block(l), Self::Block(r)) => l.block.height.cmp(&r.block.height), - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Greater, - (_, Self::Block(..)) => Ordering::Less, - - // Keep the order when comparing two event triggers - // TODO: compare block hash, tx index, and event index - (Self::Event(..), Self::Event(..)) => Ordering::Equal, - } - } -} - -impl TriggerData for StarknetTrigger { - fn error_context(&self) -> String { - match self { - Self::Block(block) => format!("block #{}", block.block.height), - Self::Event(event) => { - format!("event from 0x{}", hex::encode(&event.event.from_addr),) - } - } - } - - fn address_match(&self) -> Option<&[u8]> { - None - } -} - -impl ToAscPtr for StarknetTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - Ok(match self { - StarknetTrigger::Block(block) => asc_new(heap, &block, gas)?.erase(), - StarknetTrigger::Event(event) => asc_new(heap, &event, gas)?.erase(), - }) - } -} - -impl MappingTriggerTrait for StarknetTrigger { - fn error_context(&self) -> String { - match self { - Self::Block(block) => format!("block #{}", block.block.height), - Self::Event(event) => { - format!("event from 0x{}", hex::encode(&event.event.from_addr)) - } - } - } -} diff --git a/chain/substreams/Cargo.toml b/chain/substreams/Cargo.toml deleted file mode 100644 index a636bab047b..00000000000 --- a/chain/substreams/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "graph-chain-substreams" -version.workspace = true -edition.workspace = true - -[build-dependencies] -tonic-build = { workspace = true } - -[dependencies] -async-stream = "0.3" -envconfig = "0.10.0" -futures = "0.1.21" -http = "0.2.4" -jsonrpc-core = "18.0.0" -graph = { path = "../../graph" } -graph-runtime-wasm = { path = "../../runtime/wasm" } -lazy_static = "1.2.0" -serde = "1.0" -prost = { workspace = true } -prost-types = { workspace = true } -dirs-next = "2.0" -anyhow = "1.0" -tiny-keccak = "1.5.0" -hex = "0.4.3" -semver = "1.0.21" -base64 = "0.20.0" - -itertools = "0.12.0" - -[dev-dependencies] -tokio = { version = "1", features = ["full"] } diff --git a/chain/substreams/build.rs b/chain/substreams/build.rs deleted file mode 100644 index 8cccc11fe3a..00000000000 --- a/chain/substreams/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .protoc_arg("--experimental_allow_proto3_optional") - .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) - .expect("Failed to compile Substreams entity proto(s)"); -} diff --git a/chain/substreams/examples/README.md b/chain/substreams/examples/README.md deleted file mode 100644 index afd1882b337..00000000000 --- a/chain/substreams/examples/README.md +++ /dev/null @@ -1,13 +0,0 @@ -## Substreams example - -1. Set environmental variables -```bash -$> export SUBSTREAMS_API_TOKEN=your_sf_token -$> export SUBSTREAMS_ENDPOINT=your_sf_endpoint # you can also not define this one and use the default specified endpoint -$> export SUBSTREAMS_PACKAGE=path_to_your_spkg -``` - -2. Run `substreams` example -```bash -cargo run -p graph-chain-substreams --example substreams [module_name] # for graph entities run `graph_out` -``` diff --git a/chain/substreams/examples/substreams.rs b/chain/substreams/examples/substreams.rs deleted file mode 100644 index fbf245856ac..00000000000 --- a/chain/substreams/examples/substreams.rs +++ /dev/null @@ -1,113 +0,0 @@ -use anyhow::{format_err, Context, Error}; -use graph::blockchain::block_stream::BlockStreamEvent; -use graph::blockchain::client::ChainClient; -use graph::blockchain::substreams_block_stream::SubstreamsBlockStream; -use graph::endpoint::EndpointMetrics; -use graph::firehose::{FirehoseEndpoints, SubgraphLimit}; -use graph::prelude::{info, tokio, DeploymentHash, MetricsRegistry, Registry}; -use graph::tokio_stream::StreamExt; -use graph::{env::env_var, firehose::FirehoseEndpoint, log::logger, substreams}; -use graph_chain_substreams::mapper::Mapper; -use prost::Message; -use std::env; -use std::sync::Arc; - -#[tokio::main] -async fn main() -> Result<(), Error> { - let module_name = env::args().nth(1).unwrap(); - - let token_env = env_var("SUBSTREAMS_API_TOKEN", "".to_string()); - let mut token: Option = None; - if !token_env.is_empty() { - token = Some(token_env); - } - - let endpoint = env_var( - "SUBSTREAMS_ENDPOINT", - "https://api.streamingfast.io".to_string(), - ); - - let package_file = env_var("SUBSTREAMS_PACKAGE", "".to_string()); - if package_file.is_empty() { - panic!("Environment variable SUBSTREAMS_PACKAGE must be set"); - } - - let package = read_package(&package_file)?; - - let logger = logger(true); - // Set up Prometheus registry - let prometheus_registry = Arc::new(Registry::new()); - let metrics_registry = Arc::new(MetricsRegistry::new( - logger.clone(), - prometheus_registry.clone(), - )); - - let endpoint_metrics = EndpointMetrics::new( - logger.clone(), - &[endpoint.clone()], - Arc::new(MetricsRegistry::mock()), - ); - - let firehose = Arc::new(FirehoseEndpoint::new( - "substreams", - &endpoint, - token, - false, - false, - SubgraphLimit::Unlimited, - Arc::new(endpoint_metrics), - )); - - let client = Arc::new(ChainClient::new_firehose(FirehoseEndpoints::from(vec![ - firehose, - ]))); - - let mut stream: SubstreamsBlockStream = - SubstreamsBlockStream::new( - DeploymentHash::new("substreams".to_string()).unwrap(), - client, - None, - None, - Arc::new(Mapper { - schema: None, - skip_empty_blocks: false, - }), - package.modules.clone(), - module_name.to_string(), - vec![12369621], - vec![], - logger.clone(), - metrics_registry, - ); - - loop { - match stream.next().await { - None => { - break; - } - Some(event) => match event { - Err(_) => {} - Ok(block_stream_event) => match block_stream_event { - BlockStreamEvent::ProcessWasmBlock(_, _, _, _, _) => { - unreachable!("Cannot happen with this mapper") - } - BlockStreamEvent::Revert(_, _) => {} - BlockStreamEvent::ProcessBlock(block_with_trigger, _) => { - for change in block_with_trigger.block.changes.entity_changes { - for field in change.fields { - info!(&logger, "field: {:?}", field); - } - } - } - }, - }, - } - } - - Ok(()) -} - -fn read_package(file: &str) -> Result { - let content = std::fs::read(file).context(format_err!("read package {}", file))?; - substreams::Package::decode(content.as_ref()).context("decode command") -} diff --git a/chain/substreams/proto/codec.proto b/chain/substreams/proto/codec.proto deleted file mode 100644 index 5529cd774af..00000000000 --- a/chain/substreams/proto/codec.proto +++ /dev/null @@ -1,47 +0,0 @@ -syntax = "proto3"; - -package substreams.entity.v1; - -message EntityChanges { - repeated EntityChange entity_changes = 5; -} - -message EntityChange { - string entity = 1; - string id = 2; - uint64 ordinal = 3; - enum Operation { - UNSET = 0; // Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified - CREATE = 1; - UPDATE = 2; - DELETE = 3; - } - Operation operation = 4; - repeated Field fields = 5; -} - -message Value { - oneof typed { - int32 int32 = 1; - string bigdecimal = 2; - string bigint = 3; - string string = 4; - bytes bytes = 5; - bool bool = 6; - - //reserved 7 to 9; // For future types - - Array array = 10; - } -} - -message Array { - repeated Value value = 1; -} - -message Field { - string name = 1; - optional Value new_value = 3; - optional Value old_value = 5; -} - diff --git a/chain/substreams/src/block_ingestor.rs b/chain/substreams/src/block_ingestor.rs deleted file mode 100644 index 520e98dd461..00000000000 --- a/chain/substreams/src/block_ingestor.rs +++ /dev/null @@ -1,173 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use crate::mapper::Mapper; -use anyhow::{Context, Error}; -use graph::blockchain::{ - client::ChainClient, substreams_block_stream::SubstreamsBlockStream, BlockIngestor, -}; -use graph::prelude::MetricsRegistry; -use graph::slog::trace; -use graph::substreams::Package; -use graph::tokio_stream::StreamExt; -use graph::{ - blockchain::block_stream::BlockStreamEvent, - cheap_clone::CheapClone, - components::store::ChainStore, - prelude::{async_trait, error, info, DeploymentHash, Logger}, - util::backoff::ExponentialBackoff, -}; -use prost::Message; - -const SUBSTREAMS_HEAD_TRACKER_BYTES: &[u8; 89935] = include_bytes!( - "../../../substreams/substreams-head-tracker/substreams-head-tracker-v1.0.0.spkg" -); - -pub struct SubstreamsBlockIngestor { - chain_store: Arc, - client: Arc>, - logger: Logger, - chain_name: String, - metrics: Arc, -} - -impl SubstreamsBlockIngestor { - pub fn new( - chain_store: Arc, - client: Arc>, - logger: Logger, - chain_name: String, - metrics: Arc, - ) -> SubstreamsBlockIngestor { - SubstreamsBlockIngestor { - chain_store, - client, - logger, - chain_name, - metrics, - } - } - - async fn fetch_head_cursor(&self) -> String { - let mut backoff = - ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); - loop { - match self.chain_store.clone().chain_head_cursor() { - Ok(cursor) => return cursor.unwrap_or_default(), - Err(e) => { - error!(self.logger, "Fetching chain head cursor failed: {:#}", e); - - backoff.sleep_async().await; - } - } - } - } - - /// Consumes the incoming stream of blocks infinitely until it hits an error. In which case - /// the error is logged right away and the latest available cursor is returned - /// upstream for future consumption. - async fn process_blocks( - &self, - cursor: String, - mut stream: SubstreamsBlockStream, - ) -> String { - let mut latest_cursor = cursor; - - while let Some(message) = stream.next().await { - let (block, cursor) = match message { - Ok(BlockStreamEvent::ProcessWasmBlock( - _block_ptr, - _block_time, - _data, - _handler, - _cursor, - )) => { - unreachable!("Block ingestor should never receive raw blocks"); - } - Ok(BlockStreamEvent::ProcessBlock(triggers, cursor)) => { - (Arc::new(triggers.block), cursor) - } - Ok(BlockStreamEvent::Revert(_ptr, _cursor)) => { - trace!(self.logger, "Received undo block to ingest, skipping"); - continue; - } - Err(e) => { - info!( - self.logger, - "An error occurred while streaming blocks: {}", e - ); - break; - } - }; - - let res = self.process_new_block(block, cursor.to_string()).await; - if let Err(e) = res { - error!(self.logger, "Process block failed: {:#}", e); - break; - } - - latest_cursor = cursor.to_string() - } - - error!( - self.logger, - "Stream blocks complete unexpectedly, expecting stream to always stream blocks" - ); - latest_cursor - } - - async fn process_new_block( - &self, - block: Arc, - cursor: String, - ) -> Result<(), Error> { - trace!(self.logger, "Received new block to ingest {:?}", block); - - self.chain_store - .clone() - .set_chain_head(block, cursor) - .await - .context("Updating chain head")?; - - Ok(()) - } -} - -#[async_trait] -impl BlockIngestor for SubstreamsBlockIngestor { - async fn run(self: Box) { - let mapper = Arc::new(Mapper { - schema: None, - skip_empty_blocks: false, - }); - let mut latest_cursor = self.fetch_head_cursor().await; - let mut backoff = - ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); - let package = Package::decode(SUBSTREAMS_HEAD_TRACKER_BYTES.to_vec().as_ref()).unwrap(); - - loop { - let stream = SubstreamsBlockStream::::new( - DeploymentHash::default(), - self.client.cheap_clone(), - None, - Some(latest_cursor.clone()), - mapper.cheap_clone(), - package.modules.clone(), - "map_blocks".to_string(), - vec![-1], - vec![], - self.logger.cheap_clone(), - self.metrics.cheap_clone(), - ); - - // Consume the stream of blocks until an error is hit - latest_cursor = self.process_blocks(latest_cursor, stream).await; - - // If we reach this point, we must wait a bit before retrying - backoff.sleep_async().await; - } - } - - fn network_name(&self) -> String { - self.chain_name.clone() - } -} diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs deleted file mode 100644 index 4cdb8fdf895..00000000000 --- a/chain/substreams/src/block_stream.rs +++ /dev/null @@ -1,113 +0,0 @@ -use anyhow::Result; -use std::sync::Arc; - -use graph::{ - blockchain::{ - block_stream::{ - BlockStream, BlockStreamBuilder as BlockStreamBuilderTrait, FirehoseCursor, - }, - substreams_block_stream::SubstreamsBlockStream, - Blockchain, - }, - components::store::DeploymentLocator, - data::subgraph::UnifiedMappingApiVersion, - prelude::{async_trait, BlockNumber, BlockPtr}, - schema::InputSchema, - slog::o, -}; - -use crate::{ - mapper::{Mapper, WasmBlockMapper}, - Chain, TriggerFilter, -}; - -pub struct BlockStreamBuilder {} - -impl BlockStreamBuilder { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -/// Substreams doesn't actually use Firehose, the configuration for firehose and the grpc substream -/// is very similar, so we can re-use the configuration and the builder for it. -/// This is probably something to improve but for now it works. -impl BlockStreamBuilderTrait for BlockStreamBuilder { - async fn build_substreams( - &self, - chain: &Chain, - schema: InputSchema, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - subgraph_current_block: Option, - filter: Arc<::TriggerFilter>, - ) -> Result>> { - let logger = chain - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "SubstreamsBlockStream")); - - let stream = match &filter.mapping_handler { - Some(handler) => SubstreamsBlockStream::new( - deployment.hash, - chain.chain_client(), - subgraph_current_block, - block_cursor.as_ref().clone(), - Arc::new(WasmBlockMapper { - handler: handler.clone(), - }), - filter.modules.clone(), - filter.module_name.clone(), - filter.start_block.map(|x| vec![x]).unwrap_or_default(), - vec![], - logger, - chain.metrics_registry.clone(), - ), - - None => SubstreamsBlockStream::new( - deployment.hash, - chain.chain_client(), - subgraph_current_block, - block_cursor.as_ref().clone(), - Arc::new(Mapper { - schema: Some(schema), - skip_empty_blocks: true, - }), - filter.modules.clone(), - filter.module_name.clone(), - filter.start_block.map(|x| vec![x]).unwrap_or_default(), - vec![], - logger, - chain.metrics_registry.clone(), - ), - }; - - Ok(Box::new(stream)) - } - - async fn build_firehose( - &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _block_cursor: FirehoseCursor, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - unimplemented!() - } - - async fn build_polling( - &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - unimplemented!("polling block stream is not support for substreams") - } -} diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs deleted file mode 100644 index 1dad1764707..00000000000 --- a/chain/substreams/src/chain.rs +++ /dev/null @@ -1,218 +0,0 @@ -use crate::block_ingestor::SubstreamsBlockIngestor; -use crate::{data_source::*, EntityChanges, TriggerData, TriggerFilter, TriggersAdapter}; -use anyhow::Error; -use graph::blockchain::client::ChainClient; -use graph::blockchain::{ - BasicBlockchainBuilder, BlockIngestor, BlockTime, EmptyNodeCapabilities, NoopRuntimeAdapter, -}; -use graph::components::store::DeploymentCursorTracker; -use graph::env::EnvVars; -use graph::firehose::FirehoseEndpoints; -use graph::prelude::{BlockHash, CheapClone, Entity, LoggerFactory, MetricsRegistry}; -use graph::schema::EntityKey; -use graph::{ - blockchain::{ - self, - block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}, - BlockPtr, Blockchain, BlockchainKind, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - data::subgraph::UnifiedMappingApiVersion, - prelude::{async_trait, BlockNumber, ChainStore}, - slog::Logger, -}; - -use std::sync::Arc; - -// ParsedChanges are an internal representation of the equivalent operations defined on the -// graph-out format used by substreams. -// Unset serves as a sentinel value, if for some reason an unknown value is sent or the value -// was empty then it's probably an unintended behaviour. This code was moved here for performance -// reasons, but the validation is still performed during trigger processing so while Unset will -// very likely just indicate an error somewhere, as far as the stream is concerned we just pass -// that along and let the downstream components deal with it. -#[derive(Debug, Clone)] -pub enum ParsedChanges { - Unset, - Delete(EntityKey), - Upsert { key: EntityKey, entity: Entity }, -} - -#[derive(Default, Debug, Clone)] -pub struct Block { - pub hash: BlockHash, - pub number: BlockNumber, - pub changes: EntityChanges, - pub parsed_changes: Vec, -} - -impl blockchain::Block for Block { - fn ptr(&self) -> BlockPtr { - BlockPtr { - hash: self.hash.clone(), - number: self.number, - } - } - - fn parent_ptr(&self) -> Option { - None - } - - fn timestamp(&self) -> BlockTime { - BlockTime::NONE - } -} - -pub struct Chain { - chain_store: Arc, - block_stream_builder: Arc>, - - pub(crate) logger_factory: LoggerFactory, - pub(crate) client: Arc>, - pub(crate) metrics_registry: Arc, -} - -impl Chain { - pub fn new( - logger_factory: LoggerFactory, - firehose_endpoints: FirehoseEndpoints, - metrics_registry: Arc, - chain_store: Arc, - block_stream_builder: Arc>, - ) -> Self { - Self { - logger_factory, - client: Arc::new(ChainClient::new_firehose(firehose_endpoints)), - metrics_registry, - chain_store, - block_stream_builder, - } - } -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: substreams") - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Substreams; - - type Client = (); - type Block = Block; - type DataSource = DataSource; - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = NoopDataSourceTemplate; - type UnresolvedDataSourceTemplate = NoopDataSourceTemplate; - - /// Trigger data as parsed from the triggers adapter. - type TriggerData = TriggerData; - - /// Decoded trigger ready to be processed by the mapping. - /// New implementations should have this be the same as `TriggerData`. - type MappingTrigger = TriggerData; - - /// Trigger filter used as input to the triggers adapter. - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn triggers_adapter( - &self, - _log: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - Ok(Arc::new(TriggersAdapter {})) - } - - async fn new_block_stream( - &self, - deployment: DeploymentLocator, - store: impl DeploymentCursorTracker, - _start_blocks: Vec, - filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - self.block_stream_builder - .build_substreams( - self, - store.input_schema(), - deployment, - store.firehose_cursor(), - store.block_ptr(), - filter, - ) - .await - } - - fn is_refetch_block_required(&self) -> bool { - false - } - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - fn chain_store(&self) -> Arc { - self.chain_store.clone() - } - - async fn block_pointer_from_number( - &self, - _logger: &Logger, - number: BlockNumber, - ) -> Result { - // This is the same thing TriggersAdapter does, not sure if it's going to work but - // we also don't yet have a good way of getting this value until we sort out the - // chain store. - // TODO(filipe): Fix this once the chain_store is correctly setup for substreams. - Ok(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number, - }) - } - fn runtime_adapter(&self) -> Arc> { - Arc::new(NoopRuntimeAdapter::default()) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } - - fn block_ingestor(&self) -> anyhow::Result> { - Ok(Box::new(SubstreamsBlockIngestor::new( - self.chain_store.cheap_clone(), - self.client.cheap_clone(), - self.logger_factory.component_logger("", None), - "substreams".to_string(), - self.metrics_registry.cheap_clone(), - ))) - } -} - -impl blockchain::BlockchainBuilder for BasicBlockchainBuilder { - fn build(self, _config: &Arc) -> Chain { - let BasicBlockchainBuilder { - logger_factory, - name: _, - chain_store, - firehose_endpoints, - metrics_registry, - } = self; - - Chain { - chain_store, - block_stream_builder: Arc::new(crate::BlockStreamBuilder::new()), - logger_factory, - client: Arc::new(ChainClient::new_firehose(firehose_endpoints)), - metrics_registry, - } - } -} diff --git a/chain/substreams/src/codec.rs b/chain/substreams/src/codec.rs deleted file mode 100644 index 31781baa201..00000000000 --- a/chain/substreams/src/codec.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[rustfmt::skip] -#[path = "protobuf/substreams.entity.v1.rs"] -mod pbsubstreamsentity; - -pub use pbsubstreamsentity::*; diff --git a/chain/substreams/src/data_source.rs b/chain/substreams/src/data_source.rs deleted file mode 100644 index dba85695f88..00000000000 --- a/chain/substreams/src/data_source.rs +++ /dev/null @@ -1,664 +0,0 @@ -use std::{collections::HashSet, sync::Arc}; - -use anyhow::{anyhow, Context, Error}; -use graph::{ - blockchain, - cheap_clone::CheapClone, - components::link_resolver::LinkResolver, - prelude::{async_trait, BlockNumber, DataSourceTemplateInfo, Link}, - slog::Logger, -}; - -use prost::Message; -use serde::Deserialize; - -use crate::{chain::Chain, Block, TriggerData}; - -pub const SUBSTREAMS_KIND: &str = "substreams"; - -const DYNAMIC_DATA_SOURCE_ERROR: &str = "Substreams do not support dynamic data sources"; -const TEMPLATE_ERROR: &str = "Substreams do not support templates"; - -const ALLOWED_MAPPING_KIND: [&str; 1] = ["substreams/graph-entities"]; -const SUBSTREAMS_HANDLER_KIND: &str = "substreams"; -#[derive(Clone, Debug, PartialEq)] -/// Represents the DataSource portion of the manifest once it has been parsed -/// and the substream spkg has been downloaded + parsed. -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub initial_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!("Substreams does not support templates")) - } - - fn address(&self) -> Option<&[u8]> { - None - } - - fn start_block(&self) -> BlockNumber { - self.initial_block.unwrap_or(0) - } - - fn end_block(&self) -> Option { - None - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - None - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - self.mapping.handler.as_ref().map(|h| h.runtime.clone()) - } - - fn handler_kinds(&self) -> HashSet<&str> { - // This is placeholder, substreams do not have a handler kind. - vec![SUBSTREAMS_HANDLER_KIND].into_iter().collect() - } - - // match_and_decode only seems to be used on the default trigger processor which substreams - // bypasses so it should be fine to leave it unimplemented. - fn match_and_decode( - &self, - _trigger: &TriggerData, - _block: &Arc, - _logger: &Logger, - ) -> Result>, Error> { - unimplemented!() - } - - fn is_duplicate_of(&self, _other: &Self) -> bool { - todo!() - } - - fn as_stored_dynamic_data_source(&self) -> graph::components::store::StoredDynamicDataSource { - unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR) - } - - fn validate(&self) -> Vec { - let mut errs = vec![]; - - if &self.kind != SUBSTREAMS_KIND { - errs.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - SUBSTREAMS_KIND, - self.kind - )) - } - - if self.name.is_empty() { - errs.push(anyhow!("name cannot be empty")); - } - - if !ALLOWED_MAPPING_KIND.contains(&self.mapping.kind.as_str()) { - errs.push(anyhow!( - "mapping kind has to be one of {:?}, found {}", - ALLOWED_MAPPING_KIND, - self.mapping.kind - )) - } - - errs - } - - fn from_stored_dynamic_data_source( - _template: &::DataSourceTemplate, - _stored: graph::components::store::StoredDynamicDataSource, - ) -> Result { - Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) - } -} - -#[derive(Clone, Debug, Default, PartialEq)] -/// Module name comes from the manifest, package is the parsed spkg file. -pub struct Source { - pub module_name: String, - pub package: graph::substreams::Package, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Mapping { - pub api_version: semver::Version, - pub kind: String, - pub handler: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MappingHandler { - pub handler: String, - pub runtime: Arc>, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -/// Raw representation of the data source for deserialization purposes. -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: UnresolvedSource, - pub mapping: UnresolvedMapping, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -/// Text api_version, before parsing and validation. -pub struct UnresolvedMapping { - pub api_version: String, - pub kind: String, - pub handler: Option, - pub file: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let content = resolver.cat(logger, &self.source.package.file).await?; - - let mut package = graph::substreams::Package::decode(content.as_ref())?; - - let module = match package.modules.as_mut() { - Some(modules) => modules - .modules - .iter_mut() - .find(|module| module.name == self.source.package.module_name) - .map(|module| { - if let Some(params) = self.source.package.params { - graph::substreams::patch_module_params(params, module); - } - module - }), - None => None, - }; - - let initial_block: Option = match module { - Some(module) => match &module.kind { - Some(graph::substreams::module::Kind::KindMap(_)) => Some(module.initial_block), - _ => { - return Err(anyhow!( - "Substreams module {} must be of 'map' kind", - module.name - )) - } - }, - None => { - return Err(anyhow!( - "Substreams module {} does not exist", - self.source.package.module_name - )) - } - }; - - let initial_block: Option = initial_block - .map_or(Ok(None), |x: u64| TryInto::::try_into(x).map(Some)) - .map_err(anyhow::Error::from)?; - - let handler = match (self.mapping.handler, self.mapping.file) { - (Some(handler), Some(file)) => { - let module_bytes = resolver - .cat(logger, &file) - .await - .with_context(|| format!("failed to resolve mapping {}", file.link))?; - - Some(MappingHandler { - handler, - runtime: Arc::new(module_bytes), - }) - } - _ => None, - }; - - Ok(DataSource { - kind: SUBSTREAMS_KIND.into(), - network: self.network, - name: self.name, - source: Source { - module_name: self.source.package.module_name, - package, - }, - mapping: Mapping { - api_version: semver::Version::parse(&self.mapping.api_version)?, - kind: self.mapping.kind, - handler, - }, - context: Arc::new(None), - initial_block, - }) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -/// Source is a part of the manifest and this is needed for parsing. -pub struct UnresolvedSource { - #[serde(rename = "startBlock", default)] - start_block: Option, - package: UnresolvedPackage, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -/// The unresolved Package section of the manifest. -pub struct UnresolvedPackage { - pub module_name: String, - pub file: Link, - pub params: Option, -} - -#[derive(Debug, Clone, Default, Deserialize)] -/// This is necessary for the Blockchain trait associated types, substreams do not support -/// data source templates so this is a noop and is not expected to be called. -pub struct NoopDataSourceTemplate {} - -impl blockchain::DataSourceTemplate for NoopDataSourceTemplate { - fn name(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn api_version(&self) -> semver::Version { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn runtime(&self) -> Option>> { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn manifest_idx(&self) -> u32 { - todo!() - } - - fn kind(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } -} - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for NoopDataSourceTemplate { - async fn resolve( - self, - _resolver: &Arc, - _logger: &Logger, - _manifest_idx: u32, - ) -> Result { - unimplemented!("{}", TEMPLATE_ERROR) - } -} - -#[cfg(test)] -mod test { - use std::{str::FromStr, sync::Arc}; - - use anyhow::Error; - use graph::{ - blockchain::{DataSource as _, UnresolvedDataSource as _}, - components::link_resolver::LinkResolver, - prelude::{async_trait, serde_yaml, JsonValueStream, Link}, - slog::{o, Discard, Logger}, - substreams::module::{Kind, KindMap, KindStore}, - substreams::{ - module::input::{Input, Params}, - Module, Modules, Package, - }, - }; - use prost::Message; - - use crate::{DataSource, Mapping, UnresolvedDataSource, UnresolvedMapping, SUBSTREAMS_KIND}; - - #[test] - fn parse_data_source() { - let ds: UnresolvedDataSource = serde_yaml::from_str(TEMPLATE_DATA_SOURCE).unwrap(); - let expected = UnresolvedDataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::UnresolvedSource { - package: crate::UnresolvedPackage { - module_name: "output".into(), - file: Link { - link: "/ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT".into(), - }, - params: None, - }, - start_block: None, - }, - mapping: UnresolvedMapping { - api_version: "0.0.7".into(), - kind: "substreams/graph-entities".into(), - handler: None, - file: None, - }, - }; - assert_eq!(ds, expected); - } - - #[test] - fn parse_data_source_with_params() { - let ds: UnresolvedDataSource = - serde_yaml::from_str(TEMPLATE_DATA_SOURCE_WITH_PARAMS).unwrap(); - let expected = UnresolvedDataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::UnresolvedSource { - package: crate::UnresolvedPackage { - module_name: "output".into(), - file: Link { - link: "/ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT".into(), - }, - params: Some("x\ny\n123\n".into()), - }, - start_block: None, - }, - mapping: UnresolvedMapping { - api_version: "0.0.7".into(), - kind: "substreams/graph-entities".into(), - handler: None, - file: None, - }, - }; - assert_eq!(ds, expected); - } - - #[tokio::test] - async fn data_source_conversion() { - let ds: UnresolvedDataSource = serde_yaml::from_str(TEMPLATE_DATA_SOURCE).unwrap(); - let link_resolver: Arc = Arc::new(NoopLinkResolver {}); - let logger = Logger::root(Discard, o!()); - let ds: DataSource = ds.resolve(&link_resolver, &logger, 0).await.unwrap(); - let expected = DataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::Source { - module_name: "output".into(), - package: gen_package(), - }, - mapping: Mapping { - api_version: semver::Version::from_str("0.0.7").unwrap(), - kind: "substreams/graph-entities".into(), - handler: None, - }, - context: Arc::new(None), - initial_block: Some(123), - }; - assert_eq!(ds, expected); - } - - #[tokio::test] - async fn data_source_conversion_override_params() { - let mut package = gen_package(); - let mut modules = package.modules.unwrap(); - modules.modules.get_mut(0).map(|module| { - module.inputs = vec![graph::substreams::module::Input { - input: Some(Input::Params(Params { - value: "x\ny\n123\n".into(), - })), - }] - }); - package.modules = Some(modules); - - let ds: UnresolvedDataSource = - serde_yaml::from_str(TEMPLATE_DATA_SOURCE_WITH_PARAMS).unwrap(); - let link_resolver: Arc = Arc::new(NoopLinkResolver {}); - let logger = Logger::root(Discard, o!()); - let ds: DataSource = ds.resolve(&link_resolver, &logger, 0).await.unwrap(); - let expected = DataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::Source { - module_name: "output".into(), - package, - }, - mapping: Mapping { - api_version: semver::Version::from_str("0.0.7").unwrap(), - kind: "substreams/graph-entities".into(), - handler: None, - }, - context: Arc::new(None), - initial_block: Some(123), - }; - assert_eq!(ds, expected); - } - - #[test] - fn data_source_validation() { - let mut ds = gen_data_source(); - assert_eq!(true, ds.validate().is_empty()); - - ds.network = None; - assert_eq!(true, ds.validate().is_empty()); - - ds.kind = "asdasd".into(); - ds.name = "".into(); - ds.mapping.kind = "asdasd".into(); - let errs: Vec = ds.validate().into_iter().map(|e| e.to_string()).collect(); - assert_eq!( - errs, - vec![ - "data source has invalid `kind`, expected substreams but found asdasd", - "name cannot be empty", - "mapping kind has to be one of [\"substreams/graph-entities\"], found asdasd" - ] - ); - } - - #[test] - fn parse_data_source_with_maping() { - let ds: UnresolvedDataSource = - serde_yaml::from_str(TEMPLATE_DATA_SOURCE_WITH_MAPPING).unwrap(); - - let expected = UnresolvedDataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::UnresolvedSource { - package: crate::UnresolvedPackage { - module_name: "output".into(), - file: Link { - link: "/ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT".into(), - }, - params: Some("x\ny\n123\n".into()), - }, - start_block: None, - }, - mapping: UnresolvedMapping { - api_version: "0.0.7".into(), - kind: "substreams/graph-entities".into(), - handler: Some("bananas".to_string()), - file: Some(Link { - link: "./src/mappings.ts".to_string(), - }), - }, - }; - assert_eq!(ds, expected); - } - - fn gen_package() -> Package { - Package { - proto_files: vec![], - version: 0, - modules: Some(Modules { - modules: vec![ - Module { - name: "output".into(), - initial_block: 123, - binary_entrypoint: "output".into(), - binary_index: 0, - kind: Some(Kind::KindMap(KindMap { - output_type: "proto".into(), - })), - inputs: vec![], - output: None, - }, - Module { - name: "store_mod".into(), - initial_block: 0, - binary_entrypoint: "store_mod".into(), - binary_index: 0, - kind: Some(Kind::KindStore(KindStore { - update_policy: 1, - value_type: "proto1".into(), - })), - inputs: vec![], - output: None, - }, - Module { - name: "map_mod".into(), - initial_block: 123456, - binary_entrypoint: "other2".into(), - binary_index: 0, - kind: Some(Kind::KindMap(KindMap { - output_type: "proto2".into(), - })), - inputs: vec![], - output: None, - }, - ], - binaries: vec![], - }), - module_meta: vec![], - package_meta: vec![], - sink_config: None, - network: "".into(), - sink_module: "".into(), - } - } - - fn gen_data_source() -> DataSource { - DataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::Source { - module_name: "".to_string(), - package: gen_package(), - }, - mapping: Mapping { - api_version: semver::Version::from_str("0.0.7").unwrap(), - kind: "substreams/graph-entities".into(), - handler: None, - }, - context: Arc::new(None), - initial_block: None, - } - } - - const TEMPLATE_DATA_SOURCE: &str = r#" - kind: substreams - name: Uniswap - network: mainnet - source: - package: - moduleName: output - file: - /: /ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT - # This IPFs path would be generated from a local path at deploy time - mapping: - kind: substreams/graph-entities - apiVersion: 0.0.7 - "#; - - const TEMPLATE_DATA_SOURCE_WITH_MAPPING: &str = r#" - kind: substreams - name: Uniswap - network: mainnet - source: - package: - moduleName: output - file: - /: /ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT - # This IPFs path would be generated from a local path at deploy time - params: | - x - y - 123 - mapping: - kind: substreams/graph-entities - apiVersion: 0.0.7 - file: - /: ./src/mappings.ts - handler: bananas - "#; - - const TEMPLATE_DATA_SOURCE_WITH_PARAMS: &str = r#" - kind: substreams - name: Uniswap - network: mainnet - source: - package: - moduleName: output - file: - /: /ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT - # This IPFs path would be generated from a local path at deploy time - params: | - x - y - 123 - mapping: - kind: substreams/graph-entities - apiVersion: 0.0.7 - "#; - - #[derive(Debug)] - struct NoopLinkResolver {} - - #[async_trait] - impl LinkResolver for NoopLinkResolver { - fn with_timeout(&self, _timeout: std::time::Duration) -> Box { - unimplemented!() - } - - fn with_retries(&self) -> Box { - unimplemented!() - } - - async fn cat(&self, _logger: &Logger, _link: &Link) -> Result, Error> { - Ok(gen_package().encode_to_vec()) - } - - async fn get_block(&self, _logger: &Logger, _link: &Link) -> Result, Error> { - unimplemented!() - } - - async fn json_stream( - &self, - _logger: &Logger, - _link: &Link, - ) -> Result { - unimplemented!() - } - } -} diff --git a/chain/substreams/src/lib.rs b/chain/substreams/src/lib.rs deleted file mode 100644 index 664ceab6d65..00000000000 --- a/chain/substreams/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod block_stream; -mod chain; -mod codec; -mod data_source; -mod trigger; - -pub mod block_ingestor; -pub mod mapper; - -pub use crate::chain::Chain; -pub use block_stream::BlockStreamBuilder; -pub use chain::*; -pub use codec::EntityChanges; -pub use data_source::*; -pub use trigger::*; - -pub use codec::Field; diff --git a/chain/substreams/src/mapper.rs b/chain/substreams/src/mapper.rs deleted file mode 100644 index f08fedb462e..00000000000 --- a/chain/substreams/src/mapper.rs +++ /dev/null @@ -1,398 +0,0 @@ -use std::collections::HashMap; -use std::str::FromStr; - -use crate::codec::{entity_change, EntityChanges}; -use anyhow::{anyhow, Error}; -use graph::blockchain::block_stream::{ - BlockStreamEvent, BlockStreamMapper, BlockWithTriggers, FirehoseCursor, SubstreamsError, -}; -use graph::blockchain::BlockTime; -use graph::data::store::scalar::Bytes; -use graph::data::store::IdType; -use graph::data::value::Word; -use graph::data_source::CausalityRegion; -use graph::prelude::{async_trait, BigInt, BlockHash, BlockNumber, Logger, Value}; -use graph::prelude::{BigDecimal, BlockPtr}; -use graph::schema::InputSchema; -use graph::slog::error; -use graph::substreams::Clock; -use prost::Message; - -use crate::{Block, Chain, ParsedChanges, TriggerData}; - -// WasmBlockMapper will not perform any transformation to the block and cannot make assumptions -// about the block format. This mode just works a passthrough from the block stream to the subgraph -// mapping which will do the decoding and store actions. -pub struct WasmBlockMapper { - pub handler: String, -} - -#[async_trait] -impl BlockStreamMapper for WasmBlockMapper { - fn decode_block(&self, _output: Option<&[u8]>) -> Result, Error> { - unreachable!("WasmBlockMapper does not do block decoding") - } - - async fn block_with_triggers( - &self, - _logger: &Logger, - _block: crate::Block, - ) -> Result, Error> { - unreachable!("WasmBlockMapper does not do trigger decoding") - } - - async fn handle_substreams_block( - &self, - logger: &Logger, - clock: Clock, - cursor: FirehoseCursor, - block: Vec, - ) -> Result, Error> { - let Clock { - id, - number, - timestamp, - } = clock; - - let block_ptr = BlockPtr { - hash: BlockHash::from(id.into_bytes()), - number: BlockNumber::from(TryInto::::try_into(number)?), - }; - - let block_data = block.into_boxed_slice(); - - // `timestamp` is an `Option`, but it should always be set - let timestamp = match timestamp { - None => { - error!(logger, - "Substream block is missing a timestamp"; - "cursor" => cursor.to_string(), - "number" => number, - ); - return Err(anyhow!( - "Substream block is missing a timestamp at cursor {cursor}, block number {number}" - )); - } - Some(ts) => BlockTime::since_epoch(ts.seconds, ts.nanos as u32), - }; - - Ok(BlockStreamEvent::ProcessWasmBlock( - block_ptr, - timestamp, - block_data, - self.handler.clone(), - cursor, - )) - } -} - -// Mapper will transform the proto content coming from substreams in the graph-out format -// into the internal Block representation. If schema is passed then additional transformation -// into from the substreams block representation is performed into the Entity model used by -// the store. If schema is None then only the original block is passed. This None should only -// be used for block ingestion where entity content is empty and gets discarded. -pub struct Mapper { - pub schema: Option, - // Block ingestors need the block to be returned so they can populate the cache - // block streams, however, can shave some time by just skipping. - pub skip_empty_blocks: bool, -} - -#[async_trait] -impl BlockStreamMapper for Mapper { - fn decode_block(&self, output: Option<&[u8]>) -> Result, Error> { - let changes: EntityChanges = match output { - Some(msg) => Message::decode(msg).map_err(SubstreamsError::DecodingError)?, - None => EntityChanges { - entity_changes: [].to_vec(), - }, - }; - - let parsed_changes = match self.schema.as_ref() { - Some(schema) => parse_changes(&changes, schema)?, - None if self.skip_empty_blocks => return Ok(None), - None => vec![], - }; - - let hash = BlockHash::zero(); - let number = BlockNumber::MIN; - let block = Block { - hash, - number, - changes, - parsed_changes, - }; - - Ok(Some(block)) - } - - async fn block_with_triggers( - &self, - logger: &Logger, - block: Block, - ) -> Result, Error> { - let mut triggers = vec![]; - if block.changes.entity_changes.len() >= 1 { - triggers.push(TriggerData {}); - } - - Ok(BlockWithTriggers::new(block, triggers, logger)) - } - - async fn handle_substreams_block( - &self, - logger: &Logger, - clock: Clock, - cursor: FirehoseCursor, - block: Vec, - ) -> Result, Error> { - let block_number: BlockNumber = clock.number.try_into()?; - let block_hash = clock.id.as_bytes().to_vec().try_into()?; - - let block = self - .decode_block(Some(&block))? - .ok_or_else(|| anyhow!("expected block to not be empty"))?; - - let block = self.block_with_triggers(logger, block).await.map(|bt| { - let mut block = bt; - - block.block.number = block_number; - block.block.hash = block_hash; - block - })?; - - Ok(BlockStreamEvent::ProcessBlock(block, cursor)) - } -} - -fn parse_changes( - changes: &EntityChanges, - schema: &InputSchema, -) -> anyhow::Result> { - let mut parsed_changes = vec![]; - for entity_change in changes.entity_changes.iter() { - let mut parsed_data: HashMap = HashMap::default(); - let entity_type = schema.entity_type(&entity_change.entity)?; - - // Make sure that the `entity_id` gets set to a value - // that is safe for roundtrips through the database. In - // particular, if the type of the id is `Bytes`, we have - // to make sure that the `entity_id` starts with `0x` as - // that will be what the key for such an entity have - // when it is read from the database. - // - // Needless to say, this is a very ugly hack, and the - // real fix is what's described in [this - // issue](https://github.com/graphprotocol/graph-node/issues/4663) - let entity_id: String = match entity_type.id_type()? { - IdType::String | IdType::Int8 => entity_change.id.clone(), - IdType::Bytes => { - if entity_change.id.starts_with("0x") { - entity_change.id.clone() - } else { - format!("0x{}", entity_change.id) - } - } - }; - // Substreams don't currently support offchain data - let key = entity_type.parse_key_in(Word::from(entity_id), CausalityRegion::ONCHAIN)?; - - let id = key.id_value(); - parsed_data.insert(Word::from("id"), id); - - let changes = match entity_change.operation() { - entity_change::Operation::Create | entity_change::Operation::Update => { - for field in entity_change.fields.iter() { - let new_value: &crate::codec::value::Typed = match &field.new_value { - Some(crate::codec::Value { - typed: Some(new_value), - }) => &new_value, - _ => continue, - }; - - let value: Value = decode_value(new_value)?; - *parsed_data - .entry(Word::from(field.name.as_str())) - .or_insert(Value::Null) = value; - } - let entity = schema - .make_entity(parsed_data) - .map_err(anyhow::Error::from)?; - - ParsedChanges::Upsert { key, entity } - } - entity_change::Operation::Delete => ParsedChanges::Delete(key), - entity_change::Operation::Unset => ParsedChanges::Unset, - }; - parsed_changes.push(changes); - } - - Ok(parsed_changes) -} - -fn decode_value(value: &crate::codec::value::Typed) -> anyhow::Result { - use crate::codec::value::Typed; - - match value { - Typed::Int32(new_value) => Ok(Value::Int(*new_value)), - - Typed::Bigdecimal(new_value) => BigDecimal::from_str(new_value) - .map(Value::BigDecimal) - .map_err(|err| anyhow::Error::from(err)), - - Typed::Bigint(new_value) => BigInt::from_str(new_value) - .map(Value::BigInt) - .map_err(|err| anyhow::Error::from(err)), - - Typed::String(new_value) => { - let mut string = new_value.clone(); - - // Strip null characters since they are not accepted by Postgres. - if string.contains('\u{0000}') { - string = string.replace('\u{0000}', ""); - } - Ok(Value::String(string)) - } - - Typed::Bytes(new_value) => base64::decode(new_value) - .map(|bs| Value::Bytes(Bytes::from(bs))) - .map_err(|err| anyhow::Error::from(err)), - - Typed::Bool(new_value) => Ok(Value::Bool(*new_value)), - - Typed::Array(arr) => arr - .value - .iter() - .filter_map(|item| item.typed.as_ref().map(decode_value)) - .collect::>>() - .map(Value::List), - } -} - -#[cfg(test)] -mod test { - use std::{ops::Add, str::FromStr}; - - use super::decode_value; - use crate::codec::value::Typed; - use crate::codec::{Array, Value}; - use graph::{ - data::store::scalar::Bytes, - prelude::{BigDecimal, BigInt, Value as GraphValue}, - }; - - #[test] - fn validate_substreams_field_types() { - struct Case { - name: String, - value: Value, - expected_value: GraphValue, - } - - let cases = vec![ - Case { - name: "string value".to_string(), - value: Value { - typed: Some(Typed::String( - "d4325ee72c39999e778a9908f5fb0803f78e30c441a5f2ce5c65eee0e0eba59d" - .to_string(), - )), - }, - expected_value: GraphValue::String( - "d4325ee72c39999e778a9908f5fb0803f78e30c441a5f2ce5c65eee0e0eba59d".to_string(), - ), - }, - Case { - name: "bytes value".to_string(), - value: Value { - typed: Some(Typed::Bytes( - base64::encode( - hex::decode( - "445247fe150195bd866516594e087e1728294aa831613f4d48b8ec618908519f", - ) - .unwrap(), - ) - .into_bytes(), - )), - }, - expected_value: GraphValue::Bytes( - Bytes::from_str( - "0x445247fe150195bd866516594e087e1728294aa831613f4d48b8ec618908519f", - ) - .unwrap(), - ), - }, - Case { - name: "int value for block".to_string(), - value: Value { - typed: Some(Typed::Int32(12369760)), - }, - expected_value: GraphValue::Int(12369760), - }, - Case { - name: "negative int value".to_string(), - value: Value { - typed: Some(Typed::Int32(-12369760)), - }, - expected_value: GraphValue::Int(-12369760), - }, - Case { - name: "big int".to_string(), - value: Value { - typed: Some(Typed::Bigint("123".to_string())), - }, - expected_value: GraphValue::BigInt(BigInt::from(123u64)), - }, - Case { - name: "big int > u64".to_string(), - value: Value { - typed: Some(Typed::Bigint( - BigInt::from(u64::MAX).add(BigInt::from(1)).to_string(), - )), - }, - expected_value: GraphValue::BigInt(BigInt::from(u64::MAX).add(BigInt::from(1))), - }, - Case { - name: "big decimal value".to_string(), - value: Value { - typed: Some(Typed::Bigdecimal("3133363633312e35".to_string())), - }, - expected_value: GraphValue::BigDecimal(BigDecimal::new( - BigInt::from(3133363633312u64), - 35, - )), - }, - Case { - name: "bool value".to_string(), - value: Value { - typed: Some(Typed::Bool(true)), - }, - expected_value: GraphValue::Bool(true), - }, - Case { - name: "string array".to_string(), - value: Value { - typed: Some(Typed::Array(Array { - value: vec![ - Value { - typed: Some(Typed::String("1".to_string())), - }, - Value { - typed: Some(Typed::String("2".to_string())), - }, - Value { - typed: Some(Typed::String("3".to_string())), - }, - ], - })), - }, - expected_value: GraphValue::List(vec!["1".into(), "2".into(), "3".into()]), - }, - ]; - - for case in cases.into_iter() { - let value: GraphValue = decode_value(&case.value.typed.unwrap()).unwrap(); - assert_eq!(case.expected_value, value, "failed case: {}", case.name) - } - } -} diff --git a/chain/substreams/src/protobuf/substreams.entity.v1.rs b/chain/substreams/src/protobuf/substreams.entity.v1.rs deleted file mode 100644 index 47368e25fba..00000000000 --- a/chain/substreams/src/protobuf/substreams.entity.v1.rs +++ /dev/null @@ -1,109 +0,0 @@ -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityChanges { - #[prost(message, repeated, tag = "5")] - pub entity_changes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityChange { - #[prost(string, tag = "1")] - pub entity: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub id: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub ordinal: u64, - #[prost(enumeration = "entity_change::Operation", tag = "4")] - pub operation: i32, - #[prost(message, repeated, tag = "5")] - pub fields: ::prost::alloc::vec::Vec, -} -/// Nested message and enum types in `EntityChange`. -pub mod entity_change { - #[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, - ::prost::Enumeration - )] - #[repr(i32)] - pub enum Operation { - /// Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified - Unset = 0, - Create = 1, - Update = 2, - Delete = 3, - } - impl Operation { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Operation::Unset => "UNSET", - Operation::Create => "CREATE", - Operation::Update => "UPDATE", - Operation::Delete => "DELETE", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "UNSET" => Some(Self::Unset), - "CREATE" => Some(Self::Create), - "UPDATE" => Some(Self::Update), - "DELETE" => Some(Self::Delete), - _ => None, - } - } - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Value { - #[prost(oneof = "value::Typed", tags = "1, 2, 3, 4, 5, 6, 10")] - pub typed: ::core::option::Option, -} -/// Nested message and enum types in `Value`. -pub mod value { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Typed { - #[prost(int32, tag = "1")] - Int32(i32), - #[prost(string, tag = "2")] - Bigdecimal(::prost::alloc::string::String), - #[prost(string, tag = "3")] - Bigint(::prost::alloc::string::String), - #[prost(string, tag = "4")] - String(::prost::alloc::string::String), - #[prost(bytes, tag = "5")] - Bytes(::prost::alloc::vec::Vec), - #[prost(bool, tag = "6")] - Bool(bool), - #[prost(message, tag = "10")] - Array(super::Array), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Array { - #[prost(message, repeated, tag = "1")] - pub value: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Field { - #[prost(string, tag = "1")] - pub name: ::prost::alloc::string::String, - #[prost(message, optional, tag = "3")] - pub new_value: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub old_value: ::core::option::Option, -} diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs deleted file mode 100644 index 0e42eb7734a..00000000000 --- a/chain/substreams/src/trigger.rs +++ /dev/null @@ -1,251 +0,0 @@ -use std::sync::Arc; - -use anyhow::Error; -use graph::{ - blockchain::{ - self, block_stream::BlockWithTriggers, BlockPtr, EmptyNodeCapabilities, MappingTriggerTrait, - }, - components::{ - store::{DeploymentLocator, SubgraphFork}, - subgraph::{MappingError, ProofOfIndexingEvent, SharedProofOfIndexing}, - }, - data_source, - prelude::{ - anyhow, async_trait, BlockHash, BlockNumber, BlockState, CheapClone, RuntimeHostBuilder, - }, - slog::Logger, - substreams::Modules, -}; -use graph_runtime_wasm::module::ToAscPtr; -use lazy_static::__Deref; - -use crate::{Block, Chain, NoopDataSourceTemplate, ParsedChanges}; - -#[derive(Eq, PartialEq, PartialOrd, Ord, Debug)] -pub struct TriggerData {} - -impl MappingTriggerTrait for TriggerData { - fn error_context(&self) -> String { - "Failed to process substreams block".to_string() - } -} - -impl blockchain::TriggerData for TriggerData { - // TODO(filipe): Can this be improved with some data from the block? - fn error_context(&self) -> String { - "Failed to process substreams block".to_string() - } - - fn address_match(&self) -> Option<&[u8]> { - None - } -} - -impl ToAscPtr for TriggerData { - // substreams doesn't rely on wasm on the graph-node so this is not needed. - fn to_asc_ptr( - self, - _heap: &mut H, - _gas: &graph::runtime::gas::GasCounter, - ) -> Result, graph::runtime::HostExportError> { - unimplemented!() - } -} - -#[derive(Debug, Clone, Default)] -pub struct TriggerFilter { - pub(crate) modules: Option, - pub(crate) module_name: String, - pub(crate) start_block: Option, - pub(crate) data_sources_len: u8, - // the handler to call for subgraph mappings, if this is set then the binary block content - // should be passed to the mappings. - pub(crate) mapping_handler: Option, -} - -#[cfg(debug_assertions)] -impl TriggerFilter { - pub fn modules(&self) -> &Option { - &self.modules - } - - pub fn module_name(&self) -> &str { - &self.module_name - } - - pub fn start_block(&self) -> &Option { - &self.start_block - } - - pub fn data_sources_len(&self) -> u8 { - self.data_sources_len - } -} - -// TriggerFilter should bypass all triggers and just rely on block since all the data received -// should already have been processed. -impl blockchain::TriggerFilter for TriggerFilter { - fn extend_with_template(&mut self, _data_source: impl Iterator) { - } - - /// this function is not safe to call multiple times, only one DataSource is supported for - /// - fn extend<'a>( - &mut self, - mut data_sources: impl Iterator + Clone, - ) { - let Self { - modules, - module_name, - start_block, - data_sources_len, - mapping_handler, - } = self; - - if *data_sources_len >= 1 { - return; - } - - if let Some(ds) = data_sources.next() { - *data_sources_len = 1; - *modules = ds.source.package.modules.clone(); - *module_name = ds.source.module_name.clone(); - *start_block = ds.initial_block; - *mapping_handler = ds.mapping.handler.as_ref().map(|h| h.handler.clone()); - } - } - - fn node_capabilities(&self) -> EmptyNodeCapabilities { - EmptyNodeCapabilities::default() - } - - fn to_firehose_filter(self) -> Vec { - unimplemented!("this should never be called for this type") - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl blockchain::TriggersAdapter for TriggersAdapter { - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - unimplemented!() - } - - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result>, Error> { - unimplemented!() - } - - async fn triggers_in_block( - &self, - _logger: &Logger, - _block: Block, - _filter: &TriggerFilter, - ) -> Result, Error> { - unimplemented!() - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - unimplemented!() - } - - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - // This seems to work for a lot of the firehose chains. - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -fn write_poi_event( - proof_of_indexing: &SharedProofOfIndexing, - poi_event: &ProofOfIndexingEvent, - causality_region: &str, - logger: &Logger, -) { - if let Some(proof_of_indexing) = proof_of_indexing { - let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); - proof_of_indexing.write(logger, causality_region, poi_event); - } -} - -pub struct TriggerProcessor { - pub locator: DeploymentLocator, -} - -impl TriggerProcessor { - pub fn new(locator: DeploymentLocator) -> Self { - Self { locator } - } -} - -#[async_trait] -impl graph::prelude::TriggerProcessor for TriggerProcessor -where - T: RuntimeHostBuilder, -{ - async fn process_trigger( - &self, - logger: &Logger, - _: Box + Send + '_>, - block: &Arc, - _trigger: &data_source::TriggerData, - mut state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - _debug_fork: &Option>, - _subgraph_metrics: &Arc, - _instrument: bool, - ) -> Result, MappingError> { - for parsed_change in block.parsed_changes.clone().into_iter() { - match parsed_change { - ParsedChanges::Unset => { - // Potentially an issue with the server side or - // we are running an outdated version. In either case we should abort. - return Err(MappingError::Unknown(anyhow!("Detected UNSET entity operation, either a server error or there's a new type of operation and we're running an outdated protobuf"))); - } - ParsedChanges::Upsert { key, entity } => { - write_poi_event( - proof_of_indexing, - &ProofOfIndexingEvent::SetEntity { - entity_type: key.entity_type.as_str(), - id: &key.entity_id.to_string(), - data: &entity, - }, - causality_region, - logger, - ); - - state.entity_cache.set(key, entity)?; - } - ParsedChanges::Delete(entity_key) => { - let entity_type = entity_key.entity_type.cheap_clone(); - let id = entity_key.entity_id.clone(); - state.entity_cache.remove(entity_key); - - write_poi_event( - proof_of_indexing, - &ProofOfIndexingEvent::RemoveEntity { - entity_type: entity_type.as_str(), - id: &id.to_string(), - }, - causality_region, - logger, - ); - } - } - } - - Ok(state) - } -} diff --git a/core/Cargo.toml b/core/Cargo.toml index 1c2175ca38f..a4b38367213 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -4,37 +4,36 @@ version.workspace = true edition.workspace = true [dependencies] -async-trait = "0.1.50" -atomic_refcell = "0.1.13" -async-stream = "0.3" +async-trait = { workspace = true} bytes = "1.0" -futures01 = { package = "futures", version = "0.1.31" } -futures = { version = "0.3.4", features = ["compat"] } graph = { path = "../graph" } -# This dependency is temporary. The multiblockchain refactoring is not -# finished as long as this dependency exists -graph-chain-arweave = { path = "../chain/arweave" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-near = { path = "../chain/near" } -graph-chain-cosmos = { path = "../chain/cosmos" } -graph-chain-substreams = { path = "../chain/substreams" } -graph-chain-starknet = { path = "../chain/starknet" } -lazy_static = "1.2.0" -lru_time_cache = "0.11" -semver = "1.0.21" -serde = "1.0" -serde_json = "1.0" -serde_yaml = "0.9.21" +graph-runtime-wasm = { path = "../runtime/wasm" } +serde_yaml = { workspace = true } +tokio = { workspace = true } +tokio-retry = { workspace = true } # Switch to crates.io once tower 0.5 is released tower = { git = "https://github.com/tower-rs/tower.git", features = ["full"] } -graph-runtime-wasm = { path = "../runtime/wasm" } -cid = "0.11.0" +thiserror = { workspace = true } anyhow = "1.0" +# Dependencies related to Amp subgraphs +alloy.workspace = true +arrow.workspace = true +chrono.workspace = true +futures.workspace = true +indoc.workspace = true +itertools.workspace = true +parking_lot.workspace = true +prometheus.workspace = true +slog.workspace = true +strum.workspace = true +tokio-util.workspace = true + [dev-dependencies] tower-test = { git = "https://github.com/tower-rs/tower.git" } -ipfs-api-backend-hyper = "0.6" -ipfs-api = { version = "0.17.0", features = [ - "with-hyper-rustls", -], default-features = false } -uuid = { version = "1.6.1", features = ["v4"] } +wiremock = "0.6.5" + +[lints] +workspace = true diff --git a/core/graphman/Cargo.toml b/core/graphman/Cargo.toml new file mode 100644 index 00000000000..858cc1b9012 --- /dev/null +++ b/core/graphman/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "graphman" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +diesel = { workspace = true } +diesel-async = { workspace = true } +graph = { workspace = true } +graph-store-postgres = { workspace = true } +graphman-store = { workspace = true } +itertools = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } diff --git a/core/graphman/src/commands/deployment/info.rs b/core/graphman/src/commands/deployment/info.rs new file mode 100644 index 00000000000..155a821d715 --- /dev/null +++ b/core/graphman/src/commands/deployment/info.rs @@ -0,0 +1,82 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::anyhow; +use graph::blockchain::BlockPtr; +use graph::components::store::BlockNumber; +use graph::components::store::DeploymentId; +use graph::components::store::StatusStore; +use graph::data::subgraph::schema::SubgraphHealth; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::Store; +use itertools::Itertools; + +use crate::GraphmanError; +use crate::deployment::Deployment; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +#[derive(Clone, Debug)] +pub struct DeploymentStatus { + pub is_paused: Option, + pub is_synced: bool, + pub health: SubgraphHealth, + pub earliest_block_number: BlockNumber, + pub latest_block: Option, + pub chain_head_block: Option, +} + +pub async fn load_deployments( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, + version: &DeploymentVersionSelector, +) -> Result, GraphmanError> { + let mut primary_conn = primary_pool.get().await?; + + crate::deployment::load_deployments(&mut primary_conn, deployment, version).await +} + +pub async fn load_deployment_statuses( + store: Arc, + deployments: &[Deployment], +) -> Result, GraphmanError> { + use graph::data::subgraph::status::Filter; + + let deployment_ids = deployments + .iter() + .map(|deployment| DeploymentId::new(deployment.id)) + .collect_vec(); + + let deployment_statuses = store + .status(Filter::DeploymentIds(deployment_ids)) + .await? + .into_iter() + .map(|status| { + let id = status.id.0; + + let chain = status + .chains + .first() + .ok_or_else(|| { + GraphmanError::Store(anyhow!( + "deployment status has no chains on deployment '{id}'" + )) + })? + .to_owned(); + + Ok(( + id, + DeploymentStatus { + is_paused: status.paused, + is_synced: status.synced, + health: status.health, + earliest_block_number: chain.earliest_block_number.to_owned(), + latest_block: chain.latest_block.map(|x| x.to_ptr()), + chain_head_block: chain.chain_head_block.map(|x| x.to_ptr()), + }, + )) + }) + .collect::>()?; + + Ok(deployment_statuses) +} diff --git a/core/graphman/src/commands/deployment/mod.rs b/core/graphman/src/commands/deployment/mod.rs new file mode 100644 index 00000000000..4cac2277bbe --- /dev/null +++ b/core/graphman/src/commands/deployment/mod.rs @@ -0,0 +1,5 @@ +pub mod info; +pub mod pause; +pub mod reassign; +pub mod resume; +pub mod unassign; diff --git a/core/graphman/src/commands/deployment/pause.rs b/core/graphman/src/commands/deployment/pause.rs new file mode 100644 index 00000000000..e2405b5f68b --- /dev/null +++ b/core/graphman/src/commands/deployment/pause.rs @@ -0,0 +1,91 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::components::store::StoreEvent; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct ActiveDeployment { + locator: DeploymentLocator, + site: Site, +} + +#[derive(Debug, Error)] +pub enum PauseDeploymentError { + #[error("deployment '{0}' is already paused")] + AlreadyPaused(String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +impl ActiveDeployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } +} + +pub async fn load_active_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + let (_, is_paused) = catalog_conn + .assignment_status(&site) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("assignment status not found for '{locator}'")) + })?; + + if is_paused { + return Err(PauseDeploymentError::AlreadyPaused(locator.to_string())); + } + + Ok(ActiveDeployment { locator, site }) +} + +pub async fn pause_active_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + active_deployment: ActiveDeployment, +) -> Result<(), GraphmanError> { + let primary_conn = primary_pool.get_permitted().await?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let changes = catalog_conn.pause_subgraph(&active_deployment.site).await?; + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await?; + + Ok(()) +} diff --git a/core/graphman/src/commands/deployment/reassign.rs b/core/graphman/src/commands/deployment/reassign.rs new file mode 100644 index 00000000000..f4279946917 --- /dev/null +++ b/core/graphman/src/commands/deployment/reassign.rs @@ -0,0 +1,145 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::components::store::StoreEvent; +use graph::prelude::AssignmentChange; +use graph::prelude::NodeId; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct Deployment { + locator: DeploymentLocator, + site: Site, +} + +impl Deployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } + + pub async fn assigned_node( + &self, + primary_pool: ConnectionPool, + ) -> Result, GraphmanError> { + let primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + let node = catalog_conn + .assigned_node(&self.site) + .await + .map_err(GraphmanError::from)?; + Ok(node) + } +} + +#[derive(Debug, Error)] +pub enum ReassignDeploymentError { + #[error("deployment '{0}' is already assigned to '{1}'")] + AlreadyAssigned(String, String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +#[derive(Clone, Debug)] +pub enum ReassignResult { + Ok, + CompletedWithWarnings(Vec), +} + +pub async fn load_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + Ok(Deployment { locator, site }) +} + +pub async fn reassign_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + deployment: &Deployment, + node: &NodeId, + curr_node: Option, +) -> Result { + let primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + let changes: Vec = match &curr_node { + Some(curr) => { + if curr == node { + vec![] + } else { + catalog_conn + .reassign_subgraph(&deployment.site, node) + .await + .map_err(GraphmanError::from)? + } + } + None => catalog_conn + .assign_subgraph(&deployment.site, node) + .await + .map_err(GraphmanError::from)?, + }; + + if changes.is_empty() { + return Err(ReassignDeploymentError::AlreadyAssigned( + deployment.locator.to_string(), + node.to_string(), + )); + } + + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await + .map_err(GraphmanError::from)?; + + let mirror = catalog::Mirror::primary_only(primary_pool); + let count = mirror + .assignments(node) + .await + .map_err(GraphmanError::from)? + .len(); + if count == 1 { + let warning_msg = format!( + "This is the only deployment assigned to '{}'. Please make sure that the node ID is spelled correctly.", + node.as_str() + ); + Ok(ReassignResult::CompletedWithWarnings(vec![warning_msg])) + } else { + Ok(ReassignResult::Ok) + } +} diff --git a/core/graphman/src/commands/deployment/resume.rs b/core/graphman/src/commands/deployment/resume.rs new file mode 100644 index 00000000000..4fa0162f83e --- /dev/null +++ b/core/graphman/src/commands/deployment/resume.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::prelude::StoreEvent; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct PausedDeployment { + locator: DeploymentLocator, + site: Site, +} + +#[derive(Debug, Error)] +pub enum ResumeDeploymentError { + #[error("deployment '{0}' is not paused")] + NotPaused(String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +impl PausedDeployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } +} + +pub async fn load_paused_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + let (_, is_paused) = catalog_conn + .assignment_status(&site) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("assignment status not found for '{locator}'")) + })?; + + if !is_paused { + return Err(ResumeDeploymentError::NotPaused(locator.to_string())); + } + + Ok(PausedDeployment { locator, site }) +} + +pub async fn resume_paused_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + paused_deployment: PausedDeployment, +) -> Result<(), GraphmanError> { + let primary_conn = primary_pool.get_permitted().await?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let changes = catalog_conn + .resume_subgraph(&paused_deployment.site) + .await?; + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await?; + + Ok(()) +} diff --git a/core/graphman/src/commands/deployment/unassign.rs b/core/graphman/src/commands/deployment/unassign.rs new file mode 100644 index 00000000000..cef72e947f1 --- /dev/null +++ b/core/graphman/src/commands/deployment/unassign.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::components::store::StoreEvent; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct AssignedDeployment { + locator: DeploymentLocator, + site: Site, +} + +impl AssignedDeployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } +} + +#[derive(Debug, Error)] +pub enum UnassignDeploymentError { + #[error("deployment '{0}' is already unassigned")] + AlreadyUnassigned(String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +pub async fn load_assigned_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + match catalog_conn + .assigned_node(&site) + .await + .map_err(GraphmanError::from)? + { + Some(_) => Ok(AssignedDeployment { locator, site }), + None => Err(UnassignDeploymentError::AlreadyUnassigned( + locator.to_string(), + )), + } +} + +pub async fn unassign_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + deployment: AssignedDeployment, +) -> Result<(), GraphmanError> { + let primary_conn = primary_pool.get_permitted().await?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let changes = catalog_conn.unassign_subgraph(&deployment.site).await?; + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await?; + + Ok(()) +} diff --git a/core/graphman/src/commands/mod.rs b/core/graphman/src/commands/mod.rs new file mode 100644 index 00000000000..98629027b58 --- /dev/null +++ b/core/graphman/src/commands/mod.rs @@ -0,0 +1 @@ +pub mod deployment; diff --git a/core/graphman/src/deployment.rs b/core/graphman/src/deployment.rs new file mode 100644 index 00000000000..6538017edaa --- /dev/null +++ b/core/graphman/src/deployment.rs @@ -0,0 +1,157 @@ +use anyhow::anyhow; +use diesel::BoolExpressionMethods; +use diesel::ExpressionMethods; +use diesel::JoinOnDsl; +use diesel::NullableExpressionMethods; +use diesel::PgTextExpressionMethods; +use diesel::QueryDsl; +use diesel::Queryable; +use diesel::dsl::sql; +use diesel::sql_types::Text; +use diesel_async::RunQueryDsl; +use graph::components::store::DeploymentId; +use graph::components::store::DeploymentLocator; +use graph::data::subgraph::DeploymentHash; +use graph_store_postgres::AsyncPgConnection; +use graph_store_postgres::command_support::catalog; +use itertools::Itertools; + +use crate::GraphmanError; + +#[derive(Clone, Debug, Queryable)] +pub struct Deployment { + pub id: i32, + pub hash: String, + pub namespace: String, + pub name: String, + pub node_id: Option, + pub shard: String, + pub chain: String, + pub version_status: String, + pub is_active: bool, +} + +#[derive(Clone, Debug)] +pub enum DeploymentSelector { + Name(String), + Subgraph { hash: String, shard: Option }, + Schema(String), + All, +} + +#[derive(Clone, Debug)] +pub enum DeploymentVersionSelector { + Current, + Pending, + Used, + All, +} + +impl Deployment { + pub fn locator(&self) -> DeploymentLocator { + DeploymentLocator::new( + DeploymentId::new(self.id), + DeploymentHash::new(self.hash.clone()).unwrap(), + ) + } +} + +pub(crate) async fn load_deployments( + primary_conn: &mut AsyncPgConnection, + deployment: &DeploymentSelector, + version: &DeploymentVersionSelector, +) -> Result, GraphmanError> { + use catalog::deployment_schemas as ds; + use catalog::subgraph as sg; + use catalog::subgraph_deployment_assignment as sgda; + use catalog::subgraph_version as sgv; + + let mut query = ds::table + .inner_join(sgv::table.on(sgv::deployment.eq(ds::subgraph))) + .inner_join(sg::table.on(sgv::subgraph.eq(sg::id))) + .left_outer_join(sgda::table.on(sgda::id.eq(ds::id))) + .select(( + ds::id, + sgv::deployment, + ds::name, + sg::name, + sgda::node_id.nullable(), + ds::shard, + ds::network, + sql::( + "( + case + when subgraphs.subgraph.pending_version = subgraphs.subgraph_version.id + then 'pending' + when subgraphs.subgraph.current_version = subgraphs.subgraph_version.id + then 'current' + else + 'unused' + end + ) status", + ), + ds::active, + )) + .into_boxed(); + + match deployment { + DeploymentSelector::Name(name) => { + let pattern = format!("%{}%", name.replace("%", "")); + query = query.filter(sg::name.ilike(pattern)); + } + DeploymentSelector::Subgraph { hash, shard } => { + query = query.filter(ds::subgraph.eq(hash)); + + if let Some(shard) = shard { + query = query.filter(ds::shard.eq(shard)); + } + } + DeploymentSelector::Schema(name) => { + query = query.filter(ds::name.eq(name)); + } + DeploymentSelector::All => { + // No query changes required. + } + }; + + let current_version_filter = sg::current_version.eq(sgv::id.nullable()); + let pending_version_filter = sg::pending_version.eq(sgv::id.nullable()); + + match version { + DeploymentVersionSelector::Current => { + query = query.filter(current_version_filter); + } + DeploymentVersionSelector::Pending => { + query = query.filter(pending_version_filter); + } + DeploymentVersionSelector::Used => { + query = query.filter(current_version_filter.or(pending_version_filter)); + } + DeploymentVersionSelector::All => { + // No query changes required. + } + } + + query.load(primary_conn).await.map_err(Into::into) +} + +pub(crate) async fn load_deployment_locator( + primary_conn: &mut AsyncPgConnection, + deployment: &DeploymentSelector, + version: &DeploymentVersionSelector, +) -> Result { + let deployment_locator = load_deployments(primary_conn, deployment, version) + .await? + .into_iter() + .map(|deployment| deployment.locator()) + .unique() + .exactly_one() + .map_err(|err| { + let count = err.into_iter().count(); + GraphmanError::Store(anyhow!( + "expected exactly one deployment for '{deployment:?}', found {count}" + )) + })?; + + Ok(deployment_locator) +} diff --git a/core/graphman/src/error.rs b/core/graphman/src/error.rs new file mode 100644 index 00000000000..731b2574f0e --- /dev/null +++ b/core/graphman/src/error.rs @@ -0,0 +1,19 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum GraphmanError { + #[error("store error: {0:#}")] + Store(#[source] anyhow::Error), +} + +impl From for GraphmanError { + fn from(err: graph::components::store::StoreError) -> Self { + Self::Store(err.into()) + } +} + +impl From for GraphmanError { + fn from(err: diesel::result::Error) -> Self { + Self::Store(err.into()) + } +} diff --git a/core/graphman/src/execution_tracker.rs b/core/graphman/src/execution_tracker.rs new file mode 100644 index 00000000000..806d78defed --- /dev/null +++ b/core/graphman/src/execution_tracker.rs @@ -0,0 +1,86 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use graphman_store::ExecutionId; +use graphman_store::GraphmanStore; +use tokio::sync::Notify; + +/// The execution status is updated at this interval. +const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20); + +/// Used with long-running command executions to maintain their status as active. +pub struct GraphmanExecutionTracker { + id: ExecutionId, + heartbeat_stopper: Arc, + store: Arc, +} + +impl GraphmanExecutionTracker +where + S: GraphmanStore + Send + Sync + 'static, +{ + /// Creates a new execution tracker that spawns a separate background task that keeps + /// the execution active by periodically updating its status. + pub fn new(store: Arc, id: ExecutionId) -> Self { + let heartbeat_stopper = Arc::new(Notify::new()); + + let tracker = Self { + id, + store, + heartbeat_stopper, + }; + + tracker.spawn_heartbeat(); + tracker + } + + fn spawn_heartbeat(&self) { + let id = self.id; + let heartbeat_stopper = self.heartbeat_stopper.clone(); + let store = self.store.clone(); + + graph::spawn(async move { + store.mark_execution_as_running(id).await.unwrap(); + + let stop_heartbeat = heartbeat_stopper.notified(); + tokio::pin!(stop_heartbeat); + + loop { + tokio::select! { + biased; + + _ = &mut stop_heartbeat => { + break; + }, + + _ = tokio::time::sleep(DEFAULT_HEARTBEAT_INTERVAL) => { + store.mark_execution_as_running(id).await.unwrap(); + }, + } + } + }); + } + + /// Completes the execution with an error. + pub async fn track_failure(self, error_message: String) -> Result<()> { + self.heartbeat_stopper.notify_one(); + + self.store + .mark_execution_as_failed(self.id, error_message) + .await + } + + /// Completes the execution with a success. + pub async fn track_success(self) -> Result<()> { + self.heartbeat_stopper.notify_one(); + + self.store.mark_execution_as_succeeded(self.id).await + } +} + +impl Drop for GraphmanExecutionTracker { + fn drop(&mut self) { + self.heartbeat_stopper.notify_one(); + } +} diff --git a/core/graphman/src/lib.rs b/core/graphman/src/lib.rs new file mode 100644 index 00000000000..71f8e77a848 --- /dev/null +++ b/core/graphman/src/lib.rs @@ -0,0 +1,15 @@ +//! This crate contains graphman commands that can be executed via +//! the GraphQL API as well as via the CLI. +//! +//! Each command is broken into small execution steps to allow different interfaces to perform +//! some additional interface-specific operations between steps. An example of this is printing +//! intermediate information to the user in the CLI, or prompting for additional input. + +mod error; + +pub mod commands; +pub mod deployment; +pub mod execution_tracker; + +pub use self::error::GraphmanError; +pub use self::execution_tracker::GraphmanExecutionTracker; diff --git a/core/graphman_store/Cargo.toml b/core/graphman_store/Cargo.toml new file mode 100644 index 00000000000..fee9daff663 --- /dev/null +++ b/core/graphman_store/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "graphman-store" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +chrono = { workspace = true } +diesel = { workspace = true } +strum = { workspace = true } diff --git a/core/graphman_store/src/lib.rs b/core/graphman_store/src/lib.rs new file mode 100644 index 00000000000..8b1965df7cd --- /dev/null +++ b/core/graphman_store/src/lib.rs @@ -0,0 +1,129 @@ +//! This crate allows graphman commands to store data in a persistent storage. +//! +//! Note: The trait is extracted as a separate crate to avoid cyclic dependencies between graphman +//! commands and store implementations. + +use anyhow::Result; +use async_trait::async_trait; +use chrono::DateTime; +use chrono::Utc; +use diesel::AsExpression; +use diesel::FromSqlRow; +use diesel::Queryable; +use diesel::deserialize::FromSql; +use diesel::pg::Pg; +use diesel::pg::PgValue; +use diesel::serialize::Output; +use diesel::serialize::ToSql; +use diesel::sql_types::BigSerial; +use diesel::sql_types::Varchar; +use strum::Display; +use strum::EnumString; +use strum::IntoStaticStr; + +/// Describes all the capabilities that graphman commands need from a persistent storage. +/// +/// The primary use case for this is background execution of commands. +#[async_trait] +pub trait GraphmanStore { + /// Creates a new pending execution of the specified type. + /// The implementation is expected to manage execution IDs and return unique IDs on each call. + /// + /// Creating a new execution does not mean that a command is actually running or will run. + async fn new_execution(&self, kind: CommandKind) -> Result; + + /// Returns all stored execution data. + async fn load_execution(&self, id: ExecutionId) -> Result; + + /// When an execution begins to make progress, this method is used to update its status. + /// + /// For long-running commands, it is expected that this method will be called at some interval + /// to show that the execution is still making progress. + /// + /// The implementation is expected to not allow updating the status of completed executions. + async fn mark_execution_as_running(&self, id: ExecutionId) -> Result<()>; + + /// This is a finalizing operation and is expected to be called only once, + /// when an execution fails. + /// + /// The implementation is not expected to prevent overriding the final state of an execution. + async fn mark_execution_as_failed(&self, id: ExecutionId, error_message: String) -> Result<()>; + + /// This is a finalizing operation and is expected to be called only once, + /// when an execution succeeds. + /// + /// The implementation is not expected to prevent overriding the final state of an execution. + async fn mark_execution_as_succeeded(&self, id: ExecutionId) -> Result<()>; +} + +/// Data stored about a command execution. +#[derive(Clone, Debug, Queryable)] +pub struct Execution { + pub id: ExecutionId, + pub kind: CommandKind, + pub status: ExecutionStatus, + pub error_message: Option, + pub created_at: DateTime, + pub updated_at: Option>, + pub completed_at: Option>, +} + +/// A unique ID of a command execution. +#[derive(Clone, Copy, Debug, AsExpression, FromSqlRow)] +#[diesel(sql_type = BigSerial)] +pub struct ExecutionId(pub i64); + +/// Types of commands that can store data about their execution. +#[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Display, IntoStaticStr, EnumString)] +#[diesel(sql_type = Varchar)] +#[strum(serialize_all = "snake_case")] +pub enum CommandKind { + RestartDeployment, +} + +/// All possible states of a command execution. +#[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Display, IntoStaticStr, EnumString)] +#[diesel(sql_type = Varchar)] +#[strum(serialize_all = "snake_case")] +pub enum ExecutionStatus { + Initializing, + Running, + Failed, + Succeeded, +} + +impl FromSql for ExecutionId { + fn from_sql(bytes: PgValue) -> diesel::deserialize::Result { + Ok(ExecutionId(i64::from_sql(bytes)?)) + } +} + +impl ToSql for ExecutionId { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> diesel::serialize::Result { + >::to_sql(&self.0, &mut out.reborrow()) + } +} + +impl FromSql for CommandKind { + fn from_sql(bytes: PgValue) -> diesel::deserialize::Result { + Ok(std::str::from_utf8(bytes.as_bytes())?.parse()?) + } +} + +impl ToSql for CommandKind { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> diesel::serialize::Result { + >::to_sql(self.into(), &mut out.reborrow()) + } +} + +impl FromSql for ExecutionStatus { + fn from_sql(bytes: PgValue) -> diesel::deserialize::Result { + Ok(std::str::from_utf8(bytes.as_bytes())?.parse()?) + } +} + +impl ToSql for ExecutionStatus { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> diesel::serialize::Result { + >::to_sql(self.into(), &mut out.reborrow()) + } +} diff --git a/core/src/amp_subgraph/manager.rs b/core/src/amp_subgraph/manager.rs new file mode 100644 index 00000000000..61ec421ca17 --- /dev/null +++ b/core/src/amp_subgraph/manager.rs @@ -0,0 +1,171 @@ +use std::sync::Arc; + +use alloy::primitives::BlockNumber; +use anyhow::Context; +use async_trait::async_trait; +use graph::{ + amp, + components::{ + link_resolver::{LinkResolver, LinkResolverContext}, + metrics::MetricsRegistry, + store::{DeploymentLocator, SubgraphStore}, + subgraph::SubgraphInstanceManager, + }, + env::EnvVars, + log::factory::LoggerFactory, + prelude::CheapClone, +}; +use slog::{debug, error}; +use tokio_util::sync::CancellationToken; + +use super::{Metrics, Monitor, runner}; + +/// Manages Amp subgraph runner futures. +/// +/// Creates and schedules Amp subgraph runner futures for execution on demand. +/// Also handles stopping previously started Amp subgraph runners. +pub struct Manager { + logger_factory: LoggerFactory, + metrics_registry: Arc, + env_vars: Arc, + monitor: Monitor, + subgraph_store: Arc, + link_resolver: Arc, + amp_client: Arc, +} + +impl Manager +where + SS: SubgraphStore, + NC: amp::Client, +{ + /// Creates a new Amp subgraph manager. + pub fn new( + logger_factory: &LoggerFactory, + metrics_registry: Arc, + env_vars: Arc, + cancel_token: &CancellationToken, + subgraph_store: Arc, + link_resolver: Arc, + amp_client: Arc, + ) -> Self { + let logger = logger_factory.component_logger("AmpSubgraphManager", None); + let logger_factory = logger_factory.with_parent(logger); + + let monitor = Monitor::new(&logger_factory, cancel_token); + + Self { + logger_factory, + metrics_registry, + env_vars, + monitor, + subgraph_store, + link_resolver, + amp_client, + } + } +} + +#[async_trait] +impl SubgraphInstanceManager for Manager +where + SS: SubgraphStore, + NC: amp::Client + Send + Sync + 'static, +{ + async fn start_subgraph( + self: Arc, + deployment: DeploymentLocator, + stop_block: Option, + ) { + let manager = self.cheap_clone(); + + self.monitor.start( + deployment.cheap_clone(), + Box::new(move |cancel_token| { + Box::pin(async move { + let logger = manager.logger_factory.subgraph_logger(&deployment); + + let store = manager + .subgraph_store + .cheap_clone() + .writable(logger.cheap_clone(), deployment.id, Vec::new().into()) + .await + .context("failed to create writable store")?; + + let metrics = Metrics::new( + &logger, + manager.metrics_registry.cheap_clone(), + store.cheap_clone(), + deployment.hash.cheap_clone(), + ); + + let link_resolver = manager + .link_resolver + .for_manifest(&deployment.hash.to_string()) + .context("failed to create link resolver")?; + + let manifest_bytes = link_resolver + .cat( + &LinkResolverContext::new(&deployment.hash, &logger), + &deployment.hash.to_ipfs_link(), + ) + .await + .context("failed to load subgraph manifest")?; + + let raw_manifest = serde_yaml::from_slice(&manifest_bytes) + .context("failed to parse subgraph manifest")?; + + let mut manifest = amp::Manifest::resolve::( + &logger, + manager.link_resolver.cheap_clone(), + manager.amp_client.cheap_clone(), + manager.env_vars.max_spec_version.cheap_clone(), + deployment.hash.cheap_clone(), + raw_manifest, + ) + .await?; + + if let Some(stop_block) = stop_block { + for data_source in manifest.data_sources.iter_mut() { + data_source.source.end_block = stop_block as BlockNumber; + } + } + + store + .start_subgraph_deployment(&logger) + .await + .context("failed to start subgraph deployment")?; + + let runner_context = runner::Context::new( + &logger, + &manager.env_vars.amp, + manager.amp_client.cheap_clone(), + store, + deployment.hash.cheap_clone(), + manifest, + metrics, + ); + + let runner_result = runner::new_runner(runner_context, cancel_token).await; + + match manager.subgraph_store.stop_subgraph(&deployment).await { + Ok(()) => { + debug!(logger, "Subgraph writer stopped"); + } + Err(e) => { + error!(logger, "Failed to stop subgraph writer"; + "e" => ?e + ); + } + } + + runner_result + }) + }), + ); + } + + async fn stop_subgraph(&self, deployment: DeploymentLocator) { + self.monitor.stop(deployment); + } +} diff --git a/core/src/amp_subgraph/metrics.rs b/core/src/amp_subgraph/metrics.rs new file mode 100644 index 00000000000..7088cf9a48f --- /dev/null +++ b/core/src/amp_subgraph/metrics.rs @@ -0,0 +1,267 @@ +use std::{sync::Arc, time::Duration}; + +use alloy::primitives::BlockNumber; +use graph::{ + cheap_clone::CheapClone, + components::{ + metrics::{MetricsRegistry, stopwatch::StopwatchMetrics}, + store::WritableStore, + }, + prelude::DeploymentHash, +}; +use indoc::indoc; +use prometheus::{IntCounter, IntGauge}; +use slog::Logger; + +/// Contains metrics specific to a deployment. +pub(super) struct Metrics { + pub(super) deployment_status: DeploymentStatus, + pub(super) deployment_head: DeploymentHead, + pub(super) deployment_target: DeploymentTarget, + pub(super) deployment_synced: DeploymentSynced, + pub(super) indexing_duration: IndexingDuration, + pub(super) blocks_processed: BlocksProcessed, + pub(super) stopwatch: StopwatchMetrics, +} + +impl Metrics { + /// Creates new deployment specific metrics. + pub(super) fn new( + logger: &Logger, + metrics_registry: Arc, + store: Arc, + deployment: DeploymentHash, + ) -> Self { + let stopwatch = StopwatchMetrics::new( + logger.cheap_clone(), + deployment.cheap_clone(), + "process", + metrics_registry.cheap_clone(), + store.shard().to_string(), + ); + + let const_labels = [ + ("deployment", deployment.to_string()), + ("shard", store.shard().to_string()), + ]; + + Self { + deployment_status: DeploymentStatus::new(&metrics_registry, const_labels.clone()), + deployment_head: DeploymentHead::new(&metrics_registry, const_labels.clone()), + deployment_target: DeploymentTarget::new(&metrics_registry, const_labels.clone()), + deployment_synced: DeploymentSynced::new(&metrics_registry, const_labels.clone()), + indexing_duration: IndexingDuration::new(&metrics_registry, const_labels.clone()), + blocks_processed: BlocksProcessed::new(&metrics_registry, const_labels.clone()), + stopwatch, + } + } +} + +/// Reports the current indexing status of a deployment. +pub(super) struct DeploymentStatus(IntGauge); + +impl DeploymentStatus { + const STATUS_STARTING: i64 = 1; + const STATUS_RUNNING: i64 = 2; + const STATUS_STOPPED: i64 = 3; + const STATUS_FAILED: i64 = 4; + + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_status", + indoc!( + " + Indicates the current indexing status of a deployment. + Possible values: + 1 - graph-node is preparing to start indexing; + 2 - deployment is being indexed; + 3 - indexing is stopped by request; + 4 - indexing failed; + " + ), + const_labels, + ) + .expect("failed to register `deployment_status` gauge"); + + Self(int_gauge) + } + + /// Records that the graph-node is preparing to start indexing. + pub fn starting(&self) { + self.0.set(Self::STATUS_STARTING); + } + + /// Records that the deployment is being indexed. + pub fn running(&self) { + self.0.set(Self::STATUS_RUNNING); + } + + /// Records that the indexing stopped by request. + pub fn stopped(&self) { + self.0.set(Self::STATUS_STOPPED); + } + + /// Records that the indexing failed. + pub fn failed(&self) { + self.0.set(Self::STATUS_FAILED); + } +} + +/// Tracks the most recent block number processed by a deployment. +pub(super) struct DeploymentHead(IntGauge); + +impl DeploymentHead { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_head", + "Tracks the most recent block number processed by a deployment", + const_labels + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + // TODO: Pass the network from the Amp manifest + .chain([("network".to_string(), "".to_string())]), + ) + .expect("failed to register `deployment_head` gauge"); + + Self(int_gauge) + } + + /// Updates the most recent block number processed by this deployment. + pub(super) fn update(&self, new_most_recent_block_number: BlockNumber) { + self.0.set( + i64::try_from(new_most_recent_block_number) + .expect("new most recent block number does not fit into `i64`"), + ); + } +} + +/// Tracks the maximum block number currently available for indexing within a deployment. +pub(super) struct DeploymentTarget(IntGauge); + +impl DeploymentTarget { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_target", + "Tracks the maximum block number currently available for indexing within a deployment", + const_labels, + ) + .expect("failed to register `amp_deployment_target` gauge"); + + Self(int_gauge) + } + + /// Updates the target block number of this deployment. + pub(super) fn update(&self, new_target_block_number: BlockNumber) { + self.0.set( + i64::try_from(new_target_block_number) + .expect("new target block number does not fit into `i64`"), + ); + } +} + +/// Indicates whether a deployment has reached the chain head or the end block since it was deployed. +pub(super) struct DeploymentSynced(IntGauge); + +impl DeploymentSynced { + const NOT_SYNCED: i64 = 0; + const SYNCED: i64 = 1; + + pub fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_synced", + indoc!( + " + Indicates whether a deployment has reached the chain head or the end block since it was deployed. + Possible values: + 0 - deployment is not synced; + 1 - deployment is synced; + " + ), + const_labels, + ) + .expect("failed to register `deployment_synced` gauge"); + + Self(int_gauge) + } + + /// Records the current sync status of this deployment. + pub fn record(&self, synced: bool) { + self.0.set(if synced { + Self::SYNCED + } else { + Self::NOT_SYNCED + }); + } +} + +/// Tracks the total duration in seconds of deployment indexing. +#[derive(Clone)] +pub(super) struct IndexingDuration(IntCounter); + +impl IndexingDuration { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_counter = metrics_registry + .new_int_counter( + "deployment_indexing_duration_seconds", + "Tracks the total duration in seconds of deployment indexing", + const_labels, + ) + .expect("failed to register `amp_deployment_indexing_duration_seconds` counter"); + + Self(int_counter) + } + + /// Records a new indexing duration of this deployment. + pub(super) fn record(&self, duration: Duration) { + self.0.inc_by(duration.as_secs()) + } +} + +/// Tracks the total number of blocks processed by a deployment. +pub(super) struct BlocksProcessed(IntCounter); + +impl BlocksProcessed { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_counter = metrics_registry + .new_int_counter( + "deployment_blocks_processed_count", + "Tracks the total number of blocks processed by a deployment", + const_labels, + ) + .expect("failed to register `deployment_blocks_processed_count` counter"); + + Self(int_counter) + } + + /// Records a new processed block. + pub(super) fn record_one(&self) { + self.record(1); + } + + /// Records the new processed blocks. + pub(super) fn record(&self, number_of_blocks_processed: usize) { + self.0.inc_by(number_of_blocks_processed as u64); + } +} diff --git a/core/src/amp_subgraph/mod.rs b/core/src/amp_subgraph/mod.rs new file mode 100644 index 00000000000..3d3846742aa --- /dev/null +++ b/core/src/amp_subgraph/mod.rs @@ -0,0 +1,8 @@ +mod manager; +mod metrics; +mod monitor; +mod runner; + +use self::{metrics::Metrics, monitor::Monitor}; + +pub use self::manager::Manager; diff --git a/core/src/amp_subgraph/monitor.rs b/core/src/amp_subgraph/monitor.rs new file mode 100644 index 00000000000..314f464c0d1 --- /dev/null +++ b/core/src/amp_subgraph/monitor.rs @@ -0,0 +1,563 @@ +//! This module is responsible for executing subgraph runner futures. +//! +//! # Terminology used in this module +//! +//! `active subgraph` - A subgraph that was started and is still kept in memory in the list of started subgraphs. +//! `running subgraph` - A subgraph that has an instance that is making progress or stopping. +//! `subgraph instance` - A background task that executes the subgraph runner future. + +use std::{ + collections::{HashMap, hash_map::Entry}, + fmt, + sync::{ + Arc, + atomic::{AtomicU32, Ordering::SeqCst}, + }, + time::Duration, +}; + +use anyhow::Result; +use futures::future::BoxFuture; +use graph::{ + cheap_clone::CheapClone, components::store::DeploymentLocator, log::factory::LoggerFactory, +}; +use slog::{Logger, debug, error, info, warn}; +use tokio::{ + sync::mpsc::{self, error::SendError}, + task::JoinHandle, + time::timeout, +}; +use tokio_util::sync::CancellationToken; + +/// Represents the maximum amount of time a subgraph instance is allowed to run +/// after it receives a cancel signal. +/// +/// If a subgraph instance does not complete its execution in this amount of time +/// it is considered unresponsive and is aborted. +const SUBGRAPH_INSTANCE_GRACE_PERIOD: Duration = { + if cfg!(test) { + Duration::from_millis(300) + } else if cfg!(debug_assertions) { + Duration::from_secs(30) + } else { + Duration::from_secs(300) + } +}; + +/// Represents the subgraph runner future. +/// +/// This is the future that performs the subgraph indexing. +/// It is expected to return only on deterministic failures or when indexing is completed. +/// All retry functionality must be handled internally by this future. +pub(super) type BoxRunner = + Box BoxFuture<'static, Result<()>> + Send + 'static>; + +/// Manages the lifecycle of subgraph runners. +/// +/// Ensures that there is at most one subgraph instance running +/// for any subgraph deployment at any point in time. +/// Handles starting, stopping and restarting subgraphs. +pub(super) struct Monitor { + logger_factory: Arc, + + /// Every subgraph instance is assigned a cancel token derived from this token. + /// + /// This means that the `Monitor` can send cancel signals to all subgraph instances at once, + /// and to each subgraph instance individually. + cancel_token: CancellationToken, + + /// The channel that is used to send subgraph commands. + /// + /// Every subgraph start and stop request results in a command that is sent to the + /// background task that manages the subgraph instances. + command_tx: mpsc::UnboundedSender, + + /// When a subgraph starts it is assigned a sequential ID. + /// The ID is then kept in memory in the list of active subgraphs. + /// + /// When the subgraph completes execution it should be removed from the + /// list of active subgraphs, so that it can be restarted. + /// + /// This ID is required to be able to check if the active subgraph + /// is the same subgraph instance that was stopped. + /// + /// If the IDs do not match, it means that the subgraph was force restarted, + /// ignoring the state of the previous subgraph instance, or that the subgraph + /// was restarted after the previous subgraph instance completed its execution + /// but before the remove request was processed. + subgraph_instance_id: Arc, +} + +impl Monitor { + /// Creates a new subgraph monitor. + /// + /// Spawns a background task that manages the subgraph start and stop requests. + /// + /// A new cancel token is derived from the `cancel_token` and only the derived token is used by the + /// subgraph monitor and its background task. + pub(super) fn new(logger_factory: &LoggerFactory, cancel_token: &CancellationToken) -> Self { + let logger = logger_factory.component_logger("AmpSubgraphMonitor", None); + let logger_factory = Arc::new(logger_factory.with_parent(logger)); + + // A derived token makes sure it is not possible to accidentally cancel the parent token + let cancel_token = cancel_token.child_token(); + + // It is safe to use an unbounded channel here, because it's pretty much unrealistic that the + // command processor will fall behind so much that the channel buffer will take up all the memory. + // The command processor is non-blocking and delegates long-running processes to detached tasks. + let (command_tx, command_rx) = mpsc::unbounded_channel::(); + + tokio::spawn(Self::command_processor( + logger_factory.cheap_clone(), + cancel_token.cheap_clone(), + command_tx.clone(), + command_rx, + )); + + Self { + logger_factory, + cancel_token, + command_tx, + subgraph_instance_id: Arc::new(AtomicU32::new(0)), + } + } + + /// Starts a subgraph. + /// + /// Sends a subgraph start request to this subgraph monitor that + /// eventually starts the subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active, it starts when the request is processed + /// - If the subgraph is active, it stops, and then restarts + /// - Ensures that there is only one subgraph instance for this subgraph deployment + /// - Multiple consecutive calls in a short time period force restart the subgraph, + /// aborting the active subgraph instance + pub(super) fn start(&self, deployment: DeploymentLocator, runner: BoxRunner) { + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("method" => "start")); + + info!(logger, "Starting subgraph"); + log_send_error( + &logger, + self.command_tx.send(Command::Start { + id: self.subgraph_instance_id.fetch_add(1, SeqCst), + deployment, + runner, + }), + ); + } + + /// Stops the subgraph. + /// + /// Sends a subgraph stop request to this subgraph monitor that + /// eventually stops the subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active does nothing + /// - If the subgraph is active, sends a cancel signal that gracefully stops the subgraph + /// - If the subgraph fails to stop after an extended period of time it aborts + pub(super) fn stop(&self, deployment: DeploymentLocator) { + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("method" => "stop")); + + info!(logger, "Stopping subgraph"); + log_send_error(&logger, self.command_tx.send(Command::Stop { deployment })); + } + + /// Processes commands sent through the command channel. + /// + /// Tracks active subgraphs and keeps a list of pending start commands. + /// Pending start commands are start commands that execute after the related subgraph stops. + async fn command_processor( + logger_factory: Arc, + cancel_token: CancellationToken, + command_tx: mpsc::UnboundedSender, + mut command_rx: mpsc::UnboundedReceiver, + ) { + let logger = logger_factory.component_logger("CommandProcessor", None); + let mut subgraph_instances: HashMap = HashMap::new(); + let mut pending_start_commands: HashMap = HashMap::new(); + + loop { + tokio::select! { + Some(command) = command_rx.recv() => { + match &command { + Command::Start { .. } => { + Self::process_start_command( + &logger_factory, + &cancel_token, + &mut subgraph_instances, + &mut pending_start_commands, + &command_tx, + command + ); + }, + Command::Stop { .. } => { + Self::process_stop_command( + &logger_factory, + &mut subgraph_instances, + &mut pending_start_commands, + command + ); + }, + Command::Clear { .. } => { + Self::process_clear_command( + &logger_factory, + &mut subgraph_instances, + &mut pending_start_commands, + &command_tx, + command + ); + }, + } + }, + _ = cancel_token.cancelled() => { + debug!(logger, "Stopping command processor"); + + // All active subgraphs will shutdown gracefully + // because their cancel tokens are derived from this cancelled token. + return; + } + } + } + } + + /// Starts a subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active, it starts right away + /// - If the subgraph is active, a cancel signal is sent to the active subgraph instance + /// and this start request is stored in the list of pending start commands + /// - If the subgraph is active and there is already a pending start command, + /// the active subgraph instance aborts, and the subgraph force restarts right away + /// - If the subgraph is active, but its instance is not actually running, + /// the subgraph starts right away + fn process_start_command( + logger_factory: &LoggerFactory, + cancel_token: &CancellationToken, + subgraph_instances: &mut HashMap, + pending_start_commands: &mut HashMap, + command_tx: &mpsc::UnboundedSender, + command: Command, + ) { + let Command::Start { + id, + deployment, + runner, + } = command + else { + unreachable!(); + }; + + let logger = logger_factory.subgraph_logger(&deployment); + let command_logger = logger.new(slog::o!("command" => "start")); + + let cancel_token = cancel_token.child_token(); + let pending_start_command = pending_start_commands.remove(&deployment); + + match subgraph_instances.entry(deployment.cheap_clone()) { + Entry::Vacant(entry) => { + debug!(command_logger, "Subgraph is not active, starting"); + + let subgraph_instance = Self::start_subgraph( + logger, + cancel_token, + id, + deployment, + runner, + command_tx.clone(), + ); + + entry.insert(subgraph_instance); + } + Entry::Occupied(mut entry) => { + let subgraph_instance = entry.get_mut(); + subgraph_instance.cancel_token.cancel(); + + if pending_start_command.is_some() { + debug!(command_logger, "Subgraph is active, force restarting"); + + subgraph_instance.handle.abort(); + + *subgraph_instance = Self::start_subgraph( + logger, + cancel_token, + id, + deployment, + runner, + command_tx.clone(), + ); + + return; + } + + if subgraph_instance.handle.is_finished() { + debug!(command_logger, "Subgraph is not running, starting"); + + *subgraph_instance = Self::start_subgraph( + logger, + cancel_token, + id, + deployment, + runner, + command_tx.clone(), + ); + + return; + } + + debug!(command_logger, "Gracefully restarting subgraph"); + + pending_start_commands.insert( + deployment.cheap_clone(), + Command::Start { + id, + deployment, + runner, + }, + ); + } + } + } + + /// Stops a subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active, does nothing + /// - If the subgraph is active, sends a cancel signal to the active subgraph instance + fn process_stop_command( + logger_factory: &LoggerFactory, + subgraph_instances: &mut HashMap, + pending_start_commands: &mut HashMap, + command: Command, + ) { + let Command::Stop { deployment } = command else { + unreachable!(); + }; + + let logger = logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("command" => "stop")); + + if let Some(subgraph_instance) = subgraph_instances.get(&deployment) { + debug!(logger, "Sending cancel signal"); + subgraph_instance.cancel_token.cancel(); + } else { + debug!(logger, "Subgraph is not active"); + } + + pending_start_commands.remove(&deployment); + } + + /// Removes a subgraph from the list of active subgraphs allowing the subgraph to be restarted. + fn process_clear_command( + logger_factory: &LoggerFactory, + subgraph_instances: &mut HashMap, + pending_start_commands: &mut HashMap, + command_tx: &mpsc::UnboundedSender, + command: Command, + ) { + let Command::Clear { id, deployment } = command else { + unreachable!(); + }; + + let logger = logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("command" => "clear")); + + match subgraph_instances.get(&deployment) { + Some(subgraph_instance) if subgraph_instance.id == id => { + debug!(logger, "Removing active subgraph"); + subgraph_instances.remove(&deployment); + } + Some(_subgraph_instance) => { + debug!(logger, "Active subgraph does not need to be removed"); + return; + } + None => { + debug!(logger, "Subgraph is not active"); + } + } + + if let Some(pending_start_command) = pending_start_commands.remove(&deployment) { + debug!(logger, "Resending a pending start command"); + log_send_error(&logger, command_tx.send(pending_start_command)); + } + } + + /// Spawns a background task that executes the subgraph runner future. + /// + /// An additional background task is spawned to handle the graceful shutdown of the subgraph runner, + /// and to ensure correct behaviour even if the subgraph runner panics. + fn start_subgraph( + logger: Logger, + cancel_token: CancellationToken, + id: u32, + deployment: DeploymentLocator, + runner: BoxRunner, + command_tx: mpsc::UnboundedSender, + ) -> SubgraphInstance { + let mut runner_handle = tokio::spawn({ + let logger = logger.new(slog::o!("process" => "subgraph_runner")); + let cancel_token = cancel_token.cheap_clone(); + + async move { + info!(logger, "Subgraph started"); + + match runner(cancel_token).await { + Ok(()) => { + info!(logger, "Subgraph stopped"); + } + Err(e) => { + error!(logger, "Subgraph failed"; + "error" => ?e + ); + } + } + } + }); + + let supervisor_handle = tokio::spawn({ + let logger = logger.new(slog::o!("process" => "subgraph_supervisor")); + let cancel_token = cancel_token.cheap_clone(); + + fn handle_runner_result(logger: &Logger, result: Result<(), tokio::task::JoinError>) { + match result { + Ok(()) => { + debug!(logger, "Subgraph completed execution"); + } + Err(e) if e.is_panic() => { + error!(logger, "Subgraph panicked"; + "error" => ?e + ); + } + Err(e) => { + error!(logger, "Subgraph failed"; + "error" => ?e + ); + } + } + } + + async move { + debug!(logger, "Subgraph supervisor started"); + + tokio::select! { + _ = cancel_token.cancelled() => { + debug!(logger, "Received cancel signal, waiting for subgraph to stop"); + + match timeout(SUBGRAPH_INSTANCE_GRACE_PERIOD, &mut runner_handle).await { + Ok(result) => { + handle_runner_result(&logger, result); + }, + Err(_) => { + warn!(logger, "Subgraph did not stop after grace period, aborting"); + + runner_handle.abort(); + let _ = runner_handle.await; + + warn!(logger, "Subgraph aborted"); + } + } + }, + result = &mut runner_handle => { + handle_runner_result(&logger, result); + cancel_token.cancel(); + } + } + + debug!(logger, "Sending clear command"); + log_send_error(&logger, command_tx.send(Command::Clear { id, deployment })); + } + }); + + SubgraphInstance { + id, + handle: supervisor_handle, + cancel_token, + } + } +} + +impl Drop for Monitor { + fn drop(&mut self) { + // Send cancel signals to all active subgraphs so that they don't remain without an associated monitor + self.cancel_token.cancel(); + } +} + +/// Represents a background task that executes the subgraph runner future. +struct SubgraphInstance { + id: u32, + handle: JoinHandle<()>, + cancel_token: CancellationToken, +} + +/// Every command used by the subgraph monitor. +enum Command { + /// A request to start executing the subgraph runner future. + Start { + id: u32, + deployment: DeploymentLocator, + runner: BoxRunner, + }, + + /// A request to stop executing the subgraph runner future. + Stop { deployment: DeploymentLocator }, + + /// A request to remove the subgraph from the list of active subgraphs. + Clear { + id: u32, + deployment: DeploymentLocator, + }, +} + +impl fmt::Debug for Command { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Start { + id, + deployment, + runner: _, + } => f + .debug_struct("Start") + .field("id", id) + .field("deployment", deployment) + .finish_non_exhaustive(), + Self::Stop { deployment } => f + .debug_struct("Stop") + .field("deployment", deployment) + .finish(), + Self::Clear { id, deployment } => f + .debug_struct("Clear") + .field("id", id) + .field("deployment", deployment) + .finish(), + } + } +} + +fn log_send_error(logger: &Logger, result: Result<(), SendError>) { + match result { + Ok(()) => { + // No need to log anything + } + + // This should only happen if the parent cancel token of the subgraph monitor was cancelled + Err(e) => { + error!(logger, "Failed to send command"; + "command" => ?e.0, + "error" => ?e + ); + } + } +} diff --git a/core/src/amp_subgraph/runner/compat.rs b/core/src/amp_subgraph/runner/compat.rs new file mode 100644 index 00000000000..f7152965cf3 --- /dev/null +++ b/core/src/amp_subgraph/runner/compat.rs @@ -0,0 +1,52 @@ +//! This is a temporary compatibility module until the graph-node is fully migrated to `alloy`. + +use alloy::primitives::{BlockHash, BlockNumber}; +use chrono::{DateTime, Utc}; + +mod legacy { + pub(super) use graph::{ + blockchain::{BlockHash, BlockPtr, BlockTime}, + components::store::BlockNumber, + data::store::scalar::Timestamp, + }; +} + +pub(super) trait Compat { + fn compat(&self) -> T; +} + +impl Compat for BlockNumber { + fn compat(&self) -> legacy::BlockNumber { + (*self).try_into().unwrap() + } +} + +impl Compat for legacy::BlockNumber { + fn compat(&self) -> BlockNumber { + (*self).try_into().unwrap() + } +} + +impl Compat for BlockHash { + fn compat(&self) -> legacy::BlockHash { + legacy::BlockHash(self.0.into()) + } +} + +impl Compat for legacy::BlockHash { + fn compat(&self) -> BlockHash { + BlockHash::from_slice(&self.0) + } +} + +impl Compat for DateTime { + fn compat(&self) -> legacy::BlockTime { + legacy::Timestamp(*self).into() + } +} + +impl Compat for (BlockNumber, BlockHash) { + fn compat(&self) -> legacy::BlockPtr { + legacy::BlockPtr::new(self.1.compat(), self.0.compat()) + } +} diff --git a/core/src/amp_subgraph/runner/context.rs b/core/src/amp_subgraph/runner/context.rs new file mode 100644 index 00000000000..d8b5b7b638e --- /dev/null +++ b/core/src/amp_subgraph/runner/context.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use alloy::primitives::{BlockHash, BlockNumber}; +use graph::{ + amp::{Codec, Manifest, log::Logger as _}, + cheap_clone::CheapClone, + components::store::WritableStore, + data::subgraph::DeploymentHash, + env::AmpEnv, + util::backoff::ExponentialBackoff, +}; +use slog::Logger; + +use super::Compat; +use crate::amp_subgraph::Metrics; + +pub(in super::super) struct Context { + pub(super) logger: Logger, + pub(super) client: Arc, + pub(super) store: Arc, + pub(super) buffer_size: usize, + pub(super) block_range: usize, + pub(super) backoff: ExponentialBackoff, + pub(super) deployment: DeploymentHash, + pub(super) manifest: Manifest, + pub(super) metrics: Metrics, + pub(super) codec: Codec, +} + +impl Context { + pub(in super::super) fn new( + logger: &Logger, + env: &AmpEnv, + client: Arc, + store: Arc, + deployment: DeploymentHash, + manifest: Manifest, + metrics: Metrics, + ) -> Self { + let logger = logger.component("AmpSubgraphRunner"); + let backoff = ExponentialBackoff::new(env.query_retry_min_delay, env.query_retry_max_delay); + let codec = Codec::new(manifest.schema.cheap_clone()); + + Self { + logger, + client, + store, + buffer_size: env.buffer_size, + block_range: env.block_range, + backoff, + deployment, + manifest, + metrics, + codec, + } + } + + pub(super) fn indexing_completed(&self) -> bool { + let Some(last_synced_block) = self.latest_synced_block() else { + return false; + }; + + self.manifest + .data_sources + .iter() + .all(|data_source| last_synced_block >= data_source.source.end_block) + } + + pub(super) fn latest_synced_block(&self) -> Option { + self.latest_synced_block_ptr() + .map(|(block_number, _)| block_number) + } + + pub(super) fn latest_synced_block_ptr(&self) -> Option<(BlockNumber, BlockHash)> { + self.store + .block_ptr() + .map(|block_ptr| (block_ptr.number.compat(), block_ptr.hash.compat())) + } + + pub(super) fn start_block(&self) -> BlockNumber { + self.manifest + .data_sources + .iter() + .map(|data_source| data_source.source.start_block) + .min() + .unwrap() + } + + pub(super) fn end_block(&self) -> BlockNumber { + self.manifest + .data_sources + .iter() + .map(|data_source| data_source.source.end_block) + .max() + .unwrap() + } +} diff --git a/core/src/amp_subgraph/runner/data_processing.rs b/core/src/amp_subgraph/runner/data_processing.rs new file mode 100644 index 00000000000..d5bf991d505 --- /dev/null +++ b/core/src/amp_subgraph/runner/data_processing.rs @@ -0,0 +1,267 @@ +use std::sync::Arc; + +use alloy::primitives::{BlockHash, BlockNumber}; +use anyhow::anyhow; +use arrow::array::RecordBatch; +use chrono::{DateTime, Utc}; +use graph::{ + amp::{ + codec::{DecodeOutput, DecodedEntity, Decoder, utils::auto_block_timestamp_decoder}, + stream_aggregator::{RecordBatchGroup, RecordBatchGroups, StreamRecordBatch}, + }, + blockchain::block_stream::FirehoseCursor, + cheap_clone::CheapClone, + components::store::{EntityCache, EntityLfuCache, ModificationsAndCache, SeqGenerator}, +}; +use slog::{debug, trace}; + +use super::{Compat, Context, Error, data_stream::TablePtr}; + +pub(super) async fn process_record_batch_groups( + cx: &mut Context, + mut entity_lfu_cache: EntityLfuCache, + record_batch_groups: RecordBatchGroups, + stream_table_ptr: Arc<[TablePtr]>, + latest_block: BlockNumber, +) -> Result { + if record_batch_groups.is_empty() { + debug!(cx.logger, "Received no record batch groups"); + return Ok(entity_lfu_cache); + } + + let from_block = record_batch_groups + .first_key_value() + .map(|((block, _), _)| *block) + .unwrap(); + + let to_block = record_batch_groups + .last_key_value() + .map(|((block, _), _)| *block) + .unwrap(); + + debug!(cx.logger, "Processing record batch groups"; + "from_block" => from_block, + "to_block" => to_block + ); + + for ((block_number, block_hash), record_batch_group) in record_batch_groups { + trace!(cx.logger, "Processing record batch group"; + "block" => block_number, + "record_batches_count" => record_batch_group.record_batches.len() + ); + + entity_lfu_cache = process_record_batch_group( + cx, + entity_lfu_cache, + block_number, + block_hash, + record_batch_group, + &stream_table_ptr, + latest_block, + ) + .await + .map_err(|e| { + e.context(format!( + "failed to process record batch group at block '{block_number}'" + )) + })?; + + cx.metrics.deployment_head.update(block_number); + cx.metrics.blocks_processed.record_one(); + + trace!(cx.logger, "Completed processing record batch group"; + "block" => block_number + ); + } + + debug!(cx.logger, "Completed processing record batch groups"; + "from_block" => from_block, + "to_block" => to_block + ); + + Ok(entity_lfu_cache) +} + +async fn process_record_batch_group( + cx: &mut Context, + entity_lfu_cache: EntityLfuCache, + block_number: BlockNumber, + block_hash: BlockHash, + record_batch_group: RecordBatchGroup, + stream_table_ptr: &[TablePtr], + latest_block: BlockNumber, +) -> Result { + let _section = cx + .metrics + .stopwatch + .start_section("process_record_batch_group"); + + let RecordBatchGroup { record_batches } = record_batch_group; + + if record_batches.is_empty() { + debug!(cx.logger, "Record batch group is empty"); + return Ok(entity_lfu_cache); + } + + let mut entity_cache = EntityCache::with_current( + cx.store.cheap_clone(), + entity_lfu_cache, + SeqGenerator::new(block_number.compat()), + ); + + let block_timestamp = if cx.manifest.schema.has_aggregations() { + decode_block_timestamp(&record_batches) + .map_err(|e| e.context("failed to decode block timestamp"))? + } else { + // TODO: Block timestamp is only required for subgraph aggregations. + // Make it optional at the store level. + DateTime::::MIN_UTC + }; + + for record_batch in record_batches { + let StreamRecordBatch { + stream_index, + record_batch, + } = record_batch; + + process_record_batch( + cx, + &mut entity_cache, + record_batch, + stream_table_ptr[stream_index], + ) + .await + .map_err(|e| { + e.context(format!( + "failed to process record batch for stream '{stream_index}'" + )) + })?; + } + + let section = cx.metrics.stopwatch.start_section("as_modifications"); + let ModificationsAndCache { + modifications, + entity_lfu_cache, + evict_stats: _, + } = entity_cache + .as_modifications(block_number.compat(), &cx.metrics.stopwatch) + .await + .map_err(Error::from) + .map_err(|e| e.context("failed to extract entity modifications from the state"))?; + section.end(); + + let _section = cx.metrics.stopwatch.start_section("transact_block"); + let is_close_to_chain_head = latest_block.saturating_sub(block_number) <= 100; + + cx.store + .transact_block_operations( + (block_number, block_hash).compat(), + block_timestamp.compat(), + FirehoseCursor::None, + modifications, + &cx.metrics.stopwatch, + Vec::new(), + Vec::new(), + Vec::new(), + false, + is_close_to_chain_head, + ) + .await + .map_err(Error::from) + .map_err(|e| e.context("failed to transact block operations"))?; + + if is_close_to_chain_head { + cx.metrics.deployment_synced.record(true); + } + + Ok(entity_lfu_cache) +} + +async fn process_record_batch( + cx: &mut Context, + entity_cache: &mut EntityCache, + record_batch: RecordBatch, + (i, j): TablePtr, +) -> Result<(), Error> { + let _section = cx.metrics.stopwatch.start_section("process_record_batch"); + + let table = &cx.manifest.data_sources[i].transformer.tables[j]; + let entity_name = &table.name; + + let DecodeOutput { + entity_type, + id_type, + decoded_entities, + } = cx + .codec + .decode(record_batch, entity_name.as_str()) + .map_err(|e| { + Error::Deterministic( + e.context(format!("failed to decode entities of type '{entity_name}'")), + ) + })?; + + for decoded_entity in decoded_entities { + let DecodedEntity { + key, + mut entity_data, + } = decoded_entity; + + let key = match key { + Some(key) => key, + None => { + let entity_id = entity_cache.seq_gen.id(id_type).map_err(|e| { + Error::Deterministic(e.context(format!( + "failed to generate a new id for an entity of type '{entity_name}'" + ))) + })?; + + entity_data.push(("id".into(), entity_id.clone().into())); + entity_type.key(entity_id) + } + }; + + let entity_id = key.entity_id.clone(); + let entity = cx.manifest.schema.make_entity(entity_data).map_err(|e| { + Error::Deterministic(anyhow!(e).context(format!( + "failed to create a new entity of type '{entity_name}' with id '{entity_id}'" + ))) + })?; + + entity_cache.set(key, entity, None).await.map_err(|e| { + Error::Deterministic(e.context(format!( + "failed to store a new entity of type '{entity_name}' with id '{entity_id}'" + ))) + })?; + } + + Ok(()) +} + +/// Decodes the block timestamp from the first matching column in `record_batches`. +/// +/// Iterates through the provided record batches and returns the timestamp from +/// the first batch that contains a valid block timestamp column. +/// +/// # Preconditions +/// +/// All entries in `record_batches` must belong to the same record batch group. +fn decode_block_timestamp(record_batches: &[StreamRecordBatch]) -> Result, Error> { + let mut last_error: Option = None; + + for record_batch in record_batches { + match auto_block_timestamp_decoder(&record_batch.record_batch) { + Ok((_, decoder)) => { + return decoder + .decode(0) + .map_err(Error::Deterministic)? + .ok_or_else(|| Error::Deterministic(anyhow!("block timestamp is empty"))); + } + Err(e) => { + last_error = Some(Error::Deterministic(e)); + } + } + } + + Err(last_error.unwrap()) +} diff --git a/core/src/amp_subgraph/runner/data_stream.rs b/core/src/amp_subgraph/runner/data_stream.rs new file mode 100644 index 00000000000..693ae9384a7 --- /dev/null +++ b/core/src/amp_subgraph/runner/data_stream.rs @@ -0,0 +1,232 @@ +use std::{collections::HashMap, ops::RangeInclusive, sync::Arc}; + +use alloy::primitives::BlockNumber; +use anyhow::anyhow; +use futures::{ + StreamExt, TryStreamExt, + stream::{self, BoxStream}, +}; +use graph::{ + amp::{ + Client, + client::ResponseBatch, + error::IsDeterministic, + manifest::DataSource, + stream_aggregator::{RecordBatchGroups, StreamAggregator}, + }, + cheap_clone::CheapClone, + prelude::StopwatchMetrics, +}; +use slog::{debug, warn}; + +use super::{Context, Error}; + +pub(super) type TablePtr = (usize, usize); + +pub(super) fn new_data_stream( + cx: &Context, + latest_block: BlockNumber, +) -> BoxStream<'static, Result<(RecordBatchGroups, Arc<[TablePtr]>), Error>> +where + AC: Client + Send + Sync + 'static, +{ + let logger = cx.logger.new(slog::o!("process" => "new_data_stream")); + let client = cx.client.cheap_clone(); + let manifest = cx.manifest.clone(); + let buffer_size = cx.buffer_size; + let block_range = cx.block_range; + let stopwatch = cx.metrics.stopwatch.cheap_clone(); + + debug!(logger, "Creating data stream"; + "from_block" => cx.latest_synced_block().unwrap_or(BlockNumber::MIN), + "to_block" => latest_block, + "start_block" => cx.start_block(), + "block_range" => block_range, + ); + + // State: (latest_queried_block, end_block, is_first) + let initial_state = (cx.latest_synced_block(), BlockNumber::MIN, true); + + stream::unfold( + initial_state, + move |(latest_queried_block, mut end_block, is_first)| { + let block_ranges = next_block_ranges( + &manifest.data_sources, + block_range, + latest_queried_block, + latest_block, + ); + + if block_ranges.is_empty() { + if is_first { + warn!(logger, "There are no unprocessed block ranges"); + } + return futures::future::ready(None); + } + + let start_block = block_ranges.values().map(|r| *r.start()).min().unwrap(); + end_block = end_block.max(block_ranges.values().map(|r| *r.end()).max().unwrap()); + + let (query_streams, table_ptrs) = + build_query_streams(&*client, &logger, &manifest.data_sources, &block_ranges); + + let data_stream = build_data_stream( + &logger, + query_streams, + table_ptrs, + buffer_size, + &stopwatch, + start_block, + ); + + debug!(logger, "Created a new data stream"; + "latest_queried_block" => latest_queried_block, + "start_block" => start_block, + "end_block" => end_block, + ); + futures::future::ready(Some((data_stream, (Some(end_block), end_block, false)))) + }, + ) + .flatten() + .boxed() +} + +fn build_query_streams( + client: &AC, + logger: &slog::Logger, + data_sources: &[DataSource], + block_ranges: &HashMap>, +) -> ( + Vec<(String, BoxStream<'static, Result>)>, + Arc<[TablePtr]>, +) { + let total_queries: usize = data_sources + .iter() + .map(|ds| ds.transformer.tables.len()) + .sum(); + + let mut query_streams = Vec::with_capacity(total_queries); + let mut table_ptrs = Vec::with_capacity(total_queries); + + for (i, data_source) in data_sources.iter().enumerate() { + let Some(block_range) = block_ranges.get(&i) else { + continue; + }; + + for (j, table) in data_source.transformer.tables.iter().enumerate() { + let query = table.query.build_with_block_range(block_range); + let stream = client.query(logger, query, None); + let stream_name = format!("{}.{}", data_source.name, table.name); + + query_streams.push((stream_name, stream)); + table_ptrs.push((i, j)); + } + } + + (query_streams, table_ptrs.into()) +} + +fn build_data_stream( + logger: &slog::Logger, + query_streams: Vec<(String, BoxStream<'static, Result>)>, + table_ptrs: Arc<[TablePtr]>, + buffer_size: usize, + stopwatch: &StopwatchMetrics, + start_block: BlockNumber, +) -> BoxStream<'static, Result<(RecordBatchGroups, Arc<[TablePtr]>), Error>> +where + E: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + let mut start_block_checked = false; + let mut load_first_record_batch_group_section = + Some(stopwatch.start_section("load_first_record_batch_group")); + + StreamAggregator::new(logger, query_streams, buffer_size) + .map_ok(move |response| (response, table_ptrs.cheap_clone())) + .map_err(Error::from) + .map(move |result| { + if load_first_record_batch_group_section.is_some() { + let _section = load_first_record_batch_group_section.take(); + } + + match result { + Ok(response) => { + if !start_block_checked { + if let Some(((first_block, _), _)) = response.0.first_key_value() + && *first_block < start_block + { + return Err(Error::NonDeterministic(anyhow!("chain reorg"))); + } + + start_block_checked = true; + } + + Ok(response) + } + Err(e) => Err(e), + } + }) + .boxed() +} + +fn next_block_ranges( + data_sources: &[DataSource], + block_range: usize, + latest_queried_block: Option, + latest_block: BlockNumber, +) -> HashMap> { + let block_ranges = data_sources + .iter() + .enumerate() + .filter_map(|(i, data_source)| { + next_block_range(block_range, data_source, latest_queried_block, latest_block) + .map(|block_range| (i, block_range)) + }) + .collect::>(); + + let Some(min_block_range) = block_ranges + .iter() + .min_by_key(|(_, block_range)| *block_range.start()) + .map(|(_, min_block_range)| min_block_range.clone()) + else { + return HashMap::new(); + }; + + block_ranges + .into_iter() + .filter(|(_, block_range)| block_range.start() <= min_block_range.end()) + .collect() +} + +fn next_block_range( + block_range: usize, + data_source: &DataSource, + latest_queried_block: Option, + latest_block: BlockNumber, +) -> Option> { + let start_block = match latest_queried_block { + Some(latest_queried_block) => { + if latest_queried_block >= data_source.source.end_block { + return None; + } + + latest_queried_block + 1 + } + None => data_source.source.start_block, + }; + + let end_block = [ + start_block.saturating_add(block_range as BlockNumber), + data_source.source.end_block, + latest_block, + ] + .into_iter() + .min() + .unwrap(); + + if start_block > end_block { + return None; + } + + Some(start_block..=end_block) +} diff --git a/core/src/amp_subgraph/runner/error.rs b/core/src/amp_subgraph/runner/error.rs new file mode 100644 index 00000000000..8c7077e1c68 --- /dev/null +++ b/core/src/amp_subgraph/runner/error.rs @@ -0,0 +1,43 @@ +use graph::amp::error::IsDeterministic; +use thiserror::Error; + +#[derive(Debug, Error)] +pub(super) enum Error { + #[error("runner failed with a non-deterministic error: {0:#}")] + NonDeterministic(#[source] anyhow::Error), + + #[error("runner failed with a deterministic error: {0:#}")] + Deterministic(#[source] anyhow::Error), +} + +impl Error { + pub(super) fn context(self, context: C) -> Self + where + C: std::fmt::Display + Send + Sync + 'static, + { + match self { + Self::NonDeterministic(e) => Self::NonDeterministic(e.context(context)), + Self::Deterministic(e) => Self::Deterministic(e.context(context)), + } + } + + pub(super) fn is_deterministic(&self) -> bool { + match self { + Self::Deterministic(_) => true, + Self::NonDeterministic(_) => false, + } + } +} + +impl From for Error +where + T: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + fn from(e: T) -> Self { + if e.is_deterministic() { + Self::Deterministic(e.into()) + } else { + Self::NonDeterministic(e.into()) + } + } +} diff --git a/core/src/amp_subgraph/runner/latest_blocks.rs b/core/src/amp_subgraph/runner/latest_blocks.rs new file mode 100644 index 00000000000..17bdc23dd29 --- /dev/null +++ b/core/src/amp_subgraph/runner/latest_blocks.rs @@ -0,0 +1,180 @@ +use alloy::primitives::BlockNumber; +use anyhow::anyhow; +use arrow::array::RecordBatch; +use futures::{StreamExt, TryFutureExt, future::try_join_all, stream::BoxStream}; +use graph::amp::{ + Client, + client::ResponseBatch, + codec::{Decoder, utils::block_number_decoder}, + error::IsDeterministic, + manifest::DataSource, +}; +use itertools::Itertools; +use slog::debug; + +use super::{Context, Error}; + +pub(super) type TablePtr = (usize, usize); + +pub(super) struct LatestBlocks(Vec<(TablePtr, BlockNumber)>); + +impl LatestBlocks { + pub(super) async fn load(cx: &Context) -> Result + where + AC: Client, + { + debug!(cx.logger, "Loading latest blocks"); + let _section = cx.metrics.stopwatch.start_section("load_latest_blocks"); + + let latest_block_futs = cx + .manifest + .data_sources + .iter() + .enumerate() + .flat_map(|(i, data_source)| { + data_source + .source + .tables + .iter() + .enumerate() + .map(move |(j, table)| ((i, j), &data_source.source.dataset, table)) + }) + .unique_by(|(_, dataset, table)| (dataset.to_string(), table.to_string())) + .map(|(table_ptr, dataset, table)| { + latest_block(cx, dataset, table) + .map_ok(move |latest_block| (table_ptr, latest_block)) + .map_err(move |e| { + e.context(format!( + "failed to load latest block for '{dataset}.{table}'" + )) + }) + }); + + try_join_all(latest_block_futs).await.map(Self) + } + + pub(super) fn filter_completed(self, cx: &Context) -> Self + where + AC: Client, + { + let latest_synced_block = cx.latest_synced_block(); + + Self( + self.0 + .into_iter() + .filter(|((i, _), _)| { + !indexing_completed(&cx.manifest.data_sources[*i], &latest_synced_block) + }) + .collect(), + ) + } + + pub(super) fn min(&self) -> BlockNumber { + self.0 + .iter() + .min_by_key(|(_, latest_block)| *latest_block) + .map(|(_, latest_block)| *latest_block) + .unwrap() + } + + pub(super) async fn changed(self, cx: &Context) -> Result<(), Error> + where + AC: Client, + { + debug!(cx.logger, "Waiting for new blocks"); + let _section = cx.metrics.stopwatch.start_section("latest_blocks_changed"); + + let min_latest_block = self.min(); + let latest_synced_block = cx.latest_synced_block(); + + let latest_block_changed_futs = self + .0 + .into_iter() + .filter(|(_, latest_block)| *latest_block == min_latest_block) + .filter(|((i, _), _)| { + !indexing_completed(&cx.manifest.data_sources[*i], &latest_synced_block) + }) + .map(|((i, j), latest_block)| { + let source = &cx.manifest.data_sources[i].source; + let dataset = &source.dataset; + let table = &source.tables[j]; + + latest_block_changed(cx, dataset, table, latest_block).map_err(move |e| { + e.context(format!( + "failed to check if the latest block changed in '{dataset}.{table}'" + )) + }) + }); + + let _response = try_join_all(latest_block_changed_futs).await?; + + Ok(()) + } + + pub(super) fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +fn indexing_completed(data_source: &DataSource, latest_synced_block: &Option) -> bool { + latest_synced_block + .as_ref() + .is_some_and(|latest_synced_block| *latest_synced_block >= data_source.source.end_block) +} + +async fn latest_block( + cx: &Context, + dataset: &str, + table: &str, +) -> Result +where + AC: Client, +{ + let query = format!("SELECT MAX(_block_num) FROM {dataset}.{table}"); + let stream = cx.client.query(&cx.logger, query, None); + let record_batch = read_once(stream).await?; + + let latest_block = block_number_decoder(&record_batch, 0) + .map_err(Error::Deterministic)? + .decode(0) + .map_err(Error::Deterministic)? + .ok_or_else(|| Error::NonDeterministic(anyhow!("table is empty")))?; + + Ok(latest_block) +} + +async fn latest_block_changed( + cx: &Context, + dataset: &str, + table: &str, + latest_block: BlockNumber, +) -> Result<(), Error> +where + AC: Client, +{ + let query = format!( + "SELECT _block_num FROM {dataset}.{table} WHERE _block_num > {latest_block} SETTINGS stream = true" + ); + let stream = cx.client.query(&cx.logger, query, None); + let _record_batch = read_once(stream).await?; + + Ok(()) +} + +async fn read_once( + mut stream: BoxStream<'static, Result>, +) -> Result +where + E: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + let response = stream + .next() + .await + .ok_or_else(|| Error::NonDeterministic(anyhow!("stream is empty")))? + .map_err(Error::from)?; + + match response { + ResponseBatch::Batch { data } => Ok(data), + _ => Err(Error::NonDeterministic(anyhow!("response is empty"))), + } +} diff --git a/core/src/amp_subgraph/runner/mod.rs b/core/src/amp_subgraph/runner/mod.rs new file mode 100644 index 00000000000..4e0e85294b6 --- /dev/null +++ b/core/src/amp_subgraph/runner/mod.rs @@ -0,0 +1,190 @@ +mod compat; +mod context; +mod data_processing; +mod data_stream; +mod error; +mod latest_blocks; +mod reorg_handler; + +use std::time::{Duration, Instant}; + +use anyhow::Result; +use futures::StreamExt; +use graph::{ + amp::Client, cheap_clone::CheapClone, components::store::EntityLfuCache, + data::subgraph::schema::SubgraphError, util::lfu_cache::LfuCache, +}; +use slog::{debug, error, warn}; +use tokio_util::sync::CancellationToken; + +use self::{ + compat::Compat, data_processing::process_record_batch_groups, data_stream::new_data_stream, + error::Error, latest_blocks::LatestBlocks, reorg_handler::check_and_handle_reorg, +}; + +pub(super) use self::context::Context; + +pub(super) async fn new_runner( + mut cx: Context, + cancel_token: CancellationToken, +) -> Result<()> +where + AC: Client + Send + Sync + 'static, +{ + let indexing_duration_handle = tokio::spawn({ + let mut instant = Instant::now(); + let indexing_duration = cx.metrics.indexing_duration.clone(); + + async move { + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + + let prev_instant = std::mem::replace(&mut instant, Instant::now()); + indexing_duration.record(prev_instant.elapsed()); + } + } + }); + + let result = cancel_token + .run_until_cancelled(run_indexing_with_retries(&mut cx)) + .await; + + indexing_duration_handle.abort(); + + match result { + Some(result) => result?, + None => { + debug!(cx.logger, "Processed cancel signal"); + } + } + + cx.metrics.deployment_status.stopped(); + + debug!(cx.logger, "Waiting for the store to finish processing"); + cx.store.flush().await?; + Ok(()) +} + +async fn run_indexing(cx: &mut Context) -> Result<(), Error> +where + AC: Client + Send + Sync + 'static, +{ + cx.metrics.deployment_status.starting(); + + if let Some(latest_synced_block) = cx.latest_synced_block() { + cx.metrics.deployment_head.update(latest_synced_block); + } + + cx.metrics + .deployment_synced + .record(cx.store.is_deployment_synced()); + + loop { + cx.metrics.deployment_status.running(); + + debug!(cx.logger, "Running indexing"; + "latest_synced_block_ptr" => ?cx.latest_synced_block_ptr() + ); + + let mut latest_blocks = LatestBlocks::load(cx).await?; + check_and_handle_reorg(cx, &latest_blocks).await?; + + if cx.indexing_completed() { + cx.metrics.deployment_synced.record(true); + + debug!(cx.logger, "Indexing completed"); + return Ok(()); + } + + latest_blocks = latest_blocks.filter_completed(cx); + let latest_block = latest_blocks.min(); + + cx.metrics + .deployment_target + .update(latest_block.min(cx.end_block())); + + let mut deployment_is_failed = cx.store.health().await?.is_failed(); + let mut entity_lfu_cache: EntityLfuCache = LfuCache::new(); + let mut stream = new_data_stream(cx, latest_block); + + while let Some(result) = stream.next().await { + let (record_batch_groups, stream_table_ptr) = result?; + + entity_lfu_cache = process_record_batch_groups( + cx, + entity_lfu_cache, + record_batch_groups, + stream_table_ptr, + latest_block, + ) + .await?; + + if deployment_is_failed && let Some(block_ptr) = cx.store.block_ptr() { + cx.store.unfail_non_deterministic_error(&block_ptr).await?; + deployment_is_failed = false; + } + } + + // Check if the Amp Flight server has data covering through every data + // source's endBlock. This handles the case where endBlock has no entity + // data — the persisted block pointer never advances to endBlock, but the + // server's latest block confirms all queries have been served. + if latest_block >= cx.end_block() { + cx.metrics.deployment_synced.record(true); + + debug!(cx.logger, "Indexing completed; endBlock reached via server latest block"; + "latest_block" => latest_block, + "end_block" => cx.end_block() + ); + return Ok(()); + } + + debug!(cx.logger, "Completed indexing iteration"; + "latest_synced_block_ptr" => ?cx.latest_synced_block_ptr() + ); + + latest_blocks.changed(cx).await?; + cx.backoff.reset(); + } +} + +async fn run_indexing_with_retries(cx: &mut Context) -> Result<()> +where + AC: Client + Send + Sync + 'static, +{ + loop { + match run_indexing(cx).await { + Ok(()) => return Ok(()), + Err(e) => { + cx.metrics.deployment_status.failed(); + + let deterministic = e.is_deterministic(); + + cx.store + .fail_subgraph(SubgraphError { + subgraph_id: cx.deployment.cheap_clone(), + message: format!("{e:#}"), + block_ptr: None, // TODO: Find a way to propagate the block ptr here + handler: None, + deterministic, + }) + .await?; + + if deterministic { + error!(cx.logger, "Subgraph failed with a deterministic error"; + "e" => ?e + ); + return Err(e.into()); + } + + warn!(cx.logger, "Subgraph failed with a non-deterministic error"; + "e" => ?e, + "retry_delay_seconds" => cx.backoff.delay().as_secs() + ); + + cx.backoff.sleep_async().await; + debug!(cx.logger, "Restarting indexing"); + } + } + } +} diff --git a/core/src/amp_subgraph/runner/reorg_handler.rs b/core/src/amp_subgraph/runner/reorg_handler.rs new file mode 100644 index 00000000000..8f59169304b --- /dev/null +++ b/core/src/amp_subgraph/runner/reorg_handler.rs @@ -0,0 +1,163 @@ +use alloy::primitives::{BlockHash, BlockNumber}; +use anyhow::anyhow; +use futures::{StreamExt, TryFutureExt, future::try_join_all}; +use graph::{ + amp::{ + Client, + client::{LatestBlockBeforeReorg, RequestMetadata, ResponseBatch, ResumeStreamingQuery}, + }, + blockchain::block_stream::FirehoseCursor, +}; +use itertools::Itertools; +use slog::debug; + +use super::{Compat, Context, Error, LatestBlocks}; + +pub(super) async fn check_and_handle_reorg( + cx: &Context, + latest_blocks: &LatestBlocks, +) -> Result<(), Error> +where + AC: Client, +{ + let logger = cx + .logger + .new(slog::o!("process" => "check_and_handle_reorg")); + + let Some((latest_synced_block_number, latest_synced_block_hash)) = cx.latest_synced_block_ptr() + else { + debug!(logger, "There are no synced blocks; Skipping reorg check"); + return Ok(()); + }; + + debug!(logger, "Running reorg check"); + + let Some(latest_block_before_reorg) = detect_deepest_reorg( + cx, + latest_blocks, + latest_synced_block_number, + latest_synced_block_hash, + ) + .await? + else { + debug!(logger, "Successfully checked for reorg: No reorg detected"; + "latest_synced_block" => latest_synced_block_number + ); + return Ok(()); + }; + + let _section = cx.metrics.stopwatch.start_section("handle_reorg"); + + debug!(logger, "Handling reorg"; + "latest_synced_block" => latest_synced_block_number, + "latest_block_before_reorg" => ?latest_block_before_reorg.block_number + ); + + let (block_number, block_hash) = match ( + latest_block_before_reorg.block_number, + latest_block_before_reorg.block_hash, + ) { + (Some(block_number), Some(block_hash)) => (block_number, block_hash), + (_, _) => { + // TODO: Handle reorgs to the genesis block + return Err(Error::Deterministic(anyhow!( + "invalid reorg: rewind to the genesis block not supported" + ))); + } + }; + + if block_number > latest_synced_block_number { + return Err(Error::Deterministic(anyhow!( + "invalid reorg: latest block before reorg cannot be higher than the invalidated block" + ))); + } else if block_number == latest_synced_block_number && block_hash == latest_synced_block_hash { + return Err(Error::Deterministic(anyhow!( + "invalid reorg: latest block before reorg cannot be equal to the invalidated block" + ))); + } + + cx.store + .revert_block_operations((block_number, block_hash).compat(), FirehoseCursor::None) + .await + .map_err(Error::from)?; + + Ok(()) +} + +async fn detect_deepest_reorg( + cx: &Context, + latest_blocks: &LatestBlocks, + latest_synced_block_number: BlockNumber, + latest_synced_block_hash: BlockHash, +) -> Result, Error> +where + AC: Client, +{ + let detect_reorg_futs = latest_blocks + .iter() + .filter(|(_, latest_block)| *latest_block >= latest_synced_block_number) + .map(|((i, j), _)| { + let data_source = &cx.manifest.data_sources[*i]; + let network = &data_source.network; + let dataset = &data_source.source.dataset; + let table = &data_source.source.tables[*j]; + + detect_reorg( + cx, + network, + dataset, + table, + latest_synced_block_number, + latest_synced_block_hash, + ) + .map_err(move |e| e.context(format!("failed to detect reorg in '{dataset}.{table}'"))) + }); + + let deepest_reorg = try_join_all(detect_reorg_futs) + .await? + .into_iter() + .flatten() + .min_by_key(|latest_block_before_reorg| latest_block_before_reorg.block_number); + + Ok(deepest_reorg) +} + +async fn detect_reorg( + cx: &Context, + network: &str, + dataset: &str, + table: &str, + latest_synced_block_number: BlockNumber, + latest_synced_block_hash: BlockHash, +) -> Result, Error> +where + AC: Client, +{ + let query = format!("SELECT _block_num FROM {dataset}.{table} SETTINGS stream = true"); + let mut stream = cx.client.query( + &cx.logger, + query, + Some(RequestMetadata { + resume_streaming_query: Some(vec![ResumeStreamingQuery { + network: network.to_string(), + block_number: latest_synced_block_number, + block_hash: latest_synced_block_hash, + }]), + }), + ); + + let response = stream + .next() + .await + .ok_or_else(|| Error::NonDeterministic(anyhow!("stream is empty")))? + .map_err(Error::from)?; + + match response { + ResponseBatch::Batch { .. } => Ok(None), + ResponseBatch::Reorg(reorg) => reorg + .into_iter() + .exactly_one() + .map_err(|_e| Error::Deterministic(anyhow!("multi-chain datasets are not supported"))) + .map(Some), + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 2236519bf3f..118ce3ebaae 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,10 +1,6 @@ -pub mod polling_monitor; - -mod link_resolver; -mod subgraph; +mod subgraph_manifest; -pub use crate::link_resolver::LinkResolver; -pub use crate::subgraph::{ - SubgraphAssignmentProvider, SubgraphInstanceManager, SubgraphRegistrar, SubgraphRunner, - SubgraphTriggerProcessor, -}; +pub mod amp_subgraph; +pub mod polling_monitor; +pub mod subgraph; +pub mod subgraph_provider; diff --git a/core/src/link_resolver.rs b/core/src/link_resolver.rs deleted file mode 100644 index 364dbd76f10..00000000000 --- a/core/src/link_resolver.rs +++ /dev/null @@ -1,406 +0,0 @@ -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use anyhow::anyhow; -use async_trait::async_trait; -use bytes::BytesMut; -use futures01::{stream::poll_fn, try_ready}; -use futures03::stream::FuturesUnordered; -use graph::env::EnvVars; -use graph::util::futures::RetryConfigNoTimeout; -use lru_time_cache::LruCache; -use serde_json::Value; - -use graph::{ - ipfs_client::{IpfsClient, StatApi}, - prelude::{LinkResolver as LinkResolverTrait, *}, -}; - -fn retry_policy( - always_retry: bool, - op: &'static str, - logger: &Logger, -) -> RetryConfigNoTimeout { - // Even if retries were not requested, networking errors are still retried until we either get - // a valid HTTP response or a timeout. - if always_retry { - retry(op, logger).no_limit() - } else { - retry(op, logger) - .no_limit() - .when(|res: &Result<_, reqwest::Error>| match res { - Ok(_) => false, - Err(e) => !(e.is_status() || e.is_timeout()), - }) - } - .no_timeout() // The timeout should be set in the internal future. -} - -/// The IPFS APIs don't have a quick "do you have the file" function. Instead, we -/// just rely on whether an API times out. That makes sense for IPFS, but not for -/// our application. We want to be able to quickly select from a potential list -/// of clients where hopefully one already has the file, and just get the file -/// from that. -/// -/// The strategy here then is to use a stat API as a proxy for "do you have the -/// file". Whichever client has or gets the file first wins. This API is a good -/// choice, because it doesn't involve us actually starting to download the file -/// from each client, which would be wasteful of bandwidth and memory in the -/// case multiple clients respond in a timely manner. In addition, we may make -/// good use of the stat returned. -async fn select_fastest_client_with_stat( - clients: Arc>, - logger: Logger, - api: StatApi, - path: String, - timeout: Duration, - do_retry: bool, -) -> Result<(u64, IpfsClient), Error> { - let mut err: Option = None; - - let mut stats: FuturesUnordered<_> = clients - .iter() - .enumerate() - .map(|(i, c)| { - let c = c.cheap_clone(); - let path = path.clone(); - retry_policy(do_retry, "IPFS stat", &logger).run(move || { - let path = path.clone(); - let c = c.cheap_clone(); - async move { - c.stat_size(api, path, timeout) - .map_ok(move |s| (s, i)) - .await - } - }) - }) - .collect(); - - while let Some(result) = stats.next().await { - match result { - Ok((stat, index)) => { - return Ok((stat, clients[index].cheap_clone())); - } - Err(e) => err = Some(e.into()), - } - } - - Err(err.unwrap_or_else(|| { - anyhow!( - "No IPFS clients were supplied to handle the call to object.stat. File: {}", - path - ) - })) -} - -// Returns an error if the stat is bigger than `max_file_bytes` -fn restrict_file_size(path: &str, size: u64, max_file_bytes: usize) -> Result<(), Error> { - if size > max_file_bytes as u64 { - return Err(anyhow!( - "IPFS file {} is too large. It can be at most {} bytes but is {} bytes", - path, - max_file_bytes, - size - )); - } - Ok(()) -} - -#[derive(Clone)] -pub struct LinkResolver { - clients: Arc>, - cache: Arc>>>, - timeout: Duration, - retry: bool, - env_vars: Arc, -} - -impl LinkResolver { - pub fn new(clients: Vec, env_vars: Arc) -> Self { - Self { - clients: Arc::new(clients.into_iter().collect()), - cache: Arc::new(Mutex::new(LruCache::with_capacity( - env_vars.mappings.max_ipfs_cache_size as usize, - ))), - timeout: env_vars.mappings.ipfs_timeout, - retry: false, - env_vars, - } - } -} - -impl Debug for LinkResolver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LinkResolver") - .field("timeout", &self.timeout) - .field("retry", &self.retry) - .field("env_vars", &self.env_vars) - .finish() - } -} - -impl CheapClone for LinkResolver { - fn cheap_clone(&self) -> Self { - self.clone() - } -} - -#[async_trait] -impl LinkResolverTrait for LinkResolver { - fn with_timeout(&self, timeout: Duration) -> Box { - let mut s = self.cheap_clone(); - s.timeout = timeout; - Box::new(s) - } - - fn with_retries(&self) -> Box { - let mut s = self.cheap_clone(); - s.retry = true; - Box::new(s) - } - - /// Supports links of the form `/ipfs/ipfs_hash` or just `ipfs_hash`. - async fn cat(&self, logger: &Logger, link: &Link) -> Result, Error> { - // Discard the `/ipfs/` prefix (if present) to get the hash. - let path = link.link.trim_start_matches("/ipfs/").to_owned(); - - if let Some(data) = self.cache.lock().unwrap().get(&path) { - trace!(logger, "IPFS cache hit"; "hash" => &path); - return Ok(data.clone()); - } - trace!(logger, "IPFS cache miss"; "hash" => &path); - - let (size, client) = select_fastest_client_with_stat( - self.clients.cheap_clone(), - logger.cheap_clone(), - StatApi::Files, - path.clone(), - self.timeout, - self.retry, - ) - .await?; - - let max_cache_file_size = self.env_vars.mappings.max_ipfs_cache_file_size; - let max_file_size = self.env_vars.mappings.max_ipfs_file_bytes; - restrict_file_size(&path, size, max_file_size)?; - - let req_path = path.clone(); - let timeout = self.timeout; - let data = retry_policy(self.retry, "ipfs.cat", logger) - .run(move || { - let path = req_path.clone(); - let client = client.clone(); - async move { Ok(client.cat_all(&path, timeout).await?.to_vec()) } - }) - .await?; - - // The size reported by `files/stat` is not guaranteed to be exact, so check the limit again. - restrict_file_size(&path, data.len() as u64, max_file_size)?; - - // Only cache files if they are not too large - if data.len() <= max_cache_file_size { - let mut cache = self.cache.lock().unwrap(); - if !cache.contains_key(&path) { - cache.insert(path.clone(), data.clone()); - } - } else { - debug!(logger, "File too large for cache"; - "path" => path, - "size" => data.len() - ); - } - - Ok(data) - } - - async fn get_block(&self, logger: &Logger, link: &Link) -> Result, Error> { - trace!(logger, "IPFS block get"; "hash" => &link.link); - let (size, client) = select_fastest_client_with_stat( - self.clients.cheap_clone(), - logger.cheap_clone(), - StatApi::Block, - link.link.clone(), - self.timeout, - self.retry, - ) - .await?; - - let max_file_size = self.env_vars.mappings.max_ipfs_file_bytes; - restrict_file_size(&link.link, size, max_file_size)?; - - let link = link.link.clone(); - let data = retry_policy(self.retry, "ipfs.getBlock", logger) - .run(move || { - let link = link.clone(); - let client = client.clone(); - async move { - let data = client.get_block(link.clone()).await?.to_vec(); - Result::, reqwest::Error>::Ok(data) - } - }) - .await?; - - Ok(data) - } - - async fn json_stream(&self, logger: &Logger, link: &Link) -> Result { - // Discard the `/ipfs/` prefix (if present) to get the hash. - let path = link.link.trim_start_matches("/ipfs/"); - - let (size, client) = select_fastest_client_with_stat( - self.clients.cheap_clone(), - logger.cheap_clone(), - StatApi::Files, - path.to_string(), - self.timeout, - self.retry, - ) - .await?; - - let max_file_size = self.env_vars.mappings.max_ipfs_map_file_size; - restrict_file_size(path, size, max_file_size)?; - - let mut stream = client.cat(path, None).await?.fuse().boxed().compat(); - - let mut buf = BytesMut::with_capacity(1024); - - // Count the number of lines we've already successfully deserialized. - // We need that to adjust the line number in error messages from serde_json - // to translate from line numbers in the snippet we are deserializing - // to the line number in the overall file - let mut count = 0; - - let stream: JsonValueStream = Box::pin( - poll_fn(move || -> Poll, Error> { - loop { - if let Some(offset) = buf.iter().position(|b| *b == b'\n') { - let line_bytes = buf.split_to(offset + 1); - count += 1; - if line_bytes.len() > 1 { - let line = std::str::from_utf8(&line_bytes)?; - let res = match serde_json::from_str::(line) { - Ok(v) => Ok(Async::Ready(Some(JsonStreamValue { - value: v, - line: count, - }))), - Err(e) => { - // Adjust the line number in the serde error. This - // is fun because we can only get at the full error - // message, and not the error message without line number - let msg = e.to_string(); - let msg = msg.split(" at line ").next().unwrap(); - Err(anyhow!( - "{} at line {} column {}: '{}'", - msg, - e.line() + count - 1, - e.column(), - line - )) - } - }; - return res; - } - } else { - // We only get here if there is no complete line in buf, and - // it is therefore ok to immediately pass an Async::NotReady - // from stream through. - // If we get a None from poll, but still have something in buf, - // that means the input was not terminated with a newline. We - // add that so that the last line gets picked up in the next - // run through the loop. - match try_ready!(stream.poll().map_err(|e| anyhow::anyhow!("{}", e))) { - Some(b) => buf.extend_from_slice(&b), - None if !buf.is_empty() => buf.extend_from_slice(&[b'\n']), - None => return Ok(Async::Ready(None)), - } - } - } - }) - .compat(), - ); - - Ok(stream) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use graph::env::EnvVars; - use serde_json::json; - - #[tokio::test] - async fn max_file_size() { - let mut env_vars = EnvVars::default(); - env_vars.mappings.max_ipfs_file_bytes = 200; - - let file: &[u8] = &[0u8; 201]; - let client = IpfsClient::localhost(); - let resolver = super::LinkResolver::new(vec![client.clone()], Arc::new(env_vars)); - - let logger = Logger::root(slog::Discard, o!()); - - let link = client.add(file.into()).await.unwrap().hash; - let err = LinkResolver::cat(&resolver, &logger, &Link { link: link.clone() }) - .await - .unwrap_err(); - assert_eq!( - err.to_string(), - format!( - "IPFS file {} is too large. It can be at most 200 bytes but is 212 bytes", - link - ) - ); - } - - async fn json_round_trip(text: &'static str, env_vars: EnvVars) -> Result, Error> { - let client = IpfsClient::localhost(); - let resolver = super::LinkResolver::new(vec![client.clone()], Arc::new(env_vars)); - - let logger = Logger::root(slog::Discard, o!()); - let link = client.add(text.as_bytes().into()).await.unwrap().hash; - - let stream = LinkResolver::json_stream(&resolver, &logger, &Link { link }).await?; - stream.map_ok(|sv| sv.value).try_collect().await - } - - #[tokio::test] - async fn read_json_stream() { - let values = json_round_trip("\"with newline\"\n", EnvVars::default()).await; - assert_eq!(vec![json!("with newline")], values.unwrap()); - - let values = json_round_trip("\"without newline\"", EnvVars::default()).await; - assert_eq!(vec![json!("without newline")], values.unwrap()); - - let values = json_round_trip("\"two\" \n \"things\"", EnvVars::default()).await; - assert_eq!(vec![json!("two"), json!("things")], values.unwrap()); - - let values = json_round_trip( - "\"one\"\n \"two\" \n [\"bad\" \n \"split\"]", - EnvVars::default(), - ) - .await; - assert_eq!( - "EOF while parsing a list at line 4 column 0: ' [\"bad\" \n'", - values.unwrap_err().to_string() - ); - } - - #[tokio::test] - async fn ipfs_map_file_size() { - let file = "\"small test string that trips the size restriction\""; - let mut env_vars = EnvVars::default(); - env_vars.mappings.max_ipfs_map_file_size = file.len() - 1; - - let err = json_round_trip(file, env_vars).await.unwrap_err(); - - assert!(err.to_string().contains(" is too large")); - - env_vars = EnvVars::default(); - let values = json_round_trip(file, env_vars).await; - assert_eq!( - vec!["small test string that trips the size restriction"], - values.unwrap() - ); - } -} diff --git a/core/src/polling_monitor/arweave_service.rs b/core/src/polling_monitor/arweave_service.rs index f1d77dae197..cc42f7af5ea 100644 --- a/core/src/polling_monitor/arweave_service.rs +++ b/core/src/polling_monitor/arweave_service.rs @@ -1,25 +1,24 @@ use anyhow::Error; use bytes::Bytes; -use futures::future::BoxFuture; +use graph::futures03::future::BoxFuture; use graph::{ - components::link_resolver::{ArweaveClient, ArweaveResolver, FileSizeLimit}, + components::link_resolver::{ArweaveResolver, FileSizeLimit}, data_source::offchain::Base64, + derive::CheapClone, prelude::CheapClone, }; use std::{sync::Arc, time::Duration}; -use tower::{buffer::Buffer, ServiceBuilder, ServiceExt}; +use tower::{ServiceBuilder, ServiceExt, buffer::Buffer}; pub type ArweaveService = Buffer, Error>>>; pub fn arweave_service( - client: Arc, - timeout: Duration, + client: Arc, rate_limit: u16, max_file_size: FileSizeLimit, ) -> ArweaveService { let arweave = ArweaveServiceInner { client, - timeout, max_file_size, }; @@ -33,23 +32,12 @@ pub fn arweave_service( Buffer::new(svc, u32::MAX as usize) } -#[derive(Clone)] +#[derive(Clone, CheapClone)] struct ArweaveServiceInner { - client: Arc, - timeout: Duration, + client: Arc, max_file_size: FileSizeLimit, } -impl CheapClone for ArweaveServiceInner { - fn cheap_clone(&self) -> Self { - Self { - client: self.client.cheap_clone(), - timeout: self.timeout, - max_file_size: self.max_file_size.cheap_clone(), - } - } -} - impl ArweaveServiceInner { async fn call_inner(self, req: Base64) -> Result, Error> { self.client diff --git a/core/src/polling_monitor/ipfs_service.rs b/core/src/polling_monitor/ipfs_service.rs index 5364e77c252..6847bb82acd 100644 --- a/core/src/polling_monitor/ipfs_service.rs +++ b/core/src/polling_monitor/ipfs_service.rs @@ -1,28 +1,32 @@ -use anyhow::{anyhow, Error}; -use bytes::Bytes; -use futures::future::BoxFuture; -use graph::{ - ipfs_client::{CidFile, IpfsClient, StatApi}, - prelude::CheapClone, -}; +use std::sync::Arc; use std::time::Duration; -use tower::{buffer::Buffer, ServiceBuilder, ServiceExt}; -const CLOUDFLARE_TIMEOUT: u16 = 524; -const GATEWAY_TIMEOUT: u16 = 504; +use anyhow::Error; +use anyhow::anyhow; +use bytes::Bytes; +use graph::futures03::future::BoxFuture; +use graph::ipfs::{ContentPath, IpfsClient, IpfsContext, RetryPolicy}; +use graph::{derive::CheapClone, prelude::CheapClone}; +use tower::{ServiceBuilder, ServiceExt, buffer::Buffer}; + +pub type IpfsService = Buffer, Error>>>; -pub type IpfsService = Buffer, Error>>>; +#[derive(Debug, Clone, CheapClone)] +pub struct IpfsRequest { + pub ctx: IpfsContext, + pub path: ContentPath, +} pub fn ipfs_service( - client: IpfsClient, - max_file_size: u64, + client: Arc, + max_file_size: usize, timeout: Duration, rate_limit: u16, ) -> IpfsService { let ipfs = IpfsServiceInner { client, - max_file_size, timeout, + max_file_size, }; let svc = ServiceBuilder::new() @@ -35,63 +39,42 @@ pub fn ipfs_service( Buffer::new(svc, u32::MAX as usize) } -#[derive(Clone)] +#[derive(Clone, CheapClone)] struct IpfsServiceInner { - client: IpfsClient, - max_file_size: u64, + client: Arc, timeout: Duration, -} - -impl CheapClone for IpfsServiceInner { - fn cheap_clone(&self) -> Self { - Self { - client: self.client.cheap_clone(), - max_file_size: self.max_file_size, - timeout: self.timeout, - } - } + max_file_size: usize, } impl IpfsServiceInner { - async fn call_inner(self, req: CidFile) -> Result, Error> { - let CidFile { cid, path } = req; - let multihash = cid.hash().code(); + async fn call_inner( + self, + IpfsRequest { ctx, path }: IpfsRequest, + ) -> Result, Error> { + let multihash = path.cid().hash().code(); if !SAFE_MULTIHASHES.contains(&multihash) { return Err(anyhow!("CID multihash {} is not allowed", multihash)); } - let cid_str = match path { - Some(path) => format!("{}/{}", cid, path), - None => cid.to_string(), - }; - - let size = match self + let res = self .client - .stat_size(StatApi::Files, cid_str.clone(), self.timeout) - .await - { - Ok(size) => size, - Err(e) => match e.status().map(|e| e.as_u16()) { - Some(GATEWAY_TIMEOUT) | Some(CLOUDFLARE_TIMEOUT) => return Ok(None), - _ if e.is_timeout() => return Ok(None), - _ => return Err(e.into()), - }, - }; - - if size > self.max_file_size { - return Err(anyhow!( - "IPFS file {} is too large. It can be at most {} bytes but is {} bytes", - cid_str, + .cat( + &ctx, + &path, self.max_file_size, - size - )); + Some(self.timeout), + RetryPolicy::None, + ) + .await; + + match res { + Ok(file_bytes) => Ok(Some(file_bytes)), + Err(err) if err.is_timeout() => { + // Timeouts in IPFS mean that the content is not available, so we return `None`. + Ok(None) + } + Err(err) => Err(err.into()), } - - Ok(self - .client - .cat_all(&cid_str, self.timeout) - .await - .map(Some)?) } } @@ -118,56 +101,67 @@ const SAFE_MULTIHASHES: [u64; 15] = [ #[cfg(test)] mod test { - use ipfs::IpfsApi; - use ipfs_api as ipfs; - use std::{fs, str::FromStr, time::Duration}; - use tower::ServiceExt; + use std::time::Duration; - use cid::Cid; - use graph::{ - components::link_resolver::{ArweaveClient, ArweaveResolver}, - data::value::Word, - ipfs_client::IpfsClient, - tokio, - }; + use graph::components::link_resolver::ArweaveClient; + use graph::data::value::Word; + use graph::ipfs::test_utils::add_files_to_local_ipfs_node_for_testing; + use graph::ipfs::{IpfsContext, IpfsMetrics, IpfsRpcClient, ServerAddress}; + use graph::log::discard; + use tower::ServiceExt; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers as m; - use uuid::Uuid; + use super::*; - #[tokio::test] + #[graph::test] async fn cat_file_in_folder() { - let path = "./tests/fixtures/ipfs_folder"; - let uid = Uuid::new_v4().to_string(); - fs::write(format!("{}/random.txt", path), &uid).unwrap(); - - let cl: ipfs::IpfsClient = ipfs::IpfsClient::default(); - - let rsp = cl.add_path(path).await.unwrap(); + let random_bytes = "One morning, when Gregor Samsa woke \ + from troubled dreams, he found himself transformed in his bed \ + into a horrible vermin" + .as_bytes() + .to_vec(); + let ipfs_file = ("dir/file.txt", random_bytes.clone()); + + let add_resp = add_files_to_local_ipfs_node_for_testing([ipfs_file]) + .await + .unwrap(); - let ipfs_folder = rsp.iter().find(|rsp| rsp.name == "ipfs_folder").unwrap(); + let dir_cid = add_resp.into_iter().find(|x| x.name == "dir").unwrap().hash; - let local = IpfsClient::localhost(); - let cid = Cid::from_str(&ipfs_folder.hash).unwrap(); - let file = "random.txt".to_string(); + let client = IpfsRpcClient::new_unchecked( + ServerAddress::test_rpc_api(), + IpfsMetrics::test(), + &graph::log::discard(), + ) + .unwrap(); - let svc = super::ipfs_service(local, 100000, Duration::from_secs(5), 10); + let svc = ipfs_service(Arc::new(client), 100000, Duration::from_secs(30), 10); + let path = ContentPath::new(format!("{dir_cid}/file.txt")).unwrap(); let content = svc - .oneshot(super::CidFile { - cid, - path: Some(file), + .oneshot(IpfsRequest { + ctx: IpfsContext::test(), + path, }) .await .unwrap() .unwrap(); - assert_eq!(content.to_vec(), uid.as_bytes().to_vec()); + + assert_eq!(content.to_vec(), random_bytes); } - #[tokio::test] + // Ignored because arweave.net is unreliable right now + #[graph::test] + #[ignore] async fn arweave_get() { const ID: &str = "8APeQ5lW0-csTcBaGdPBDLAL2ci2AT9pTn2tppGPU_8"; - let cl = ArweaveClient::default(); - let body = cl.get(&Word::from(ID)).await.unwrap(); + let Some(body) = ArweaveClient::get_test(&Word::from(ID)).await else { + return; + }; let body = String::from_utf8(body).unwrap(); let expected = r#" @@ -175,4 +169,41 @@ mod test { "#.trim_start().trim_end(); assert_eq!(expected, body); } + + #[graph::test] + async fn no_client_retries_to_allow_polling_monitor_to_handle_retries_internally() { + const CID: &str = "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"; + + let server = MockServer::start().await; + let ipfs_client = + IpfsRpcClient::new_unchecked(server.uri(), IpfsMetrics::test(), &discard()).unwrap(); + let ipfs_service = ipfs_service(Arc::new(ipfs_client), 10, Duration::from_secs(1), 1); + let path = ContentPath::new(CID).unwrap(); + + Mock::given(m::method("POST")) + .and(m::path("/api/v0/cat")) + .and(m::query_param("arg", CID)) + .respond_with(ResponseTemplate::new(500)) + .up_to_n_times(1) + .expect(1) + .mount(&server) + .await; + + Mock::given(m::method("POST")) + .and(m::path("/api/v0/cat")) + .and(m::query_param("arg", CID)) + .respond_with(ResponseTemplate::new(200)) + .expect(..=1) + .mount(&server) + .await; + + // This means that we never reached the successful response. + ipfs_service + .oneshot(IpfsRequest { + ctx: IpfsContext::test(), + path, + }) + .await + .unwrap_err(); + } } diff --git a/core/src/polling_monitor/mod.rs b/core/src/polling_monitor/mod.rs index 3f43254dcad..f8a24db97d7 100644 --- a/core/src/polling_monitor/mod.rs +++ b/core/src/polling_monitor/mod.rs @@ -1,6 +1,7 @@ mod arweave_service; mod ipfs_service; mod metrics; +mod request; use std::collections::HashMap; use std::fmt::Display; @@ -9,28 +10,29 @@ use std::sync::Arc; use std::task::Poll; use std::time::Duration; -use futures::future::BoxFuture; -use futures::stream::StreamExt; -use futures::{stream, Future, FutureExt, TryFutureExt}; use graph::cheap_clone::CheapClone; +use graph::env::ENV_VARS; +use graph::futures03::future::BoxFuture; +use graph::futures03::stream::StreamExt; +use graph::futures03::{Future, FutureExt, TryFutureExt, stream}; use graph::parking_lot::Mutex; use graph::prelude::tokio; use graph::prometheus::{Counter, Gauge}; -use graph::slog::{debug, Logger}; +use graph::slog::{Logger, debug}; use graph::util::monitored::MonitoredVecDeque as VecDeque; use tokio::sync::{mpsc, watch}; use tower::retry::backoff::{Backoff, ExponentialBackoff, ExponentialBackoffMaker, MakeBackoff}; use tower::util::rng::HasherRng; use tower::{Service, ServiceExt}; +use self::request::RequestId; + pub use self::metrics::PollingMonitorMetrics; -pub use arweave_service::{arweave_service, ArweaveService}; -pub use ipfs_service::{ipfs_service, IpfsService}; +pub use arweave_service::{ArweaveService, arweave_service}; +pub use ipfs_service::{IpfsRequest, IpfsService, ipfs_service}; const MIN_BACKOFF: Duration = Duration::from_secs(5); -const MAX_BACKOFF: Duration = Duration::from_secs(600); - struct Backoffs { backoff_maker: ExponentialBackoffMaker, backoffs: HashMap, @@ -42,7 +44,7 @@ impl Backoffs { Self { backoff_maker: ExponentialBackoffMaker::new( MIN_BACKOFF, - MAX_BACKOFF, + ENV_VARS.mappings.fds_max_backoff, 1.0, HasherRng::new(), ) @@ -51,7 +53,7 @@ impl Backoffs { } } - fn next_backoff(&mut self, id: ID) -> impl Future { + fn next_backoff(&mut self, id: ID) -> impl Future + Send + use { self.backoffs .entry(id) .or_insert_with(|| self.backoff_maker.make_backoff()) @@ -98,15 +100,15 @@ impl Queue { /// /// The service returns the request ID along with errors or responses. The response is an /// `Option`, to represent the object not being found. -pub fn spawn_monitor( +pub fn spawn_monitor( service: S, - response_sender: mpsc::UnboundedSender<(ID, Res)>, + response_sender: mpsc::UnboundedSender<(Req, Res)>, logger: Logger, metrics: Arc, -) -> PollingMonitor +) -> PollingMonitor where - S: Service, Error = E> + Send + 'static, - ID: Display + Clone + Default + Eq + Send + Sync + Hash + 'static, + S: Service, Error = E> + Send + 'static, + Req: RequestId + Clone + Send + Sync + 'static, E: Display + Send + 'static, S::Future: Send, { @@ -126,13 +128,13 @@ where break None; } - let id = queue.pop_front(); - match id { - Some(id) => break Some((id, ())), + let req = queue.pop_front(); + match req { + Some(req) => break Some((req, ())), // Nothing on the queue, wait for a queue wake up or cancellation. None => { - futures::future::select( + graph::futures03::future::select( // Unwrap: `queue` holds a sender. queue_woken.changed().map(|r| r.unwrap()).boxed(), cancel_check.closed().boxed(), @@ -155,39 +157,39 @@ where // the `CallAll` from being polled. This can cause starvation as those requests may // be holding on to resources such as slots for concurrent calls. match response { - Ok((id, Some(response))) => { - backoffs.remove(&id); - let send_result = response_sender.send((id, response)); + Ok((req, Some(response))) => { + backoffs.remove(req.request_id()); + let send_result = response_sender.send((req, response)); if send_result.is_err() { // The receiver has been dropped, cancel this task. break; } } - // Object not found, push the id to the back of the queue. - Ok((id, None)) => { - debug!(logger, "not found on polling"; "object_id" => id.to_string()); - + // Object not found, push the request to the back of the queue. + Ok((req, None)) => { + debug!(logger, "not found on polling"; "object_id" => req.request_id().to_string()); metrics.not_found.inc(); - queue.push_back(id); + + // We'll try again after a backoff. + backoff(req, &queue, &mut backoffs); } - // Error polling, log it and push the id to the back of the queue. - Err((id, e)) => { - debug!(logger, "error polling"; - "error" => format!("{:#}", e), - "object_id" => id.to_string()); + // Error polling, log it and push the request to the back of the queue. + Err((Some(req), e)) => { + debug!(logger, "error polling"; "error" => format!("{:#}", e), "object_id" => req.request_id().to_string()); metrics.errors.inc(); // Requests that return errors could mean there is a permanent issue with // fetching the given item, or could signal the endpoint is overloaded. // Either way a backoff makes sense. - let queue = queue.cheap_clone(); - let backoff = backoffs.next_backoff(id.clone()); - graph::spawn(async move { - backoff.await; - queue.push_back(id); - }); + backoff(req, &queue, &mut backoffs); + } + + // poll_ready call failure + Err((None, e)) => { + debug!(logger, "error polling"; "error" => format!("{:#}", e)); + metrics.errors.inc(); } } } @@ -197,16 +199,28 @@ where PollingMonitor { queue } } +fn backoff(req: Req, queue: &Arc>, backoffs: &mut Backoffs) +where + Req: RequestId + Send + Sync + 'static, +{ + let queue = queue.cheap_clone(); + let backoff = backoffs.next_backoff(req.request_id().clone()); + graph::spawn(async move { + backoff.await; + queue.push_back(req); + }); +} + /// Handle for adding objects to be monitored. -pub struct PollingMonitor { - queue: Arc>, +pub struct PollingMonitor { + queue: Arc>, } -impl PollingMonitor { - /// Add an object id to the polling queue. New requests have priority and are pushed to the +impl PollingMonitor { + /// Add a request to the polling queue. New requests have priority and are pushed to the /// front of the queue. - pub fn monitor(&self, id: ID) { - self.queue.push_front(id); + pub fn monitor(&self, req: Req) { + self.queue.push_front(req); } } @@ -217,17 +231,16 @@ struct ReturnRequest { impl Service for ReturnRequest where S: Service, - Req: Clone + Default + Send + Sync + 'static, + Req: Clone + Send + Sync + 'static, S::Error: Send, S::Future: Send + 'static, { type Response = (Req, S::Response); - type Error = (Req, S::Error); + type Error = (Option, S::Error); type Future = BoxFuture<'static, Result>; fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - // `Req::default` is a value that won't be used since if `poll_ready` errors, the service is shot anyways. - self.service.poll_ready(cx).map_err(|e| (Req::default(), e)) + self.service.poll_ready(cx).map_err(|e| (None, e)) } fn call(&mut self, req: Req) -> Self::Future { @@ -235,7 +248,7 @@ where self.service .call(req.clone()) .map_ok(move |x| (req, x)) - .map_err(move |e| (req1, e)) + .map_err(move |e| (Some(req1), e)) .boxed() } } @@ -268,7 +281,7 @@ mod tests { (handle, monitor, rx) } - #[tokio::test] + #[graph::test] async fn polling_monitor_shared_svc() { let (svc, mut handle) = mock::pair(); let shared_svc = tower::buffer::Buffer::new(tower::limit::ConcurrencyLimit::new(svc, 1), 1); @@ -290,7 +303,7 @@ mod tests { assert_eq!(rx1.recv().await, Some(("req-0", "res-0"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_simple() { let (mut handle, monitor, mut rx) = setup(); @@ -300,7 +313,7 @@ mod tests { assert_eq!(rx.recv().await, Some(("req-0", "res-0"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_unordered() { let (mut handle, monitor, mut rx) = setup(); @@ -317,7 +330,7 @@ mod tests { assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_failed_push_to_back() { let (mut handle, monitor, mut rx) = setup(); @@ -341,8 +354,10 @@ mod tests { assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_cancelation() { + const REQ: &str = "req-0"; + // Cancelation on receiver drop, no pending request. let (mut handle, _monitor, rx) = setup(); drop(rx); @@ -350,9 +365,20 @@ mod tests { // Cancelation on receiver drop, with pending request. let (mut handle, monitor, rx) = setup(); - monitor.monitor("req-0"); + monitor.monitor(REQ); drop(rx); - assert!(handle.next_request().await.is_none()); + let mut next = handle.next_request().await; + if let Some((req, responder)) = next { + // The request may or may not have been pulled from the queue + // before cancelation. + assert_eq!(REQ, req); + // Explicitly complete the request so the monitor task can process it + // and detect the cancellation. Dropping the responder without responding + // is racy in a multi-threaded runtime. + responder.send_error(anyhow!("cancelled")); + next = handle.next_request().await; + } + assert!(next.is_none()); // Cancelation on receiver drop, while queue is waiting. let (mut handle, _monitor, rx) = setup(); diff --git a/core/src/polling_monitor/request.rs b/core/src/polling_monitor/request.rs new file mode 100644 index 00000000000..42375fb38fb --- /dev/null +++ b/core/src/polling_monitor/request.rs @@ -0,0 +1,39 @@ +use std::fmt::Display; +use std::hash::Hash; + +use graph::{data_source::offchain::Base64, ipfs::ContentPath}; + +use crate::polling_monitor::ipfs_service::IpfsRequest; + +/// Request ID is used to create backoffs on request failures. +pub trait RequestId { + type Id: Clone + Display + Eq + Hash + Send + Sync + 'static; + + /// Returns the ID of the request. + fn request_id(&self) -> &Self::Id; +} + +impl RequestId for IpfsRequest { + type Id = ContentPath; + + fn request_id(&self) -> &ContentPath { + &self.path + } +} + +impl RequestId for Base64 { + type Id = Base64; + + fn request_id(&self) -> &Base64 { + self + } +} + +#[cfg(debug_assertions)] +impl RequestId for &'static str { + type Id = &'static str; + + fn request_id(&self) -> &Self::Id { + self + } +} diff --git a/core/src/subgraph/context/instance/hosts.rs b/core/src/subgraph/context/instance/hosts.rs index 73701fadb29..9c18e12ce1e 100644 --- a/core/src/subgraph/context/instance/hosts.rs +++ b/core/src/subgraph/context/instance/hosts.rs @@ -57,7 +57,7 @@ impl> OnchainHosts { } pub fn push(&mut self, host: Arc) { - assert!(host.data_source().as_onchain().is_some()); + assert!(host.data_source().is_chain_based()); self.hosts.push(host.cheap_clone()); let idx = self.hosts.len() - 1; @@ -194,7 +194,7 @@ impl> OffchainHosts { pub fn matches_by_address<'a>( &'a self, address: Option<&[u8]>, - ) -> Box + Send + 'a> { + ) -> Box + Send + 'a> { let Some(address) = address else { return Box::new(self.by_block.values().flatten().map(|host| host.as_ref())); }; diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index 26ccad08efa..494ae227e97 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -1,19 +1,19 @@ mod hosts; use anyhow::ensure; -use futures01::sync::mpsc::Sender; +use graph::futures01::sync::mpsc::Sender; use graph::{ blockchain::{Blockchain, TriggerData as _}, data_source::{ - causality_region::CausalityRegionSeq, offchain, CausalityRegion, DataSource, - DataSourceTemplate, TriggerData, + CausalityRegion, DataSource, DataSourceTemplate, TriggerData, + causality_region::CausalityRegionSeq, offchain, }, prelude::*, }; use hosts::{OffchainHosts, OnchainHosts}; use std::collections::HashMap; -pub(super) struct SubgraphInstance> { +pub(crate) struct SubgraphInstance> { subgraph_id: DeploymentHash, network: String, host_builder: T, @@ -22,13 +22,17 @@ pub(super) struct SubgraphInstance> { pub(super) static_data_sources: Arc>>, host_metrics: Arc, - /// The hosts represent the data sources in the subgraph. There is one host per data source. + /// The hosts represent the onchain data sources in the subgraph. There is one host per data source. /// Data sources with no mappings (e.g. direct substreams) have no host. /// /// Onchain hosts must be created in increasing order of block number. `fn hosts_for_trigger` /// will return the onchain hosts in the same order as they were inserted. onchain_hosts: OnchainHosts, + /// `subgraph_hosts` represent subgraph data sources declared in the manifest. These are a special + /// kind of data source that depends on the data from another source subgraph. + subgraph_hosts: OnchainHosts, + offchain_hosts: OffchainHosts, /// Maps the hash of a module to a channel to the thread in which the module is instantiated. @@ -79,6 +83,7 @@ where network, static_data_sources: Arc::new(manifest.data_sources), onchain_hosts: OnchainHosts::new(), + subgraph_hosts: OnchainHosts::new(), offchain_hosts: OffchainHosts::new(), module_cache: HashMap::new(), templates, @@ -95,11 +100,11 @@ where ) -> Result>, Error> { let module_bytes = match &data_source.runtime() { None => return Ok(None), - Some(ref module_bytes) => module_bytes.cheap_clone(), + Some(module_bytes) => module_bytes.cheap_clone(), }; let mapping_request_sender = { - let module_hash = tiny_keccak::keccak256(module_bytes.as_ref()); + let module_hash = alloy::primitives::keccak256(module_bytes.as_ref()).0; if let Some(sender) = self.module_cache.get(&module_hash) { sender.clone() } else { @@ -138,33 +143,46 @@ where ); } - let is_onchain = data_source.is_onchain(); - let Some(host) = self.new_host(logger.clone(), data_source)? else { return Ok(None) }; + let Some(host) = self.new_host(logger.clone(), data_source)? else { + return Ok(None); + }; // Check for duplicates and add the host. - if is_onchain { - // `onchain_hosts` will remain ordered by the creation block. - // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. - ensure!( - self.onchain_hosts - .last() - .and_then(|h| h.creation_block_number()) - <= host.data_source().creation_block(), - ); + match host.data_source() { + DataSource::Onchain(_) => { + // `onchain_hosts` will remain ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + ensure!( + self.onchain_hosts + .last() + .and_then(|h| h.creation_block_number()) + <= host.data_source().creation_block(), + ); - if self.onchain_hosts.contains(&host) { - Ok(None) - } else { - self.onchain_hosts.push(host.cheap_clone()); - Ok(Some(host)) + if self.onchain_hosts.contains(&host) { + Ok(None) + } else { + self.onchain_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } } - } else { - if self.offchain_hosts.contains(&host) { - Ok(None) - } else { - self.offchain_hosts.push(host.cheap_clone()); - Ok(Some(host)) + DataSource::Offchain(_) => { + if self.offchain_hosts.contains(&host) { + Ok(None) + } else { + self.offchain_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } + } + DataSource::Subgraph(_) => { + if self.subgraph_hosts.contains(&host) { + Ok(None) + } else { + self.subgraph_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } } + DataSource::Amp(_) => unreachable!(), } } @@ -223,7 +241,10 @@ where .matches_by_address(trigger.address_match()), TriggerData::Offchain(trigger) => self .offchain_hosts - .matches_by_address(trigger.source.address().as_ref().map(|a| a.as_slice())), + .matches_by_address(trigger.source.address().as_deref()), + TriggerData::Subgraph(trigger) => self + .subgraph_hosts + .matches_by_address(Some(trigger.source.to_bytes().as_slice())), } } @@ -234,8 +255,4 @@ where pub fn hosts_len(&self) -> usize { self.onchain_hosts.len() + self.offchain_hosts.len() } - - pub fn first_host(&self) -> Option<&Arc> { - self.onchain_hosts.hosts().first() - } } diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index 51238411d4a..affbc09aa0f 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -1,44 +1,42 @@ mod instance; use crate::polling_monitor::{ - spawn_monitor, ArweaveService, IpfsService, PollingMonitor, PollingMonitorMetrics, + ArweaveService, IpfsRequest, IpfsService, PollingMonitor, PollingMonitorMetrics, spawn_monitor, }; use anyhow::{self, Error}; use bytes::Bytes; use graph::{ - blockchain::{BlockTime, Blockchain}, - components::{ - store::{DeploymentId, SubgraphFork}, - subgraph::{HostMetrics, MappingError, RuntimeHost as _, SharedProofOfIndexing}, - }, + blockchain::Blockchain, + components::{store::DeploymentId, subgraph::HostMetrics}, data::subgraph::SubgraphManifest, data_source::{ + CausalityRegion, DataSource, DataSourceTemplate, causality_region::CausalityRegionSeq, offchain::{self, Base64}, - CausalityRegion, DataSource, DataSourceTemplate, TriggerData, }, - ipfs_client::CidFile, + derive::CheapClone, + ipfs::IpfsContext, prelude::{ - BlockNumber, BlockPtr, BlockState, CancelGuard, CheapClone, DeploymentHash, - MetricsRegistry, RuntimeHostBuilder, SubgraphCountMetric, SubgraphInstanceMetrics, - TriggerProcessor, + BlockNumber, CancelGuard, CheapClone, DeploymentHash, MetricsRegistry, RuntimeHostBuilder, + SubgraphCountMetric, TriggerProcessor, }, slog::Logger, - tokio::sync::mpsc, }; -use std::sync::{Arc, RwLock}; -use std::{collections::HashMap, time::Instant}; +use std::collections::HashMap; +use std::sync::Arc; + +use graph::parking_lot::RwLock; +use tokio::sync::mpsc; use self::instance::SubgraphInstance; +use super::Decoder; -#[derive(Clone, Debug)] +#[derive(Clone, CheapClone, Debug)] pub struct SubgraphKeepAlive { alive_map: Arc>>, sg_metrics: Arc, } -impl CheapClone for SubgraphKeepAlive {} - impl SubgraphKeepAlive { pub fn new(sg_metrics: Arc) -> Self { Self { @@ -48,15 +46,19 @@ impl SubgraphKeepAlive { } pub fn remove(&self, deployment_id: &DeploymentId) { - self.alive_map.write().unwrap().remove(deployment_id); + self.alive_map.write().remove(deployment_id); self.sg_metrics.running_count.dec(); } pub fn insert(&self, deployment_id: DeploymentId, guard: CancelGuard) { - let old = self.alive_map.write().unwrap().insert(deployment_id, guard); + let old = self.alive_map.write().insert(deployment_id, guard); if old.is_none() { self.sg_metrics.running_count.inc(); } } + + pub fn contains(&self, deployment_id: &DeploymentId) -> bool { + self.alive_map.read().contains_key(deployment_id) + } } // The context keeps track of mutable in-memory state that is retained across blocks. @@ -69,11 +71,11 @@ where T: RuntimeHostBuilder, C: Blockchain, { - instance: SubgraphInstance, + pub(crate) instance: SubgraphInstance, pub instances: SubgraphKeepAlive, pub offchain_monitor: OffchainMonitor, - pub filter: Option, - trigger_processor: Box>, + pub(crate) trigger_processor: Box>, + pub(crate) decoder: Box>, } impl> IndexingContext { @@ -85,6 +87,7 @@ impl> IndexingContext { instances: SubgraphKeepAlive, offchain_monitor: OffchainMonitor, trigger_processor: Box>, + decoder: Box>, ) -> Self { let instance = SubgraphInstance::new( manifest, @@ -97,127 +100,11 @@ impl> IndexingContext { instance, instances, offchain_monitor, - filter: None, trigger_processor, + decoder, } } - pub async fn process_trigger( - &self, - logger: &Logger, - block: &Arc, - trigger: &TriggerData, - state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - debug_fork: &Option>, - subgraph_metrics: &Arc, - instrument: bool, - ) -> Result, MappingError> { - self.process_trigger_in_hosts( - logger, - self.instance.hosts_for_trigger(trigger), - block, - trigger, - state, - proof_of_indexing, - causality_region, - debug_fork, - subgraph_metrics, - instrument, - ) - .await - } - - pub async fn process_block( - &self, - logger: &Logger, - block_ptr: BlockPtr, - block_time: BlockTime, - block_data: Box<[u8]>, - handler: String, - mut state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - debug_fork: &Option>, - subgraph_metrics: &Arc, - instrument: bool, - ) -> Result, MappingError> { - let error_count = state.deterministic_errors.len(); - - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } - - let start = Instant::now(); - - // This flow is expected to have a single data source(and a corresponding host) which - // gets executed every block. - state = self - .instance - .first_host() - .expect("Expected this flow to have exactly one host") - .process_block( - logger, - block_ptr, - block_time, - block_data, - handler, - state, - proof_of_indexing.cheap_clone(), - debug_fork, - instrument, - ) - .await?; - - let elapsed = start.elapsed().as_secs_f64(); - subgraph_metrics.observe_trigger_processing_duration(elapsed); - - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); - - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } - } - - Ok(state) - } - pub async fn process_trigger_in_hosts( - &self, - logger: &Logger, - hosts: Box + Send + '_>, - block: &Arc, - trigger: &TriggerData, - state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - debug_fork: &Option>, - subgraph_metrics: &Arc, - instrument: bool, - ) -> Result, MappingError> { - self.trigger_processor - .process_trigger( - logger, - hosts, - block, - trigger, - state, - proof_of_indexing, - causality_region, - debug_fork, - subgraph_metrics, - instrument, - ) - .await - } - /// Removes data sources hosts with a creation block greater or equal to `reverted_block`, so /// that they are no longer candidates for `process_trigger`. /// @@ -227,12 +114,12 @@ impl> IndexingContext { /// `process_trigger`. /// /// File data sources that have been marked not done during this process will get re-queued - pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) -> Result<(), Error> { + pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) { let removed = self.instance.revert_data_sources(reverted_block); removed .into_iter() - .try_for_each(|source| self.offchain_monitor.add_source(source)) + .for_each(|source| self.offchain_monitor.add_source(source)) } pub fn add_dynamic_data_source( @@ -245,12 +132,12 @@ impl> IndexingContext { .map(|ds| (ds.source.clone(), ds.is_processed())); let host = self.instance.add_dynamic_data_source(logger, data_source)?; - if host.is_some() { - if let Some((source, is_processed)) = offchain_fields { - // monitor data source only if it has not yet been processed. - if !is_processed { - self.offchain_monitor.add_source(source)?; - } + if host.is_some() + && let Some((source, is_processed)) = offchain_fields + { + // monitor data source only if it has not yet been processed. + if !is_processed { + self.offchain_monitor.add_source(source); } } @@ -279,10 +166,12 @@ impl> IndexingContext { } pub struct OffchainMonitor { - ipfs_monitor: PollingMonitor, - ipfs_monitor_rx: mpsc::UnboundedReceiver<(CidFile, Bytes)>, + ipfs_monitor: PollingMonitor, + ipfs_monitor_rx: mpsc::UnboundedReceiver<(IpfsRequest, Bytes)>, arweave_monitor: PollingMonitor, arweave_monitor_rx: mpsc::UnboundedReceiver<(Base64, Bytes)>, + deployment_hash: DeploymentHash, + logger: Logger, } impl OffchainMonitor { @@ -306,31 +195,41 @@ impl OffchainMonitor { metrics.cheap_clone(), ); - let arweave_monitor = spawn_monitor(arweave_service, arweave_monitor_tx, logger, metrics); + let arweave_monitor = spawn_monitor( + arweave_service, + arweave_monitor_tx, + logger.cheap_clone(), + metrics, + ); + Self { ipfs_monitor, ipfs_monitor_rx, arweave_monitor, arweave_monitor_rx, + deployment_hash: subgraph_hash.to_owned(), + logger, } } - fn add_source(&mut self, source: offchain::Source) -> Result<(), Error> { + fn add_source(&mut self, source: offchain::Source) { match source { - offchain::Source::Ipfs(cid_file) => self.ipfs_monitor.monitor(cid_file), + offchain::Source::Ipfs(path) => self.ipfs_monitor.monitor(IpfsRequest { + ctx: IpfsContext::new(&self.deployment_hash, &self.logger), + path, + }), offchain::Source::Arweave(base64) => self.arweave_monitor.monitor(base64), }; - Ok(()) } pub fn ready_offchain_events(&mut self) -> Result, Error> { - use graph::tokio::sync::mpsc::error::TryRecvError; + use tokio::sync::mpsc::error::TryRecvError; let mut triggers = vec![]; loop { match self.ipfs_monitor_rx.try_recv() { - Ok((cid_file, data)) => triggers.push(offchain::TriggerData { - source: offchain::Source::Ipfs(cid_file), + Ok((req, data)) => triggers.push(offchain::TriggerData { + source: offchain::Source::Ipfs(req.path), data: Arc::new(data), }), Err(TryRecvError::Disconnected) => { diff --git a/core/src/subgraph/error.rs b/core/src/subgraph/error.rs index b3131255aed..00413acb258 100644 --- a/core/src/subgraph/error.rs +++ b/core/src/subgraph/error.rs @@ -1,28 +1,191 @@ use graph::data::subgraph::schema::SubgraphError; -use graph::prelude::{thiserror, Error, StoreError}; +use graph::env::ENV_VARS; +use graph::prelude::{Error, StoreError, anyhow, thiserror}; +pub trait DeterministicError: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static {} + +impl DeterministicError for SubgraphError {} + +impl DeterministicError for StoreError {} + +impl DeterministicError for anyhow::Error {} + +/// Classification of processing errors for unified error handling. +/// +/// This enum provides a consistent way to categorize errors and determine +/// the appropriate response. The error handling invariants are: +/// +/// - **Deterministic**: Stop processing the current block, persist PoI only. +/// The subgraph will be marked as failed. These errors are reproducible +/// and indicate a bug in the subgraph or a permanent data issue. +/// +/// - **NonDeterministic**: Retry with exponential backoff. These errors are +/// transient (network issues, temporary database problems) and may succeed +/// on retry. +/// +/// - **PossibleReorg**: Restart the block stream cleanly without persisting. +/// The block stream needs to be restarted to detect and handle a potential +/// blockchain reorganization. +/// +/// - **Canceled**: The subgraph was canceled (unassigned or shut down). +/// No error should be recorded; this is a clean shutdown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessingErrorKind { + /// Error is deterministic - stop processing, persist PoI only + Deterministic, + /// Error is non-deterministic - retry with backoff + NonDeterministic, + /// Possible blockchain reorg detected - restart block stream cleanly + PossibleReorg, + /// Processing was canceled - clean shutdown + Canceled, +} + +/// An error happened during processing and we need to classify errors into +/// deterministic and non-deterministic errors. This struct holds the result +/// of that classification #[derive(thiserror::Error, Debug)] -pub enum BlockProcessingError { +pub enum ProcessingError { #[error("{0:#}")] - Unknown(#[from] Error), + Unknown(Error), // The error had a deterministic cause but, for a possibly non-deterministic reason, we chose to // halt processing due to the error. #[error("{0}")] - Deterministic(SubgraphError), + Deterministic(Box), + + /// A possible blockchain reorganization was detected. + /// The block stream should be restarted to detect and handle the reorg. + #[error("possible reorg detected: {0:#}")] + PossibleReorg(Error), #[error("subgraph stopped while processing triggers")] Canceled, } -impl BlockProcessingError { +impl ProcessingError { + /// Classify the error into one of the defined error kinds. + /// + /// This method provides a unified way to determine how to handle an error: + /// - `Deterministic`: Stop processing, persist PoI only + /// - `NonDeterministic`: Retry with backoff + /// - `PossibleReorg`: Restart block stream cleanly + /// - `Canceled`: Clean shutdown, no error recording + pub fn kind(&self) -> ProcessingErrorKind { + match self { + ProcessingError::Unknown(_) => ProcessingErrorKind::NonDeterministic, + ProcessingError::Deterministic(_) => ProcessingErrorKind::Deterministic, + ProcessingError::PossibleReorg(_) => ProcessingErrorKind::PossibleReorg, + ProcessingError::Canceled => ProcessingErrorKind::Canceled, + } + } + + #[allow(dead_code)] pub fn is_deterministic(&self) -> bool { - matches!(self, BlockProcessingError::Deterministic(_)) + matches!(self.kind(), ProcessingErrorKind::Deterministic) + } + + /// Returns true if this error should stop processing the current block. + /// + /// Deterministic errors stop processing because continuing would produce + /// incorrect results. The PoI is still persisted for debugging purposes. + #[allow(dead_code)] + pub fn should_stop_processing(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Deterministic) + } + + /// Returns true if this error requires a clean restart of the block stream. + /// + /// Possible reorgs require restarting to allow the block stream to detect + /// and properly handle the reorganization. No state should be persisted + /// in this case. + #[allow(dead_code)] + pub fn should_restart(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::PossibleReorg) + } + + /// Returns true if this error is retryable with exponential backoff. + /// + /// Non-deterministic errors (network issues, temporary failures) may + /// succeed on retry and should not immediately fail the subgraph. + #[allow(dead_code)] + pub fn is_retryable(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::NonDeterministic) + } + + /// Returns true if processing was canceled (clean shutdown). + /// + /// Canceled errors indicate the subgraph was unassigned or shut down + /// intentionally and should not be treated as failures. + #[allow(dead_code)] + pub fn is_canceled(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Canceled) } + + pub fn detail(self, ctx: &str) -> ProcessingError { + match self { + ProcessingError::Unknown(e) => { + let x = e.context(ctx.to_string()); + ProcessingError::Unknown(x) + } + ProcessingError::Deterministic(e) => { + ProcessingError::Deterministic(Box::new(anyhow!("{e}").context(ctx.to_string()))) + } + ProcessingError::PossibleReorg(e) => { + ProcessingError::PossibleReorg(e.context(ctx.to_string())) + } + ProcessingError::Canceled => ProcessingError::Canceled, + } + } +} + +/// Similar to `anyhow::Context`, but for `Result`. We +/// call the method `detail` to avoid ambiguity with anyhow's `context` +/// method +pub trait DetailHelper { + fn detail(self, ctx: &str) -> Result; +} + +impl DetailHelper for Result { + fn detail(self, ctx: &str) -> Result { + self.map_err(|e| e.detail(ctx)) + } +} + +/// Implement this for errors that are always non-deterministic. +pub(crate) trait NonDeterministicErrorHelper { + fn non_deterministic(self) -> Result; +} + +impl NonDeterministicErrorHelper for Result { + fn non_deterministic(self) -> Result { + self.map_err(ProcessingError::Unknown) + } +} + +impl NonDeterministicErrorHelper for Result { + fn non_deterministic(self) -> Result { + self.map_err(|e| ProcessingError::Unknown(Error::from(e))) + } +} + +/// Implement this for errors where it depends on the details whether they +/// are deterministic or not. +pub(crate) trait ClassifyErrorHelper { + fn classify(self) -> Result; } -impl From for BlockProcessingError { - fn from(e: StoreError) -> Self { - BlockProcessingError::Unknown(e.into()) +impl ClassifyErrorHelper for Result { + fn classify(self) -> Result { + self.map_err(|e| { + if ENV_VARS.mappings.store_errors_are_nondeterministic { + // Old behavior, just in case the new behavior causes issues + ProcessingError::Unknown(Error::from(e)) + } else if e.is_deterministic() { + ProcessingError::Deterministic(Box::new(e)) + } else { + ProcessingError::Unknown(Error::from(e)) + } + }) } } diff --git a/core/src/subgraph/inputs.rs b/core/src/subgraph/inputs.rs index b2e95c753f5..03a8cec29bc 100644 --- a/core/src/subgraph/inputs.rs +++ b/core/src/subgraph/inputs.rs @@ -1,7 +1,7 @@ use graph::{ - blockchain::{Blockchain, TriggersAdapter}, + blockchain::{Blockchain, block_stream::TriggersAdapterWrapper}, components::{ - store::{DeploymentLocator, SubgraphFork, WritableStore}, + store::{DeploymentLocator, SourceableStore, SubgraphFork, WritableStore}, subgraph::ProofOfIndexingVersion, }, data::subgraph::{SubgraphFeature, UnifiedMappingApiVersion}, @@ -16,10 +16,12 @@ pub struct IndexingInputs { pub features: BTreeSet, pub start_blocks: Vec, pub end_blocks: BTreeSet, + pub source_subgraph_stores: Vec>, pub stop_block: Option, + pub max_end_block: Option, pub store: Arc, pub debug_fork: Option>, - pub triggers_adapter: Arc>, + pub triggers_adapter: Arc>, pub chain: Arc, pub templates: Arc>>, pub unified_api_version: UnifiedMappingApiVersion, @@ -39,7 +41,9 @@ impl IndexingInputs { features, start_blocks, end_blocks, + source_subgraph_stores, stop_block, + max_end_block, store: _, debug_fork, triggers_adapter, @@ -56,7 +60,9 @@ impl IndexingInputs { features: features.clone(), start_blocks: start_blocks.clone(), end_blocks: end_blocks.clone(), - stop_block: stop_block.clone(), + source_subgraph_stores: source_subgraph_stores.clone(), + stop_block: *stop_block, + max_end_block: *max_end_block, store, debug_fork: debug_fork.clone(), triggers_adapter: triggers_adapter.clone(), @@ -69,4 +75,12 @@ impl IndexingInputs { instrument: *instrument, } } + + pub fn errors_are_non_fatal(&self) -> bool { + self.features.contains(&SubgraphFeature::NonFatalErrors) + } + + pub fn errors_are_fatal(&self) -> bool { + !self.features.contains(&SubgraphFeature::NonFatalErrors) + } } diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index d0176238013..ea301ffc723 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -1,28 +1,38 @@ +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + use crate::polling_monitor::{ArweaveService, IpfsService}; +use crate::subgraph::Decoder; use crate::subgraph::context::{IndexingContext, SubgraphKeepAlive}; use crate::subgraph::inputs::IndexingInputs; use crate::subgraph::loader::load_dynamic_data_sources; use std::collections::BTreeSet; use crate::subgraph::runner::SubgraphRunner; -use graph::blockchain::block_stream::BlockStreamMetrics; +use async_trait::async_trait; +use graph::amp; +use graph::blockchain::block_stream::{BlockStreamMetrics, TriggersAdapterWrapper}; use graph::blockchain::{Blockchain, BlockchainKind, DataSource, NodeCapabilities}; use graph::components::metrics::gas::GasMetrics; +use graph::components::metrics::subgraph::DeploymentStatusMetric; +use graph::components::store::SourceableStore; use graph::components::subgraph::ProofOfIndexingVersion; -use graph::data::subgraph::{UnresolvedSubgraphManifest, SPEC_VERSION_0_0_6}; +use graph::data::subgraph::{SPEC_VERSION_0_0_6, UnresolvedSubgraphManifest}; +use graph::data::value::Word; use graph::data_source::causality_region::CausalityRegionSeq; use graph::env::EnvVars; use graph::prelude::{SubgraphInstanceManager as SubgraphInstanceManagerTrait, *}; use graph::{blockchain::BlockchainMap, components::store::DeploymentLocator}; -use graph_runtime_wasm::module::ToAscPtr; use graph_runtime_wasm::RuntimeHostBuilder; +use graph_runtime_wasm::module::ToAscPtr; use tokio::task; -use super::context::OffchainMonitor; use super::SubgraphTriggerProcessor; +use super::context::OffchainMonitor; +use crate::{subgraph::runner::SubgraphRunnerError, subgraph_manifest}; #[derive(Clone)] -pub struct SubgraphInstanceManager { +pub struct SubgraphInstanceManager { logger_factory: LoggerFactory, subgraph_store: Arc, chains: Arc, @@ -31,107 +41,87 @@ pub struct SubgraphInstanceManager { link_resolver: Arc, ipfs_service: IpfsService, arweave_service: ArweaveService, + amp_client: Option>, static_filters: bool, env_vars: Arc, + + /// By design, there should be only one subgraph runner process per subgraph, but the current + /// implementation does not completely prevent multiple runners from being active at the same + /// time, and we have already had a [bug][0] due to this limitation. Investigating the problem + /// was quite complicated because there was no way to know that the logs were coming from two + /// different processes because all the logs looked the same. Ideally, the implementation + /// should be refactored to make it more strict, but until then, we keep this counter, which + /// is incremented each time a new runner is started, and the previous count is embedded in + /// each log of the started runner, to make debugging future issues easier. + /// + /// [0]: https://github.com/graphprotocol/graph-node/issues/5452 + subgraph_start_counter: Arc, } #[async_trait] -impl SubgraphInstanceManagerTrait for SubgraphInstanceManager { +impl SubgraphInstanceManagerTrait for SubgraphInstanceManager +where + NC: amp::Client + Send + Sync + 'static, +{ async fn start_subgraph( self: Arc, loc: DeploymentLocator, - manifest: serde_yaml::Mapping, stop_block: Option, ) { + let runner_index = self.subgraph_start_counter.fetch_add(1, Ordering::SeqCst); + let logger = self.logger_factory.subgraph_logger(&loc); + let logger = logger.new(o!("runner_index" => runner_index)); + let err_logger = logger.clone(); let instance_manager = self.cheap_clone(); - let subgraph_start_future = async move { - match BlockchainKind::from_manifest(&manifest)? { - BlockchainKind::Arweave => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Ethereum => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Near => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Cosmos => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Substreams => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.cheap_clone(), - manifest, - stop_block, - Box::new(graph_chain_substreams::TriggerProcessor::new(loc.clone())), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Starknet => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await + let deployment_status_metric = self.new_deployment_status_metric(&loc); + deployment_status_metric.starting(); + + let subgraph_start_future = { + let deployment_status_metric = deployment_status_metric.clone(); + + async move { + let raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*instance_manager.subgraph_store, + &*instance_manager.link_resolver, + &loc.hash, + ) + .await?; + + match BlockchainKind::from_manifest(&raw_manifest)? { + BlockchainKind::Ethereum => { + let runner = instance_manager + .build_subgraph_runner::( + logger.clone(), + self.env_vars.cheap_clone(), + loc.clone(), + raw_manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + deployment_status_metric, + ) + .await?; + + self.start_subgraph_inner(logger, loc, runner).await + } + BlockchainKind::Near => { + let runner = instance_manager + .build_subgraph_runner::( + logger.clone(), + self.env_vars.cheap_clone(), + loc.clone(), + raw_manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + deployment_status_metric, + ) + .await?; + + self.start_subgraph_inner(logger, loc, runner).await + } } } }; @@ -144,12 +134,16 @@ impl SubgraphInstanceManagerTrait for SubgraphInstanceManager< graph::spawn(async move { match subgraph_start_future.await { Ok(()) => {} - Err(err) => error!( - err_logger, - "Failed to start subgraph"; - "error" => format!("{:#}", err), - "code" => LogCode::SubgraphStartFailure - ), + Err(err) => { + deployment_status_metric.failed(); + + error!( + err_logger, + "Failed to start subgraph"; + "error" => format!("{:#}", err), + "code" => LogCode::SubgraphStartFailure + ); + } } }); } @@ -170,7 +164,7 @@ impl SubgraphInstanceManagerTrait for SubgraphInstanceManager< } } -impl SubgraphInstanceManager { +impl SubgraphInstanceManager { pub fn new( logger_factory: &LoggerFactory, env_vars: Arc, @@ -181,6 +175,7 @@ impl SubgraphInstanceManager { link_resolver: Arc, ipfs_service: IpfsService, arweave_service: ArweaveService, + amp_client: Option>, static_filters: bool, ) -> Self { let logger = logger_factory.component_logger("SubgraphInstanceManager", None); @@ -194,20 +189,76 @@ impl SubgraphInstanceManager { instances: SubgraphKeepAlive::new(sg_metrics), link_resolver, ipfs_service, + amp_client, static_filters, env_vars, arweave_service, + subgraph_start_counter: Arc::new(AtomicU64::new(0)), } } + pub async fn get_sourceable_stores( + &self, + hashes: Vec, + is_runner_test: bool, + ) -> anyhow::Result>> { + if is_runner_test { + return Ok(Vec::new()); + } + + let mut sourceable_stores = Vec::new(); + let subgraph_store = self.subgraph_store.clone(); + + for hash in hashes { + let loc = subgraph_store + .active_locator(&hash) + .await? + .ok_or_else(|| anyhow!("no active deployment for hash {}", hash))?; + + let sourceable_store = subgraph_store.clone().sourceable(loc.id).await?; + sourceable_stores.push(sourceable_store); + } + + Ok(sourceable_stores) + } + pub async fn build_subgraph_runner( &self, logger: Logger, env_vars: Arc, deployment: DeploymentLocator, - manifest: serde_yaml::Mapping, + raw_manifest: serde_yaml::Mapping, stop_block: Option, tp: Box>>, + deployment_status_metric: DeploymentStatusMetric, + ) -> anyhow::Result>> + where + C: Blockchain, + ::MappingTrigger: ToAscPtr, + { + self.build_subgraph_runner_inner( + logger, + env_vars, + deployment, + raw_manifest, + stop_block, + tp, + deployment_status_metric, + false, + ) + .await + } + + pub async fn build_subgraph_runner_inner( + &self, + logger: Logger, + env_vars: Arc, + deployment: DeploymentLocator, + raw_manifest: serde_yaml::Mapping, + stop_block: Option, + tp: Box>>, + deployment_status_metric: DeploymentStatusMetric, + is_runner_test: bool, ) -> anyhow::Result>> where C: Blockchain, @@ -216,28 +267,28 @@ impl SubgraphInstanceManager { let subgraph_store = self.subgraph_store.cheap_clone(); let registry = self.metrics_registry.cheap_clone(); - let raw_yaml = serde_yaml::to_string(&manifest).unwrap(); - let manifest = UnresolvedSubgraphManifest::parse(deployment.hash.cheap_clone(), manifest)?; + let manifest = + UnresolvedSubgraphManifest::parse(deployment.hash.cheap_clone(), raw_manifest)?; // Allow for infinite retries for subgraph definition files. - let link_resolver = Arc::from(self.link_resolver.with_retries()); + let link_resolver = Arc::from( + self.link_resolver + .for_manifest(&deployment.hash.to_string()) + .map_err(SubgraphRegistrarError::Unknown)? + .with_retries(), + ); - // Make sure the `raw_yaml` is present on both this subgraph and the graft base. - self.subgraph_store - .set_manifest_raw_yaml(&deployment.hash, raw_yaml) + if let Some(graft) = &manifest.graft + && self.subgraph_store.is_deployed(&graft.base).await? + { + // Makes sure the raw manifest is cached in the subgraph store + let _raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*self.subgraph_store, + &*self.link_resolver, + &graft.base, + ) .await?; - if let Some(graft) = &manifest.graft { - if self.subgraph_store.is_deployed(&graft.base)? { - let file_bytes = self - .link_resolver - .cat(&logger, &graft.base.to_ipfs_link()) - .await?; - let yaml = String::from_utf8(file_bytes)?; - - self.subgraph_store - .set_manifest_raw_yaml(&graft.base, yaml) - .await?; - } } info!(logger, "Resolve subgraph files using IPFS"; @@ -246,7 +297,13 @@ impl SubgraphInstanceManager { ); let manifest = manifest - .resolve(&link_resolver, &logger, ENV_VARS.max_spec_version.clone()) + .resolve( + &deployment.hash, + &link_resolver, + self.amp_client.cheap_clone(), + &logger, + ENV_VARS.max_spec_version.clone(), + ) .await?; { @@ -281,7 +338,8 @@ impl SubgraphInstanceManager { // Write it to the database let deployment_features = manifest.deployment_features(); self.subgraph_store - .create_subgraph_features(deployment_features)?; + .create_subgraph_features(deployment_features) + .await?; // Start the subgraph deployment before reading dynamic data // sources; if the subgraph is a graft or a copy, starting it will @@ -305,8 +363,18 @@ impl SubgraphInstanceManager { .filter_map(|d| d.as_onchain().cloned()) .collect::>(); + let subgraph_data_sources = data_sources + .iter() + .filter_map(|d| d.as_subgraph()) + .collect::>(); + + let subgraph_ds_source_deployments = subgraph_data_sources + .iter() + .map(|d| d.source.address()) + .collect::>(); + let required_capabilities = C::NodeCapabilities::from_data_sources(&onchain_data_sources); - let network = manifest.network_name(); + let network: Word = manifest.network_name().into(); let chain = self .chains @@ -316,25 +384,42 @@ impl SubgraphInstanceManager { let start_blocks: Vec = data_sources .iter() - .filter_map(|d| d.as_onchain().map(|d: &C::DataSource| d.start_block())) + .filter_map(|d| d.start_block()) .collect(); let end_blocks: BTreeSet = manifest .data_sources .iter() - .filter_map(|d| { - d.as_onchain() - .map(|d: &C::DataSource| d.end_block()) - .flatten() - }) + .filter_map(|d| d.as_onchain().and_then(|d: &C::DataSource| d.end_block())) .collect(); + // We can set `max_end_block` to the maximum of `end_blocks` and stop the subgraph + // only when there are no dynamic data sources and no offchain data sources present. This is because: + // - Dynamic data sources do not have a defined `end_block`, so we can't determine + // when to stop processing them. + // - Offchain data sources might require processing beyond the end block of + // onchain data sources, so the subgraph needs to continue. + // + // Note: we explicitly check each data source rather than comparing lengths, because + // `end_blocks` is a BTreeSet and deduplicates equal values, which would cause the + // length comparison to fail when multiple data sources share the same `end_block`. + let max_end_block: Option = if data_sources.iter().all(|d| { + d.as_onchain() + .and_then(|d: &C::DataSource| d.end_block()) + .is_some() + }) { + end_blocks.iter().max().cloned() + } else { + None + }; + let templates = Arc::new(manifest.templates.clone()); // Obtain the debug fork from the subgraph store let debug_fork = self .subgraph_store - .debug_fork(&deployment.hash, logger.clone())?; + .debug_fork(&deployment.hash, logger.clone()) + .await?; // Create a subgraph instance from the manifest; this moves // ownership of the manifest and host builder into the new instance @@ -366,6 +451,7 @@ impl SubgraphInstanceManager { registry.cheap_clone(), deployment.hash.as_str(), stopwatch_metrics.clone(), + deployment_status_metric, )); let block_stream_metrics = Arc::new(BlockStreamMetrics::new( @@ -389,8 +475,9 @@ impl SubgraphInstanceManager { let deployment_head = store.block_ptr().map(|ptr| ptr.number).unwrap_or(0) as f64; block_stream_metrics.deployment_head.set(deployment_head); + let (runtime_adapter, decoder_hook) = chain.runtime().await?; let host_builder = graph_runtime_wasm::RuntimeHostBuilder::new( - chain.runtime_adapter(), + runtime_adapter, self.link_resolver.cheap_clone(), subgraph_store.ens_lookup(), ); @@ -406,14 +493,27 @@ impl SubgraphInstanceManager { let causality_region_seq = CausalityRegionSeq::from_current(store.causality_region_curr_val().await?); - let instrument = self.subgraph_store.instrument(&deployment)?; + let instrument = self.subgraph_store.instrument(&deployment).await?; + + let decoder = Box::new(Decoder::new(decoder_hook)); + + let subgraph_data_source_stores = self + .get_sourceable_stores::(subgraph_ds_source_deployments, is_runner_test) + .await?; + + let triggers_adapter = Arc::new(TriggersAdapterWrapper::new( + triggers_adapter, + subgraph_data_source_stores.clone(), + )); let inputs = IndexingInputs { deployment: deployment.clone(), features, start_blocks, end_blocks, + source_subgraph_stores: subgraph_data_source_stores, stop_block, + max_end_block, store, debug_fork, triggers_adapter, @@ -422,7 +522,7 @@ impl SubgraphInstanceManager { unified_api_version, static_filters: self.static_filters, poi_version, - network, + network: network.to_string(), instrument, }; @@ -438,6 +538,7 @@ impl SubgraphInstanceManager { self.instances.cheap_clone(), offchain_monitor, tp, + decoder, ); for data_source in data_sources { ctx.add_dynamic_data_source(&logger, data_source)?; @@ -470,7 +571,7 @@ impl SubgraphInstanceManager { ::MappingTrigger: ToAscPtr, { let registry = self.metrics_registry.cheap_clone(); - let subgraph_metrics_unregister = runner.metrics.subgraph.cheap_clone(); + let subgraph_metrics = runner.metrics.subgraph.cheap_clone(); // Keep restarting the subgraph until it terminates. The subgraph // will usually only run once, but is restarted whenever a block @@ -486,16 +587,31 @@ impl SubgraphInstanceManager { // it has a dedicated OS thread so the OS will handle the preemption. See // https://github.com/tokio-rs/tokio/issues/3493. graph::spawn_thread(deployment.to_string(), move || { - if let Err(e) = graph::block_on(task::unconstrained(runner.run())) { - error!( - &logger, - "Subgraph instance failed to run: {}", - format!("{:#}", e) - ); + match graph::block_on(task::unconstrained(runner.run())) { + Ok(()) => { + subgraph_metrics.deployment_status.stopped(); + } + Err(SubgraphRunnerError::Duplicate) => { + // We do not need to unregister metrics because they are unique per subgraph + // and another runner is still active. + return; + } + Err(err) => { + error!(&logger, "Subgraph instance failed to run: {:#}", err); + subgraph_metrics.deployment_status.failed(); + } } - subgraph_metrics_unregister.unregister(registry); + + subgraph_metrics.unregister(registry); }); Ok(()) } + + pub fn new_deployment_status_metric( + &self, + deployment: &DeploymentLocator, + ) -> DeploymentStatusMetric { + DeploymentStatusMetric::register(&self.metrics_registry, deployment) + } } diff --git a/core/src/subgraph/mod.rs b/core/src/subgraph/mod.rs index 45f8d5b98ef..8f6bc932daa 100644 --- a/core/src/subgraph/mod.rs +++ b/core/src/subgraph/mod.rs @@ -3,7 +3,6 @@ mod error; mod inputs; mod instance_manager; mod loader; -mod provider; mod registrar; mod runner; mod state; @@ -11,7 +10,6 @@ mod stream; mod trigger_processor; pub use self::instance_manager::SubgraphInstanceManager; -pub use self::provider::SubgraphAssignmentProvider; pub use self::registrar::SubgraphRegistrar; pub use self::runner::SubgraphRunner; pub use self::trigger_processor::*; diff --git a/core/src/subgraph/provider.rs b/core/src/subgraph/provider.rs deleted file mode 100644 index 00d379db01f..00000000000 --- a/core/src/subgraph/provider.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::collections::HashSet; -use std::sync::Mutex; - -use async_trait::async_trait; - -use graph::{ - components::store::{DeploymentId, DeploymentLocator}, - prelude::{SubgraphAssignmentProvider as SubgraphAssignmentProviderTrait, *}, -}; - -#[derive(Debug)] -struct DeploymentRegistry { - subgraphs_deployed: Arc>>, - subgraph_metrics: Arc, -} - -impl DeploymentRegistry { - fn new(subgraph_metrics: Arc) -> Self { - Self { - subgraphs_deployed: Arc::new(Mutex::new(HashSet::new())), - subgraph_metrics, - } - } - - fn insert(&self, id: DeploymentId) -> bool { - if !self.subgraphs_deployed.lock().unwrap().insert(id) { - return false; - } - - self.subgraph_metrics.deployment_count.inc(); - true - } - - fn remove(&self, id: &DeploymentId) -> bool { - if !self.subgraphs_deployed.lock().unwrap().remove(id) { - return false; - } - - self.subgraph_metrics.deployment_count.dec(); - true - } -} - -pub struct SubgraphAssignmentProvider { - logger_factory: LoggerFactory, - deployment_registry: DeploymentRegistry, - link_resolver: Arc, - instance_manager: Arc, -} - -impl SubgraphAssignmentProvider { - pub fn new( - logger_factory: &LoggerFactory, - link_resolver: Arc, - instance_manager: I, - subgraph_metrics: Arc, - ) -> Self { - let logger = logger_factory.component_logger("SubgraphAssignmentProvider", None); - let logger_factory = logger_factory.with_parent(logger.clone()); - - // Create the subgraph provider - SubgraphAssignmentProvider { - logger_factory, - link_resolver: link_resolver.with_retries().into(), - instance_manager: Arc::new(instance_manager), - deployment_registry: DeploymentRegistry::new(subgraph_metrics), - } - } -} - -#[async_trait] -impl SubgraphAssignmentProviderTrait for SubgraphAssignmentProvider { - async fn start( - &self, - loc: DeploymentLocator, - stop_block: Option, - ) -> Result<(), SubgraphAssignmentProviderError> { - let logger = self.logger_factory.subgraph_logger(&loc); - - // If subgraph ID already in set - if !self.deployment_registry.insert(loc.id) { - info!(logger, "Subgraph deployment is already running"); - - return Err(SubgraphAssignmentProviderError::AlreadyRunning( - loc.hash.clone(), - )); - } - - let file_bytes = self - .link_resolver - .cat(&logger, &loc.hash.to_ipfs_link()) - .await - .map_err(SubgraphAssignmentProviderError::ResolveError)?; - - let raw: serde_yaml::Mapping = serde_yaml::from_slice(&file_bytes) - .map_err(|e| SubgraphAssignmentProviderError::ResolveError(e.into()))?; - - self.instance_manager - .cheap_clone() - .start_subgraph(loc, raw, stop_block) - .await; - - Ok(()) - } - - async fn stop( - &self, - deployment: DeploymentLocator, - ) -> Result<(), SubgraphAssignmentProviderError> { - // If subgraph ID was in set - if self.deployment_registry.remove(&deployment.id) { - // Shut down subgraph processing - self.instance_manager.stop_subgraph(deployment).await; - } - Ok(()) - } -} diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index 9404b415d20..e9422671e25 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -1,41 +1,45 @@ use std::collections::HashSet; -use std::time::Instant; use async_trait::async_trait; -use graph::blockchain::Blockchain; -use graph::blockchain::BlockchainKind; -use graph::blockchain::BlockchainMap; -use graph::components::store::{DeploymentId, DeploymentLocator, SubscriptionManager}; -use graph::components::subgraph::Settings; -use graph::data::subgraph::schema::DeploymentCreate; -use graph::data::subgraph::Graft; -use graph::prelude::{ - CreateSubgraphResult, SubgraphAssignmentProvider as SubgraphAssignmentProviderTrait, - SubgraphRegistrar as SubgraphRegistrarTrait, *, +use graph::amp; +use graph::blockchain::{Blockchain, BlockchainKind, BlockchainMap}; +use graph::components::{ + link_resolver::LinkResolverContext, + network_provider::AmpChainNames, + store::{DeploymentId, DeploymentLocator, SubscriptionManager}, + subgraph::Settings, }; -use graph::tokio_retry::Retry; -use graph::util::futures::retry_strategy; -use graph::util::futures::RETRY_DEFAULT_LIMIT; +use graph::data::{ + subgraph::{Graft, schema::DeploymentCreate}, + value::Word, +}; +use graph::futures03::{self, Stream, StreamExt, future::TryFutureExt}; +use graph::prelude::{CreateSubgraphResult, SubgraphRegistrar as SubgraphRegistrarTrait, *}; +use graph::util::futures::{RETRY_DEFAULT_LIMIT, retry_strategy}; +use tokio_retry::Retry; -pub struct SubgraphRegistrar { +pub struct SubgraphRegistrar { logger: Logger, logger_factory: LoggerFactory, resolver: Arc, provider: Arc

, store: Arc, subscription_manager: Arc, + amp_client: Option>, chains: Arc, node_id: NodeId, version_switching_mode: SubgraphVersionSwitchingMode, assignment_event_stream_cancel_guard: CancelGuard, // cancels on drop settings: Arc, + amp_chain_names: Arc, } -impl SubgraphRegistrar +impl SubgraphRegistrar where - P: SubgraphAssignmentProviderTrait, + P: graph::components::subgraph::SubgraphInstanceManager, S: SubgraphStore, SM: SubscriptionManager, + AC: amp::Client + Send + Sync + 'static, { pub fn new( logger_factory: &LoggerFactory, @@ -43,10 +47,12 @@ where provider: Arc

, store: Arc, subscription_manager: Arc, + amp_client: Option>, chains: Arc, node_id: NodeId, version_switching_mode: SubgraphVersionSwitchingMode, settings: Arc, + amp_chain_names: Arc, ) -> Self { let logger = logger_factory.component_logger("SubgraphRegistrar", None); let logger_factory = logger_factory.with_parent(logger.clone()); @@ -60,22 +66,17 @@ where provider, store, subscription_manager, + amp_client, chains, node_id, version_switching_mode, assignment_event_stream_cancel_guard: CancelGuard::new(), settings, + amp_chain_names, } } - pub fn start(&self) -> impl Future { - let logger_clone1 = self.logger.clone(); - let logger_clone2 = self.logger.clone(); - let provider = self.provider.clone(); - let node_id = self.node_id.clone(); - let assignment_event_stream_cancel_handle = - self.assignment_event_stream_cancel_guard.handle(); - + pub async fn start(self: Arc) -> Result<(), Error> { // The order of the following three steps is important: // - Start assignment event stream // - Read assignments table and start assigned subgraphs @@ -90,185 +91,153 @@ where // // The discrepancy between the start time of the event stream and the table read can result // in some extraneous events on start up. Examples: - // - The event stream sees an Add event for subgraph A, but the table query finds that + // - The event stream sees an 'set' event for subgraph A, but the table query finds that // subgraph A is already in the table. - // - The event stream sees a Remove event for subgraph B, but the table query finds that + // - The event stream sees a 'removed' event for subgraph B, but the table query finds that // subgraph B has already been removed. - // The `handle_assignment_events` function handles these cases by ignoring AlreadyRunning - // (on subgraph start) which makes the operations idempotent. Subgraph stop is already idempotent. + // The `change_assignment` function handles these cases by ignoring + // such cases which makes the operations idempotent // Start event stream - let assignment_event_stream = self.assignment_events(); + let assignment_event_stream = self.cheap_clone().assignment_events().await; // Deploy named subgraphs found in store - self.start_assigned_subgraphs().and_then(move |()| { - // Spawn a task to handle assignment events. - // Blocking due to store interactions. Won't be blocking after #905. - graph::spawn_blocking( - assignment_event_stream - .compat() - .map_err(SubgraphAssignmentProviderError::Unknown) - .map_err(CancelableError::Error) - .cancelable(&assignment_event_stream_cancel_handle, || { - Err(CancelableError::Cancel) - }) - .compat() - .for_each(move |assignment_event| { - assert_eq!(assignment_event.node_id(), &node_id); - handle_assignment_event( - assignment_event, - provider.clone(), - logger_clone1.clone(), - ) - .boxed() - .compat() - }) - .map_err(move |e| match e { - CancelableError::Cancel => panic!("assignment event stream canceled"), - CancelableError::Error(e) => { - error!(logger_clone2, "Assignment event stream failed: {}", e); - panic!("assignment event stream failed: {}", e); - } - }) - .compat(), - ); + self.start_assigned_subgraphs().await?; + + let cancel_handle = self.assignment_event_stream_cancel_guard.handle(); + + // Spawn a task to handle assignment events. + let fut = assignment_event_stream.for_each({ + move |event| { + // The assignment stream should run forever. If it gets + // cancelled, that probably indicates a serious problem and + // we panic + if cancel_handle.is_canceled() { + panic!("assignment event stream canceled"); + } - Ok(()) - }) + let this = self.cheap_clone(); + async move { + this.change_assignment(event).await; + } + } + }); + + graph::spawn(fut); + Ok(()) } - pub fn assignment_events(&self) -> impl Stream + Send { - let store = self.store.clone(); - let node_id = self.node_id.clone(); - let logger = self.logger.clone(); + /// Start/stop subgraphs as needed, considering the current assignment + /// state in the database, ignoring changes that do not affect this + /// node, do not require anything to change, or for which we can not + /// find the assignment status from the database + async fn change_assignment(&self, change: AssignmentChange) { + let (deployment, operation) = change.into_parts(); - self.subscription_manager - .subscribe(FromIterator::from_iter([SubscriptionFilter::Assignment])) - .map_err(|()| anyhow!("Entity change stream failed")) - .map(|event| { - // We're only interested in the SubgraphDeploymentAssignment change; we - // know that there is at least one, as that is what we subscribed to - let filter = SubscriptionFilter::Assignment; - let assignments = event - .changes - .iter() - .filter(|change| filter.matches(change)) - .map(|change| match change { - EntityChange::Data { .. } => unreachable!(), - EntityChange::Assignment { - deployment, - operation, - } => (deployment.clone(), operation.clone()), - }) - .collect::>(); - stream::iter_ok(assignments) - }) - .flatten() - .and_then( - move |(deployment, operation)| -> Result + Send>, _> { - trace!(logger, "Received assignment change"; - "deployment" => %deployment, - "operation" => format!("{:?}", operation), - ); - - match operation { - EntityChangeOperation::Set => { - store - .assignment_status(&deployment) - .map_err(|e| { - anyhow!("Failed to get subgraph assignment entity: {}", e) - }) - .map(|assigned| -> Box + Send> { - if let Some((assigned,is_paused)) = assigned { - if assigned == node_id { - - if is_paused{ - // Subgraph is paused, so we don't start it - debug!(logger, "Deployment assignee is this node, but it is paused, so we don't start it"; "assigned_to" => assigned, "node_id" => &node_id,"paused" => is_paused); - return Box::new(stream::empty()); - } - - // Start subgraph on this node - debug!(logger, "Deployment assignee is this node, broadcasting add event"; "assigned_to" => assigned, "node_id" => &node_id); - Box::new(stream::once(Ok(AssignmentEvent::Add { - deployment, - node_id: node_id.clone(), - }))) - } else { - // Ensure it is removed from this node - debug!(logger, "Deployment assignee is not this node, broadcasting remove event"; "assigned_to" => assigned, "node_id" => &node_id); - Box::new(stream::once(Ok(AssignmentEvent::Remove { - deployment, - node_id: node_id.clone(), - }))) - } - } else { - // Was added/updated, but is now gone. - debug!(logger, "Deployment has not assignee, we will get a separate remove event later"; "node_id" => &node_id); - Box::new(stream::empty()) - } - }) - } - EntityChangeOperation::Removed => { - // Send remove event without checking node ID. - // If node ID does not match, then this is a no-op when handled in - // assignment provider. - Ok(Box::new(stream::once(Ok(AssignmentEvent::Remove { - deployment, - node_id: node_id.clone(), - })))) + trace!(self.logger, "Received assignment change"; + "deployment" => %deployment, + "operation" => format!("{:?}", operation), + ); + + match operation { + AssignmentOperation::Set => { + let assigned = match self.store.assignment_status(&deployment).await { + Ok(assigned) => assigned, + Err(e) => { + error!( + self.logger, + "Failed to get subgraph assignment entity"; "deployment" => deployment, "error" => e.to_string() + ); + return; + } + }; + + let logger = self.logger.new(o!("subgraph_id" => deployment.hash.to_string(), "node_id" => self.node_id.to_string())); + if let Some((assigned, is_paused)) = assigned { + if assigned == self.node_id { + if is_paused { + // Subgraph is paused, so we don't start it + debug!(logger, "Deployment assignee is this node"; "assigned_to" => assigned, "paused" => is_paused, "action" => "ignore"); + return; } + + // Start subgraph on this node + debug!(logger, "Deployment assignee is this node"; "assigned_to" => assigned, "action" => "add"); + self.provider + .cheap_clone() + .start_subgraph(deployment, None) + .await; + } else { + // Ensure it is removed from this node + debug!(logger, "Deployment assignee is not this node"; "assigned_to" => assigned, "action" => "remove"); + self.provider.stop_subgraph(deployment).await } - }, - ) + } else { + // Was added/updated, but is now gone. + debug!(self.logger, "Deployment assignee not found in database"; "action" => "ignore"); + } + } + AssignmentOperation::Removed => { + self.provider.stop_subgraph(deployment).await; + } + } + } + + pub async fn assignment_events(self: Arc) -> impl Stream + Send { + self.subscription_manager + .subscribe() + .map(|event| futures03::stream::iter(event.changes.clone())) .flatten() } - fn start_assigned_subgraphs(&self) -> impl Future { - let provider = self.provider.clone(); + async fn start_assigned_subgraphs(&self) -> Result<(), Error> { let logger = self.logger.clone(); let node_id = self.node_id.clone(); - future::result(self.store.active_assignments(&self.node_id)) - .map_err(|e| anyhow!("Error querying subgraph assignments: {}", e)) - .and_then(move |deployments| { - // This operation should finish only after all subgraphs are - // started. We wait for the spawned tasks to complete by giving - // each a `sender` and waiting for all of them to be dropped, so - // the receiver terminates without receiving anything. - let deployments = HashSet::::from_iter(deployments); - let deployments_len = deployments.len(); - let (sender, receiver) = futures01::sync::mpsc::channel::<()>(1); - for id in deployments { - let sender = sender.clone(); - let logger = logger.clone(); - - graph::spawn( - start_subgraph(id, provider.clone(), logger).map(move |()| drop(sender)), - ); - } - drop(sender); - receiver.collect().then(move |_| { - info!(logger, "Started all assigned subgraphs"; - "count" => deployments_len, "node_id" => &node_id); - future::ok(()) - }) - }) + let deployments = self + .store + .active_assignments(&self.node_id) + .await + .map_err(|e| anyhow!("Error querying subgraph assignments: {}", e))?; + // This operation should finish only after all subgraphs are + // started. We wait for the spawned tasks to complete by giving + // each a `sender` and waiting for all of them to be dropped, so + // the receiver terminates without receiving anything. + let deployments = HashSet::::from_iter(deployments); + let deployments_len = deployments.len(); + debug!(logger, "Starting all assigned subgraphs"; + "count" => deployments_len, "node_id" => &node_id); + let (sender, receiver) = futures03::channel::mpsc::channel::<()>(1); + for id in deployments { + let sender = sender.clone(); + let provider = self.provider.cheap_clone(); + + graph::spawn(async move { + provider.start_subgraph(id, None).await; + drop(sender) + }); + } + drop(sender); + let _: Vec<_> = receiver.collect().await; + info!(logger, "Started all assigned subgraphs"; + "count" => deployments_len, "node_id" => &node_id); + Ok(()) } } #[async_trait] -impl SubgraphRegistrarTrait for SubgraphRegistrar +impl SubgraphRegistrarTrait for SubgraphRegistrar where - P: SubgraphAssignmentProviderTrait, + P: graph::components::subgraph::SubgraphInstanceManager, S: SubgraphStore, SM: SubscriptionManager, + AC: amp::Client + Send + Sync + 'static, { async fn create_subgraph( &self, name: SubgraphName, ) -> Result { - let id = self.store.create_subgraph(name.clone())?; + let id = self.store.create_subgraph(name.clone()).await?; debug!(self.logger, "Created subgraph"; "subgraph_name" => name.to_string()); @@ -284,6 +253,7 @@ where start_block_override: Option, graft_block_override: Option, history_blocks: Option, + ignore_graft_base: bool, ) -> Result { // We don't have a location for the subgraph yet; that will be // assigned when we deploy for real. For logging purposes, make up a @@ -292,19 +262,35 @@ where .logger_factory .subgraph_logger(&DeploymentLocator::new(DeploymentId(0), hash.clone())); - let raw: serde_yaml::Mapping = { - let file_bytes = self - .resolver - .cat(&logger, &hash.to_ipfs_link()) - .await - .map_err(|e| { - SubgraphRegistrarError::ResolveError( - SubgraphManifestResolveError::ResolveError(e), + let resolver: Arc = Arc::from( + self.resolver + .for_manifest(&hash.to_string()) + .map_err(SubgraphRegistrarError::Unknown)?, + ); + + let raw = { + let mut raw: serde_yaml::Mapping = { + let file_bytes = resolver + .cat( + &LinkResolverContext::new(&hash, &logger), + &hash.to_ipfs_link(), ) - })?; + .await + .map_err(|e| { + SubgraphRegistrarError::ResolveError( + SubgraphManifestResolveError::ResolveError(e), + ) + })?; + + serde_yaml::from_slice(&file_bytes) + .map_err(|e| SubgraphRegistrarError::ResolveError(e.into()))? + }; - serde_yaml::from_slice(&file_bytes) - .map_err(|e| SubgraphRegistrarError::ResolveError(e.into()))? + if ignore_graft_base { + raw.remove("graft"); + } + + raw }; let kind = BlockchainKind::from_manifest(&raw).map_err(|e| { @@ -316,26 +302,8 @@ where history_blocks.or(self.settings.for_name(&name).map(|c| c.history_blocks)); let deployment_locator = match kind { - BlockchainKind::Arweave => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - history_blocks, - ) - .await? - } BlockchainKind::Ethereum => { - create_subgraph_version::( + create_subgraph_version::( &logger, self.store.clone(), self.chains.cheap_clone(), @@ -347,13 +315,15 @@ where node_id, debug_fork, self.version_switching_mode, - &self.resolver, + &resolver, + self.amp_client.cheap_clone(), history_blocks, + &self.amp_chain_names, ) .await? } BlockchainKind::Near => { - create_subgraph_version::( + create_subgraph_version::( &logger, self.store.clone(), self.chains.cheap_clone(), @@ -365,62 +335,10 @@ where node_id, debug_fork, self.version_switching_mode, - &self.resolver, - history_blocks, - ) - .await? - } - BlockchainKind::Cosmos => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - history_blocks, - ) - .await? - } - BlockchainKind::Substreams => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - history_blocks, - ) - .await? - } - BlockchainKind::Starknet => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, + &resolver, + self.amp_client.cheap_clone(), history_blocks, + &self.amp_chain_names, ) .await? } @@ -437,9 +355,12 @@ where } async fn remove_subgraph(&self, name: SubgraphName) -> Result<(), SubgraphRegistrarError> { - self.store.clone().remove_subgraph(name.clone())?; + use itertools::Itertools; - debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string()); + let hashes = self.store.clone().remove_subgraph(name.clone()).await?; + let hashes = hashes.into_iter().join(", "); + + debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string(), "deployments" => format!("[{}]", hashes)); Ok(()) } @@ -453,73 +374,33 @@ where hash: &DeploymentHash, node_id: &NodeId, ) -> Result<(), SubgraphRegistrarError> { - let locator = self.store.active_locator(hash)?; + let locator = self.store.active_locator(hash).await?; let deployment = locator.ok_or_else(|| SubgraphRegistrarError::DeploymentNotFound(hash.to_string()))?; - self.store.reassign_subgraph(&deployment, node_id)?; + self.store.reassign_subgraph(&deployment, node_id).await?; Ok(()) } -} -async fn handle_assignment_event( - event: AssignmentEvent, - provider: Arc, - logger: Logger, -) -> Result<(), CancelableError> { - let logger = logger.clone(); + async fn pause_subgraph(&self, hash: &DeploymentHash) -> Result<(), SubgraphRegistrarError> { + let locator = self.store.active_locator(hash).await?; + let deployment = + locator.ok_or_else(|| SubgraphRegistrarError::DeploymentNotFound(hash.to_string()))?; - debug!(logger, "Received assignment event: {:?}", event); + self.store.pause_subgraph(&deployment).await?; - match event { - AssignmentEvent::Add { - deployment, - node_id: _, - } => { - start_subgraph(deployment, provider.clone(), logger).await; - Ok(()) - } - AssignmentEvent::Remove { - deployment, - node_id: _, - } => match provider.stop(deployment).await { - Ok(()) => Ok(()), - Err(e) => Err(CancelableError::Error(e)), - }, + Ok(()) } -} - -async fn start_subgraph( - deployment: DeploymentLocator, - provider: Arc, - logger: Logger, -) { - let logger = logger - .new(o!("subgraph_id" => deployment.hash.to_string(), "sgd" => deployment.id.to_string())); - trace!(logger, "Start subgraph"); - - let start_time = Instant::now(); - let result = provider.start(deployment.clone(), None).await; + async fn resume_subgraph(&self, hash: &DeploymentHash) -> Result<(), SubgraphRegistrarError> { + let locator = self.store.active_locator(hash).await?; + let deployment = + locator.ok_or_else(|| SubgraphRegistrarError::DeploymentNotFound(hash.to_string()))?; - debug!( - logger, - "Subgraph started"; - "start_ms" => start_time.elapsed().as_millis() - ); + self.store.resume_subgraph(&deployment).await?; - match result { - Ok(()) => (), - Err(SubgraphAssignmentProviderError::AlreadyRunning(_)) => (), - Err(e) => { - // Errors here are likely an issue with the subgraph. - error!( - logger, - "Subgraph instance failed to start"; - "error" => e.to_string() - ); - } + Ok(()) } } @@ -539,9 +420,9 @@ async fn resolve_start_block( .expect("cannot identify minimum start block because there are no data sources") { 0 => Ok(None), - min_start_block => Retry::spawn(retry_strategy(Some(2), RETRY_DEFAULT_LIMIT), move || { + min_start_block => Retry::start(retry_strategy(Some(2), RETRY_DEFAULT_LIMIT), move || { chain - .block_pointer_from_number(&logger, min_start_block - 1) + .block_pointer_from_number(logger, min_start_block - 1) .inspect_err(move |e| warn!(&logger, "Failed to get block number: {}", e)) }) .await @@ -573,7 +454,7 @@ async fn resolve_graft_block( }) } -async fn create_subgraph_version( +async fn create_subgraph_version( logger: &Logger, store: Arc, chains: Arc, @@ -586,24 +467,27 @@ async fn create_subgraph_version( debug_fork: Option, version_switching_mode: SubgraphVersionSwitchingMode, resolver: &Arc, + amp_client: Option>, history_blocks_override: Option, + amp_chain_names: &AmpChainNames, ) -> Result { let raw_string = serde_yaml::to_string(&raw).unwrap(); + let unvalidated = UnvalidatedSubgraphManifest::::resolve( deployment.clone(), raw, resolver, + amp_client, logger, ENV_VARS.max_spec_version.clone(), ) .map_err(SubgraphRegistrarError::ResolveError) .await?; - // Determine if the graft_base should be validated. // Validate the graft_base if there is a pending graft, ensuring its presence. // If the subgraph is new (indicated by DeploymentNotFound), the graft_base should be validated. // If the subgraph already exists and there is no pending graft, graft_base validation is not required. - let should_validate = match store.graft_pending(&deployment) { + let should_validate = match store.graft_pending(&deployment).await { Ok(graft_pending) => graft_pending, Err(StoreError::DeploymentNotFound(_)) => true, Err(e) => return Err(SubgraphRegistrarError::StoreError(e)), @@ -613,10 +497,11 @@ async fn create_subgraph_version( .await .map_err(SubgraphRegistrarError::ManifestValidationError)?; - let network_name = manifest.network_name(); + let network_name: Word = manifest.network_name().into(); + let resolved_name = amp_chain_names.resolve(&network_name); let chain = chains - .get::(network_name.clone()) + .get::(resolved_name.clone()) .map_err(SubgraphRegistrarError::NetworkNotSupported)? .cheap_clone(); @@ -624,7 +509,7 @@ async fn create_subgraph_version( let store = store.clone(); let deployment_store = store.clone(); - if !store.subgraph_exists(&name)? { + if !store.subgraph_exists(&name).await? { debug!( logger, "Subgraph not found, could not create_subgraph_version"; @@ -696,8 +581,9 @@ async fn create_subgraph_version( &manifest.schema, deployment, node_id, - network_name, + resolved_name.into(), version_switching_mode, ) + .await .map_err(SubgraphRegistrarError::SubgraphDeploymentError) } diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs deleted file mode 100644 index 2bc03f37214..00000000000 --- a/core/src/subgraph/runner.rs +++ /dev/null @@ -1,1545 +0,0 @@ -use crate::subgraph::context::IndexingContext; -use crate::subgraph::error::BlockProcessingError; -use crate::subgraph::inputs::IndexingInputs; -use crate::subgraph::state::IndexingState; -use crate::subgraph::stream::new_block_stream; -use atomic_refcell::AtomicRefCell; -use graph::blockchain::block_stream::{BlockStreamEvent, BlockWithTriggers, FirehoseCursor}; -use graph::blockchain::{Block, BlockTime, Blockchain, DataSource as _, TriggerFilter as _}; -use graph::components::store::{EmptyStore, GetScope, ReadStore, StoredDynamicDataSource}; -use graph::components::{ - store::ModificationsAndCache, - subgraph::{MappingError, PoICausalityRegion, ProofOfIndexing, SharedProofOfIndexing}, -}; -use graph::data::store::scalar::Bytes; -use graph::data::subgraph::{ - schema::{SubgraphError, SubgraphHealth}, - SubgraphFeature, -}; -use graph::data_source::{ - offchain, CausalityRegion, DataSource, DataSourceCreationError, DataSourceTemplate, TriggerData, -}; -use graph::env::EnvVars; -use graph::prelude::*; -use graph::schema::EntityKey; -use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -const MINUTE: Duration = Duration::from_secs(60); - -const SKIP_PTR_UPDATES_THRESHOLD: Duration = Duration::from_secs(60 * 5); -const HANDLE_REVERT_SECTION_NAME: &str = "handle_revert"; -const PROCESS_BLOCK_SECTION_NAME: &str = "process_block"; -const PROCESS_WASM_BLOCK_SECTION_NAME: &str = "process_wasm_block"; -const PROCESS_TRIGGERS_SECTION_NAME: &str = "process_triggers"; -const HANDLE_CREATED_DS_SECTION_NAME: &str = "handle_new_data_sources"; - -pub struct SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - ctx: IndexingContext, - state: IndexingState, - inputs: Arc>, - logger: Logger, - pub metrics: RunnerMetrics, -} - -impl SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - pub fn new( - inputs: IndexingInputs, - ctx: IndexingContext, - logger: Logger, - metrics: RunnerMetrics, - env_vars: Arc, - ) -> Self { - Self { - inputs: Arc::new(inputs), - ctx, - state: IndexingState { - should_try_unfail_non_deterministic: true, - synced: false, - skip_ptr_updates_timer: Instant::now(), - backoff: ExponentialBackoff::with_jitter( - (MINUTE * 2).min(env_vars.subgraph_error_retry_ceil), - env_vars.subgraph_error_retry_ceil, - env_vars.subgraph_error_retry_jitter, - ), - entity_lfu_cache: LfuCache::new(), - cached_head_ptr: None, - }, - logger, - metrics, - } - } - - /// Revert the state to a previous block. When handling revert operations - /// or failed block processing, it is necessary to remove part of the existing - /// in-memory state to keep it constent with DB changes. - /// During block processing new dynamic data sources are added directly to the - /// IndexingContext of the runner. This means that if, for whatever reason, - /// the changes don;t complete then the remnants of that block processing must - /// be removed. The same thing also applies to the block cache. - /// This function must be called before continuing to process in order to avoid - /// duplicated host insertion and POI issues with dirty entity changes. - fn revert_state_to(&mut self, block_number: BlockNumber) -> Result<(), Error> { - self.state.entity_lfu_cache = LfuCache::new(); - - // 1. Revert all hosts(created by DDS) at a block higher than `block_number`. - // 2. Unmark any offchain data sources that were marked done on the blocks being removed. - // When no offchain datasources are present, 2. should be a noop. - self.ctx.revert_data_sources(block_number + 1)?; - Ok(()) - } - - #[cfg(debug_assertions)] - pub fn context(&self) -> &IndexingContext { - &self.ctx - } - - #[cfg(debug_assertions)] - pub async fn run_for_test(self, break_on_restart: bool) -> Result { - self.run_inner(break_on_restart).await - } - - fn build_filter(&self) -> C::TriggerFilter { - let current_ptr = self.inputs.store.block_ptr(); - let static_filters = - self.inputs.static_filters || self.ctx.hosts_len() > ENV_VARS.static_filters_threshold; - - // Filter out data sources that have reached their end block - let end_block_filter = |ds: &&C::DataSource| match current_ptr.as_ref() { - // We filter out datasources for which the current block is at or past their end block. - Some(block) => ds.end_block().map_or(true, |end| block.number < end), - // If there is no current block, we keep all datasources. - None => true, - }; - - // if static_filters is not enabled we just stick to the filter based on all the data sources. - if !static_filters { - return C::TriggerFilter::from_data_sources( - self.ctx.onchain_data_sources().filter(end_block_filter), - ); - } - - // if static_filters is enabled, build a minimal filter with the static data sources and - // add the necessary filters based on templates. - // This specifically removes dynamic data sources based filters because these can be derived - // from templates AND this reduces the cost of egress traffic by making the payloads smaller. - - if !self.inputs.static_filters { - info!(self.logger, "forcing subgraph to use static filters.") - } - - let data_sources = self.ctx.static_data_sources(); - - let mut filter = C::TriggerFilter::from_data_sources( - data_sources - .iter() - .filter_map(|ds| ds.as_onchain()) - // Filter out data sources that have reached their end block if the block is final. - .filter(end_block_filter), - ); - - let templates = self.ctx.templates(); - - filter.extend_with_template(templates.iter().filter_map(|ds| ds.as_onchain()).cloned()); - - filter - } - - #[cfg(debug_assertions)] - pub fn build_filter_for_test(&self) -> C::TriggerFilter { - self.build_filter() - } - - pub async fn run(self) -> Result<(), Error> { - self.run_inner(false).await.map(|_| ()) - } - - async fn run_inner(mut self, break_on_restart: bool) -> Result { - // If a subgraph failed for deterministic reasons, before start indexing, we first - // revert the deployment head. It should lead to the same result since the error was - // deterministic. - if let Some(current_ptr) = self.inputs.store.block_ptr() { - if let Some(parent_ptr) = self - .inputs - .triggers_adapter - .parent_ptr(¤t_ptr) - .await? - { - // This reverts the deployment head to the parent_ptr if - // deterministic errors happened. - // - // There's no point in calling it if we have no current or parent block - // pointers, because there would be: no block to revert to or to search - // errors from (first execution). - // - // We attempt to unfail deterministic errors to mitigate deterministic - // errors caused by wrong data being consumed from the providers. It has - // been a frequent case in the past so this helps recover on a larger scale. - let _outcome = self - .inputs - .store - .unfail_deterministic_error(¤t_ptr, &parent_ptr) - .await?; - } - } - - loop { - debug!(self.logger, "Starting or restarting subgraph"); - - let block_stream_canceler = CancelGuard::new(); - let block_stream_cancel_handle = block_stream_canceler.handle(); - // TriggerFilter needs to be rebuilt eveytime the blockstream is restarted - self.ctx.filter = Some(self.build_filter()); - - let mut block_stream = new_block_stream( - &self.inputs, - self.ctx.filter.as_ref().unwrap(), // Safe to unwrap as we just called `build_filter` in the previous line - &self.metrics.subgraph, - ) - .await? - .map_err(CancelableError::Error) - .cancelable(&block_stream_canceler, || Err(CancelableError::Cancel)); - - // Keep the stream's cancel guard around to be able to shut it down when the subgraph - // deployment is unassigned - self.ctx - .instances - .insert(self.inputs.deployment.id, block_stream_canceler); - - debug!(self.logger, "Starting block stream"); - - // Process events from the stream as long as no restart is needed - loop { - let event = { - let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); - - block_stream.next().await - }; - - // TODO: move cancel handle to the Context - // This will require some code refactor in how the BlockStream is created - let block_start = Instant::now(); - match self - .handle_stream_event(event, &block_stream_cancel_handle) - .await - .map(|res| { - self.metrics - .subgraph - .observe_block_processed(block_start.elapsed(), res.block_finished()); - res - })? { - Action::Continue => continue, - Action::Stop => { - info!(self.logger, "Stopping subgraph"); - self.inputs.store.flush().await?; - return Ok(self); - } - Action::Restart if break_on_restart => { - info!(self.logger, "Stopping subgraph on break"); - self.inputs.store.flush().await?; - return Ok(self); - } - Action::Restart => { - // Restart the store to clear any errors that it - // might have encountered and use that from now on - let store = self.inputs.store.cheap_clone(); - if let Some(store) = store.restart().await? { - let last_good_block = - store.block_ptr().map(|ptr| ptr.number).unwrap_or(0); - self.revert_state_to(last_good_block)?; - self.inputs = Arc::new(self.inputs.with_store(store)); - self.state.synced = self.inputs.store.is_deployment_synced().await?; - } - break; - } - }; - } - } - } - - /// Processes a block and returns the updated context and a boolean flag indicating - /// whether new dynamic data sources have been added to the subgraph. - async fn process_block( - &mut self, - block_stream_cancel_handle: &CancelHandle, - block: BlockWithTriggers, - firehose_cursor: FirehoseCursor, - ) -> Result { - let triggers = block.trigger_data; - let block = Arc::new(block.block); - let block_ptr = block.ptr(); - - let logger = self.logger.new(o!( - "block_number" => format!("{:?}", block_ptr.number), - "block_hash" => format!("{}", block_ptr.hash) - )); - - debug!(logger, "Start processing block"; - "triggers" => triggers.len()); - - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; - - // Causality region for onchain triggers. - let causality_region = PoICausalityRegion::from_network(&self.inputs.network); - - let mut block_state = { - let _section = self - .metrics - .stream - .stopwatch - .start_section(PROCESS_TRIGGERS_SECTION_NAME); - - // Process events one after the other, passing in entity operations - // collected previously to every new event being processed - match self - .process_triggers( - &proof_of_indexing, - &block, - triggers.into_iter().map(TriggerData::Onchain), - &causality_region, - ) - .await - { - // Triggers processed with no errors or with only deterministic errors. - Ok(block_state) => block_state, - - // Some form of unknown or non-deterministic error ocurred. - Err(MappingError::Unknown(e)) => return Err(BlockProcessingError::Unknown(e)), - Err(MappingError::PossibleReorg(e)) => { - info!(logger, - "Possible reorg detected, retrying"; - "error" => format!("{:#}", e), - ); - - // In case of a possible reorg, we want this function to do nothing and restart the - // block stream so it has a chance to detect the reorg. - // - // The state is unchanged at this point, except for having cleared the entity cache. - // Losing the cache is a bit annoying but not an issue for correctness. - // - // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. - return Ok(Action::Restart); - } - } - }; - - // Check if there are any datasources that have expired in this block. ie: the end_block - // of that data source is equal to the block number of the current block. - let has_expired_data_sources = self.inputs.end_blocks.contains(&block_ptr.number); - - // If new onchain data sources have been created, and static filters are not in use, it is necessary - // to restart the block stream with the new filters. - let created_data_sources_needs_restart = - !self.inputs.static_filters && block_state.has_created_on_chain_data_sources(); - - // Determine if the block stream needs to be restarted due to newly created on-chain data sources - // or data sources that have reached their end block. - let needs_restart = created_data_sources_needs_restart || has_expired_data_sources; - - { - let _section = self - .metrics - .stream - .stopwatch - .start_section(HANDLE_CREATED_DS_SECTION_NAME); - - // This loop will: - // 1. Instantiate created data sources. - // 2. Process those data sources for the current block. - // Until no data sources are created or MAX_DATA_SOURCES is hit. - - // Note that this algorithm processes data sources spawned on the same block _breadth - // first_ on the tree implied by the parent-child relationship between data sources. Only a - // very contrived subgraph would be able to observe this. - while block_state.has_created_data_sources() { - // Instantiate dynamic data sources, removing them from the block state. - let (data_sources, runtime_hosts) = - self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; - - let filter = C::TriggerFilter::from_data_sources( - data_sources.iter().filter_map(DataSource::as_onchain), - ); - - let block: Arc = if self.inputs.chain.is_refetch_block_required() { - let cur = firehose_cursor.clone(); - let log = logger.cheap_clone(); - let chain = self.inputs.chain.cheap_clone(); - Arc::new( - retry( - "refetch firehose block after dynamic datasource was added", - &logger, - ) - .limit(5) - .no_timeout() - .run(move || { - let cur = cur.clone(); - let log = log.cheap_clone(); - let chain = chain.cheap_clone(); - async move { chain.refetch_firehose_block(&log, cur).await } - }) - .await?, - ) - } else { - block.cheap_clone() - }; - - // Reprocess the triggers from this block that match the new data sources - let block_with_triggers = self - .inputs - .triggers_adapter - .triggers_in_block(&logger, block.as_ref().clone(), &filter) - .await?; - - let triggers = block_with_triggers.trigger_data; - - if triggers.len() == 1 { - info!( - &logger, - "1 trigger found in this block for the new data sources" - ); - } else if triggers.len() > 1 { - info!( - &logger, - "{} triggers found in this block for the new data sources", - triggers.len() - ); - } - - // Add entity operations for the new data sources to the block state - // and add runtimes for the data sources to the subgraph instance. - self.persist_dynamic_data_sources(&mut block_state, data_sources); - - // Process the triggers in each host in the same order the - // corresponding data sources have been created. - for trigger in triggers { - block_state = self - .ctx - .process_trigger_in_hosts( - &logger, - Box::new(runtime_hosts.iter().map(|host| host.as_ref())), - &block, - &TriggerData::Onchain(trigger), - block_state, - &proof_of_indexing, - &causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - self.inputs.instrument, - ) - .await - .map_err(|e| { - // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. - // This can cause an unnecessary subgraph failure, to fix it we need to figure out a - // way to revert the effect of `create_dynamic_data_sources` so we may return a - // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. - match e { - MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { - BlockProcessingError::Unknown(e) - } - } - })?; - } - } - } - - let has_errors = block_state.has_errors(); - let is_non_fatal_errors_active = self - .inputs - .features - .contains(&SubgraphFeature::NonFatalErrors); - - // Apply entity operations and advance the stream - - // Avoid writing to store if block stream has been canceled - if block_stream_cancel_handle.is_canceled() { - return Err(BlockProcessingError::Canceled); - } - - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); - update_proof_of_indexing( - proof_of_indexing, - block.timestamp(), - &self.metrics.host.stopwatch, - &mut block_state.entity_cache, - ) - .await?; - } - - let section = self - .metrics - .host - .stopwatch - .start_section("as_modifications"); - let ModificationsAndCache { - modifications: mut mods, - entity_lfu_cache: cache, - evict_stats, - } = block_state - .entity_cache - .as_modifications(block.number()) - .map_err(|e| BlockProcessingError::Unknown(e.into()))?; - section.end(); - - debug!(self.logger, "Entity cache statistics"; - "weight" => evict_stats.new_weight, - "evicted_weight" => evict_stats.evicted_weight, - "count" => evict_stats.new_count, - "evicted_count" => evict_stats.evicted_count, - "stale_update" => evict_stats.stale_update, - "hit_rate" => format!("{:.0}%", evict_stats.hit_rate_pct()), - "accesses" => evict_stats.accesses, - "evict_time_ms" => evict_stats.evict_time.as_millis()); - - // Check for offchain events and process them, including their entity modifications in the - // set to be transacted. - let offchain_events = self.ctx.offchain_monitor.ready_offchain_events()?; - let (offchain_mods, processed_offchain_data_sources, persisted_off_chain_data_sources) = - self.handle_offchain_triggers(offchain_events, &block) - .await?; - mods.extend(offchain_mods); - - // Put the cache back in the state, asserting that the placeholder cache was not used. - assert!(self.state.entity_lfu_cache.is_empty()); - self.state.entity_lfu_cache = cache; - - if !mods.is_empty() { - info!(&logger, "Applying {} entity operation(s)", mods.len()); - } - - let err_count = block_state.deterministic_errors.len(); - for (i, e) in block_state.deterministic_errors.iter().enumerate() { - let message = format!("{:#}", e).replace('\n', "\t"); - error!(&logger, "Subgraph error {}/{}", i + 1, err_count; - "error" => message, - "code" => LogCode::SubgraphSyncingFailure - ); - } - - // Transact entity operations into the store and update the - // subgraph's block stream pointer - let _section = self.metrics.host.stopwatch.start_section("transact_block"); - let start = Instant::now(); - - let store = &self.inputs.store; - - // If a deterministic error has happened, make the PoI to be the only entity that'll be stored. - if has_errors && !is_non_fatal_errors_active { - let is_poi_entity = - |entity_mod: &EntityModification| entity_mod.key().entity_type.is_poi(); - mods.retain(is_poi_entity); - // Confidence check - assert!( - mods.len() == 1, - "There should be only one PoI EntityModification" - ); - } - - let BlockState { - deterministic_errors, - mut persisted_data_sources, - .. - } = block_state; - - let first_error = deterministic_errors.first().cloned(); - - persisted_data_sources.extend(persisted_off_chain_data_sources); - store - .transact_block_operations( - block_ptr, - block.timestamp(), - firehose_cursor, - mods, - &self.metrics.host.stopwatch, - persisted_data_sources, - deterministic_errors, - processed_offchain_data_sources, - is_non_fatal_errors_active, - ) - .await - .context("Failed to transact block operations")?; - - // For subgraphs with `nonFatalErrors` feature disabled, we consider - // any error as fatal. - // - // So we do an early return to make the subgraph stop processing blocks. - // - // In this scenario the only entity that is stored/transacted is the PoI, - // all of the others are discarded. - if has_errors && !is_non_fatal_errors_active { - // Only the first error is reported. - return Err(BlockProcessingError::Deterministic(first_error.unwrap())); - } - - let elapsed = start.elapsed().as_secs_f64(); - self.metrics - .subgraph - .block_ops_transaction_duration - .observe(elapsed); - - // To prevent a buggy pending version from replacing a current version, if errors are - // present the subgraph will be unassigned. - if has_errors && !ENV_VARS.disable_fail_fast && !store.is_deployment_synced().await? { - store - .unassign_subgraph() - .map_err(|e| BlockProcessingError::Unknown(e.into()))?; - - // Use `Canceled` to avoiding setting the subgraph health to failed, an error was - // just transacted so it will be already be set to unhealthy. - return Err(BlockProcessingError::Canceled); - } - - match needs_restart { - true => Ok(Action::Restart), - false => Ok(Action::Continue), - } - } - - async fn process_triggers( - &mut self, - proof_of_indexing: &SharedProofOfIndexing, - block: &Arc, - triggers: impl Iterator>, - causality_region: &str, - ) -> Result, MappingError> { - let mut block_state = BlockState::new( - self.inputs.store.clone(), - std::mem::take(&mut self.state.entity_lfu_cache), - ); - - for trigger in triggers { - block_state = self - .ctx - .process_trigger( - &self.logger, - block, - &trigger, - block_state, - proof_of_indexing, - causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - self.inputs.instrument, - ) - .await - .map_err(move |mut e| { - let error_context = trigger.error_context(); - if !error_context.is_empty() { - e = e.context(error_context); - } - e.context("failed to process trigger".to_string()) - })?; - } - Ok(block_state) - } - async fn process_wasm_block( - &mut self, - proof_of_indexing: &SharedProofOfIndexing, - block_ptr: BlockPtr, - block_time: BlockTime, - block_data: Box<[u8]>, - handler: String, - causality_region: &str, - ) -> Result, MappingError> { - let block_state = BlockState::new( - self.inputs.store.clone(), - std::mem::take(&mut self.state.entity_lfu_cache), - ); - - self.ctx - .process_block( - &self.logger, - block_ptr, - block_time, - block_data, - handler, - block_state, - proof_of_indexing, - causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - self.inputs.instrument, - ) - .await - } - - fn create_dynamic_data_sources( - &mut self, - created_data_sources: Vec>, - ) -> Result<(Vec>, Vec>), Error> { - let mut data_sources = vec![]; - let mut runtime_hosts = vec![]; - - for info in created_data_sources { - // Try to instantiate a data source from the template - - let data_source = { - let res = match info.template { - DataSourceTemplate::Onchain(_) => C::DataSource::from_template_info(info) - .map(DataSource::Onchain) - .map_err(DataSourceCreationError::from), - DataSourceTemplate::Offchain(_) => offchain::DataSource::from_template_info( - info, - self.ctx.causality_region_next_value(), - ) - .map(DataSource::Offchain), - }; - match res { - Ok(ds) => ds, - Err(e @ DataSourceCreationError::Ignore(..)) => { - warn!(self.logger, "{}", e.to_string()); - continue; - } - Err(DataSourceCreationError::Unknown(e)) => return Err(e), - } - }; - - // Try to create a runtime host for the data source - let host = self - .ctx - .add_dynamic_data_source(&self.logger, data_source.clone())?; - - match host { - Some(host) => { - data_sources.push(data_source); - runtime_hosts.push(host); - } - None => { - warn!( - self.logger, - "no runtime host created, there is already a runtime host instantiated for \ - this data source"; - "name" => &data_source.name(), - "address" => &data_source.address() - .map(hex::encode) - .unwrap_or("none".to_string()), - ) - } - } - } - - Ok((data_sources, runtime_hosts)) - } - - async fn handle_action( - &mut self, - start: Instant, - block_ptr: BlockPtr, - action: Result, - ) -> Result { - self.state.skip_ptr_updates_timer = Instant::now(); - - let elapsed = start.elapsed().as_secs_f64(); - self.metrics - .subgraph - .block_processing_duration - .observe(elapsed); - - match action { - Ok(action) => { - // Once synced, no need to try to update the status again. - if !self.state.synced - && close_to_chain_head( - &block_ptr, - self.inputs.chain.chain_store().chain_head_ptr().await?, - // We consider a subgraph synced when it's at most 1 block behind the - // chain head. - 1, - ) - { - // Updating the sync status is an one way operation. - // This state change exists: not synced -> synced - // This state change does NOT: synced -> not synced - self.inputs.store.deployment_synced()?; - - // Stop trying to update the sync status. - self.state.synced = true; - - // Stop recording time-to-sync metrics. - self.metrics.stream.stopwatch.disable(); - } - - // Keep trying to unfail subgraph for everytime it advances block(s) until it's - // health is not Failed anymore. - if self.state.should_try_unfail_non_deterministic { - // If the deployment head advanced, we can unfail - // the non-deterministic error (if there's any). - let outcome = self - .inputs - .store - .unfail_non_deterministic_error(&block_ptr)?; - - if let UnfailOutcome::Unfailed = outcome { - // Stop trying to unfail. - self.state.should_try_unfail_non_deterministic = false; - self.metrics.stream.deployment_failed.set(0.0); - self.state.backoff.reset(); - } - } - - if let Some(stop_block) = &self.inputs.stop_block { - if block_ptr.number >= *stop_block { - info!(self.logger, "stop block reached for subgraph"); - return Ok(Action::Stop); - } - } - - return Ok(action); - } - Err(BlockProcessingError::Canceled) => { - debug!(self.logger, "Subgraph block stream shut down cleanly"); - return Ok(Action::Stop); - } - - // Handle unexpected stream errors by marking the subgraph as failed. - Err(e) => { - self.metrics.stream.deployment_failed.set(1.0); - - let message = format!("{:#}", e).replace('\n', "\t"); - let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); - let deterministic = e.is_deterministic(); - - let error = SubgraphError { - subgraph_id: self.inputs.deployment.hash.clone(), - message, - block_ptr: Some(block_ptr), - handler: None, - deterministic, - }; - - match deterministic { - true => { - // Fail subgraph: - // - Change status/health. - // - Save the error to the database. - self.inputs - .store - .fail_subgraph(error) - .await - .context("Failed to set subgraph status to `failed`")?; - - return Err(err); - } - false => { - // Shouldn't fail subgraph if it's already failed for non-deterministic - // reasons. - // - // If we don't do this check we would keep adding the same error to the - // database. - let should_fail_subgraph = - self.inputs.store.health().await? != SubgraphHealth::Failed; - - if should_fail_subgraph { - // Fail subgraph: - // - Change status/health. - // - Save the error to the database. - self.inputs - .store - .fail_subgraph(error) - .await - .context("Failed to set subgraph status to `failed`")?; - } - - // Retry logic below: - - let message = format!("{:#}", e).replace('\n', "\t"); - error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; - "attempt" => self.state.backoff.attempt, - "retry_delay_s" => self.state.backoff.delay().as_secs()); - - // Sleep before restarting. - self.state.backoff.sleep_async().await; - - self.state.should_try_unfail_non_deterministic = true; - - // And restart the subgraph. - return Ok(Action::Restart); - } - } - } - } - } - - fn persist_dynamic_data_sources( - &mut self, - block_state: &mut BlockState, - data_sources: Vec>, - ) { - if !data_sources.is_empty() { - debug!( - self.logger, - "Creating {} dynamic data source(s)", - data_sources.len() - ); - } - - // Add entity operations to the block state in order to persist - // the dynamic data sources - for data_source in data_sources.iter() { - debug!( - self.logger, - "Persisting data_source"; - "name" => &data_source.name(), - "address" => &data_source.address().map(hex::encode).unwrap_or("none".to_string()), - ); - block_state.persist_data_source(data_source.as_stored_dynamic_data_source()); - } - } -} - -impl SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - async fn handle_stream_event( - &mut self, - event: Option, CancelableError>>, - cancel_handle: &CancelHandle, - ) -> Result { - let action = match event { - Some(Ok(BlockStreamEvent::ProcessWasmBlock( - block_ptr, - block_time, - data, - handler, - cursor, - ))) => { - let _section = self - .metrics - .stream - .stopwatch - .start_section(PROCESS_WASM_BLOCK_SECTION_NAME); - self.handle_process_wasm_block( - block_ptr, - block_time, - data, - handler, - cursor, - cancel_handle, - ) - .await? - } - Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { - let _section = self - .metrics - .stream - .stopwatch - .start_section(PROCESS_BLOCK_SECTION_NAME); - self.handle_process_block(block, cursor, cancel_handle) - .await? - } - Some(Ok(BlockStreamEvent::Revert(revert_to_ptr, cursor))) => { - let _section = self - .metrics - .stream - .stopwatch - .start_section(HANDLE_REVERT_SECTION_NAME); - self.handle_revert(revert_to_ptr, cursor).await? - } - // Log and drop the errors from the block_stream - // The block stream will continue attempting to produce blocks - Some(Err(e)) => self.handle_err(e, cancel_handle).await?, - // If the block stream ends, that means that there is no more indexing to do. - // Typically block streams produce indefinitely, but tests are an example of finite block streams. - None => Action::Stop, - }; - - Ok(action) - } - - async fn handle_offchain_triggers( - &mut self, - triggers: Vec, - block: &Arc, - ) -> Result< - ( - Vec, - Vec, - Vec, - ), - Error, - > { - let mut mods = vec![]; - let mut processed_data_sources = vec![]; - let mut persisted_data_sources = vec![]; - - for trigger in triggers { - // Using an `EmptyStore` and clearing the cache for each trigger is a makeshift way to - // get causality region isolation. - let schema = ReadStore::input_schema(&self.inputs.store); - let mut block_state = BlockState::::new(EmptyStore::new(schema), LfuCache::new()); - - // PoI ignores offchain events. - // See also: poi-ignores-offchain - let proof_of_indexing = None; - let causality_region = ""; - - block_state = self - .ctx - .process_trigger( - &self.logger, - block, - &TriggerData::Offchain(trigger), - block_state, - &proof_of_indexing, - causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - self.inputs.instrument, - ) - .await - .map_err(move |err| { - let err = match err { - // Ignoring `PossibleReorg` isn't so bad since the subgraph will retry - // non-deterministic errors. - MappingError::PossibleReorg(e) | MappingError::Unknown(e) => e, - }; - err.context("failed to process trigger".to_string()) - })?; - - anyhow::ensure!( - !block_state.has_created_on_chain_data_sources(), - "Attempted to create on-chain data source in offchain data source handler. This is not yet supported.", - ); - - let (data_sources, _) = - self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; - - // Add entity operations for the new data sources to the block state - // and add runtimes for the data sources to the subgraph instance. - self.persist_dynamic_data_sources(&mut block_state, data_sources); - - // This propagates any deterministic error as a non-deterministic one. Which might make - // sense considering offchain data sources are non-deterministic. - if let Some(err) = block_state.deterministic_errors.into_iter().next() { - return Err(anyhow!("{}", err.to_string())); - } - - mods.extend( - block_state - .entity_cache - .as_modifications(block.number())? - .modifications, - ); - processed_data_sources.extend(block_state.processed_data_sources); - persisted_data_sources.extend(block_state.persisted_data_sources) - } - - Ok((mods, processed_data_sources, persisted_data_sources)) - } -} - -#[derive(Debug)] -enum Action { - Continue, - Stop, - Restart, -} - -impl Action { - /// Return `true` if the action indicates that we are done with a block - fn block_finished(&self) -> bool { - match self { - Action::Continue | Action::Restart => false, - Action::Stop => true, - } - } -} - -#[async_trait] -trait StreamEventHandler { - async fn handle_process_wasm_block( - &mut self, - block_ptr: BlockPtr, - block_time: BlockTime, - block_data: Box<[u8]>, - handler: String, - cursor: FirehoseCursor, - cancel_handle: &CancelHandle, - ) -> Result; - async fn handle_process_block( - &mut self, - block: BlockWithTriggers, - cursor: FirehoseCursor, - cancel_handle: &CancelHandle, - ) -> Result; - async fn handle_revert( - &mut self, - revert_to_ptr: BlockPtr, - cursor: FirehoseCursor, - ) -> Result; - async fn handle_err( - &mut self, - err: CancelableError, - cancel_handle: &CancelHandle, - ) -> Result; - fn needs_restart(&self, revert_to_ptr: BlockPtr, subgraph_ptr: BlockPtr) -> bool; -} - -#[async_trait] -impl StreamEventHandler for SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - async fn handle_process_wasm_block( - &mut self, - block_ptr: BlockPtr, - block_time: BlockTime, - block_data: Box<[u8]>, - handler: String, - cursor: FirehoseCursor, - cancel_handle: &CancelHandle, - ) -> Result { - let logger = self.logger.new(o!( - "block_number" => format!("{:?}", block_ptr.number), - "block_hash" => format!("{}", block_ptr.hash) - )); - - debug!(logger, "Start processing wasm block";); - - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; - - // Causality region for onchain triggers. - let causality_region = PoICausalityRegion::from_network(&self.inputs.network); - - let mut block_state = { - match self - .process_wasm_block( - &proof_of_indexing, - block_ptr.clone(), - block_time, - block_data, - handler, - &causality_region, - ) - .await - { - // Triggers processed with no errors or with only deterministic errors. - Ok(block_state) => block_state, - - // Some form of unknown or non-deterministic error ocurred. - Err(MappingError::Unknown(e)) => { - return Err(BlockProcessingError::Unknown(e).into()) - } - Err(MappingError::PossibleReorg(e)) => { - info!(logger, - "Possible reorg detected, retrying"; - "error" => format!("{:#}", e), - ); - - // In case of a possible reorg, we want this function to do nothing and restart the - // block stream so it has a chance to detect the reorg. - // - // The state is unchanged at this point, except for having cleared the entity cache. - // Losing the cache is a bit annoying but not an issue for correctness. - // - // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. - return Ok(Action::Restart); - } - } - }; - - let has_errors = block_state.has_errors(); - let is_non_fatal_errors_active = self - .inputs - .features - .contains(&SubgraphFeature::NonFatalErrors); - - // Apply entity operations and advance the stream - - // Avoid writing to store if block stream has been canceled - if cancel_handle.is_canceled() { - return Err(BlockProcessingError::Canceled.into()); - } - - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); - update_proof_of_indexing( - proof_of_indexing, - block_time, - &self.metrics.host.stopwatch, - &mut block_state.entity_cache, - ) - .await?; - } - - let section = self - .metrics - .host - .stopwatch - .start_section("as_modifications"); - let ModificationsAndCache { - modifications: mut mods, - entity_lfu_cache: cache, - evict_stats, - } = block_state - .entity_cache - .as_modifications(block_ptr.number) - .map_err(|e| BlockProcessingError::Unknown(e.into()))?; - section.end(); - - debug!(self.logger, "Entity cache statistics"; - "weight" => evict_stats.new_weight, - "evicted_weight" => evict_stats.evicted_weight, - "count" => evict_stats.new_count, - "evicted_count" => evict_stats.evicted_count, - "stale_update" => evict_stats.stale_update, - "hit_rate" => format!("{:.0}%", evict_stats.hit_rate_pct()), - "accesses" => evict_stats.accesses, - "evict_time_ms" => evict_stats.evict_time.as_millis()); - - // Put the cache back in the state, asserting that the placeholder cache was not used. - assert!(self.state.entity_lfu_cache.is_empty()); - self.state.entity_lfu_cache = cache; - - if !mods.is_empty() { - info!(&logger, "Applying {} entity operation(s)", mods.len()); - } - - let err_count = block_state.deterministic_errors.len(); - for (i, e) in block_state.deterministic_errors.iter().enumerate() { - let message = format!("{:#}", e).replace('\n', "\t"); - error!(&logger, "Subgraph error {}/{}", i + 1, err_count; - "error" => message, - "code" => LogCode::SubgraphSyncingFailure - ); - } - - // Transact entity operations into the store and update the - // subgraph's block stream pointer - let _section = self.metrics.host.stopwatch.start_section("transact_block"); - let start = Instant::now(); - - let store = &self.inputs.store; - - // If a deterministic error has happened, make the PoI to be the only entity that'll be stored. - if has_errors && !is_non_fatal_errors_active { - let is_poi_entity = - |entity_mod: &EntityModification| entity_mod.key().entity_type.is_poi(); - mods.retain(is_poi_entity); - // Confidence check - assert!( - mods.len() == 1, - "There should be only one PoI EntityModification" - ); - } - - let BlockState { - deterministic_errors, - .. - } = block_state; - - let first_error = deterministic_errors.first().cloned(); - - store - .transact_block_operations( - block_ptr, - block_time, - cursor, - mods, - &self.metrics.host.stopwatch, - vec![], - deterministic_errors, - vec![], - is_non_fatal_errors_active, - ) - .await - .context("Failed to transact block operations")?; - - // For subgraphs with `nonFatalErrors` feature disabled, we consider - // any error as fatal. - // - // So we do an early return to make the subgraph stop processing blocks. - // - // In this scenario the only entity that is stored/transacted is the PoI, - // all of the others are discarded. - if has_errors && !is_non_fatal_errors_active { - // Only the first error is reported. - return Err(BlockProcessingError::Deterministic(first_error.unwrap()).into()); - } - - let elapsed = start.elapsed().as_secs_f64(); - self.metrics - .subgraph - .block_ops_transaction_duration - .observe(elapsed); - - // To prevent a buggy pending version from replacing a current version, if errors are - // present the subgraph will be unassigned. - if has_errors && !ENV_VARS.disable_fail_fast && !store.is_deployment_synced().await? { - store - .unassign_subgraph() - .map_err(|e| BlockProcessingError::Unknown(e.into()))?; - - // Use `Canceled` to avoiding setting the subgraph health to failed, an error was - // just transacted so it will be already be set to unhealthy. - return Err(BlockProcessingError::Canceled.into()); - }; - - Ok(Action::Continue) - } - - async fn handle_process_block( - &mut self, - block: BlockWithTriggers, - cursor: FirehoseCursor, - cancel_handle: &CancelHandle, - ) -> Result { - let block_ptr = block.ptr(); - self.metrics - .stream - .deployment_head - .set(block_ptr.number as f64); - - if block.trigger_count() > 0 { - self.metrics - .subgraph - .block_trigger_count - .observe(block.trigger_count() as f64); - } - - if block.trigger_count() == 0 - && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD - && !self.state.synced - && !close_to_chain_head( - &block_ptr, - self.inputs.chain.chain_store().chain_head_ptr().await?, - // The "skip ptr updates timer" is ignored when a subgraph is at most 1000 blocks - // behind the chain head. - 1000, - ) - { - return Ok(Action::Continue); - } else { - self.state.skip_ptr_updates_timer = Instant::now(); - } - - let start = Instant::now(); - - let res = self.process_block(cancel_handle, block, cursor).await; - - self.handle_action(start, block_ptr, res).await - } - - async fn handle_revert( - &mut self, - revert_to_ptr: BlockPtr, - cursor: FirehoseCursor, - ) -> Result { - // Current deployment head in the database / WritableAgent Mutex cache. - // - // Safe unwrap because in a Revert event we're sure the subgraph has - // advanced at least once. - let subgraph_ptr = self.inputs.store.block_ptr().unwrap(); - if revert_to_ptr.number >= subgraph_ptr.number { - info!(&self.logger, "Block to revert is higher than subgraph pointer, nothing to do"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); - return Ok(Action::Continue); - } - - info!(&self.logger, "Reverting block to get back to main chain"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); - - if let Err(e) = self - .inputs - .store - .revert_block_operations(revert_to_ptr.clone(), cursor) - .await - { - error!(&self.logger, "Could not revert block. Retrying"; "error" => %e); - - // Exit inner block stream consumption loop and go up to loop that restarts subgraph - return Ok(Action::Restart); - } - - self.metrics - .stream - .reverted_blocks - .set(subgraph_ptr.number as f64); - self.metrics - .stream - .deployment_head - .set(subgraph_ptr.number as f64); - - self.revert_state_to(revert_to_ptr.number)?; - - let needs_restart: bool = self.needs_restart(revert_to_ptr, subgraph_ptr); - - let action = if needs_restart { - Action::Restart - } else { - Action::Continue - }; - - Ok(action) - } - - async fn handle_err( - &mut self, - err: CancelableError, - cancel_handle: &CancelHandle, - ) -> Result { - if cancel_handle.is_canceled() { - debug!(&self.logger, "Subgraph block stream shut down cleanly"); - return Ok(Action::Stop); - } - - debug!( - &self.logger, - "Block stream produced a non-fatal error"; - "error" => format!("{}", err), - ); - - Ok(Action::Continue) - } - - /// Determines if the subgraph needs to be restarted. - /// Currently returns true when there are data sources that have reached their end block - /// in the range between `revert_to_ptr` and `subgraph_ptr`. - fn needs_restart(&self, revert_to_ptr: BlockPtr, subgraph_ptr: BlockPtr) -> bool { - self.inputs - .end_blocks - .range(revert_to_ptr.number..=subgraph_ptr.number) - .next() - .is_some() - } -} - -/// Transform the proof of indexing changes into entity updates that will be -/// inserted when as_modifications is called. -async fn update_proof_of_indexing( - proof_of_indexing: ProofOfIndexing, - block_time: BlockTime, - stopwatch: &StopwatchMetrics, - entity_cache: &mut EntityCache, -) -> Result<(), Error> { - // Helper to store the digest as a PoI entity in the cache - fn store_poi_entity( - entity_cache: &mut EntityCache, - key: EntityKey, - digest: Bytes, - block_time: BlockTime, - ) -> Result<(), Error> { - let digest_name = entity_cache.schema.poi_digest(); - let mut data = vec![ - ( - graph::data::store::ID.clone(), - Value::from(key.entity_id.to_string()), - ), - (digest_name, Value::from(digest)), - ]; - if entity_cache.schema.has_aggregations() { - let block_time = Value::Int8(block_time.as_secs_since_epoch() as i64); - data.push((entity_cache.schema.poi_block_time(), block_time)); - } - let poi = entity_cache.make_entity(data)?; - entity_cache.set(key, poi) - } - - let _section_guard = stopwatch.start_section("update_proof_of_indexing"); - - let mut proof_of_indexing = proof_of_indexing.take(); - - for (causality_region, stream) in proof_of_indexing.drain() { - // Create the special POI entity key specific to this causality_region - // There are two things called causality regions here, one is the causality region for - // the poi which is a string and the PoI entity id. The other is the data source - // causality region to which the PoI belongs as an entity. Currently offchain events do - // not affect PoI so it is assumed to be `ONCHAIN`. - // See also: poi-ignores-offchain - let entity_key = entity_cache - .schema - .poi_type() - .key_in(causality_region, CausalityRegion::ONCHAIN); - - // Grab the current digest attribute on this entity - let poi_digest = entity_cache.schema.poi_digest().clone(); - let prev_poi = entity_cache - .get(&entity_key, GetScope::Store) - .map_err(Error::from)? - .map(|entity| match entity.get(poi_digest.as_str()) { - Some(Value::Bytes(b)) => b.clone(), - _ => panic!("Expected POI entity to have a digest and for it to be bytes"), - }); - - // Finish the POI stream, getting the new POI value. - let updated_proof_of_indexing = stream.pause(prev_poi.as_deref()); - let updated_proof_of_indexing: Bytes = (&updated_proof_of_indexing[..]).into(); - - // Put this onto an entity with the same digest attribute - // that was expected before when reading. - store_poi_entity( - entity_cache, - entity_key, - updated_proof_of_indexing, - block_time, - )?; - } - - Ok(()) -} - -/// Checks if the Deployment BlockPtr is at least X blocks behind to the chain head. -fn close_to_chain_head( - deployment_head_ptr: &BlockPtr, - chain_head_ptr: Option, - n: BlockNumber, -) -> bool { - matches!((deployment_head_ptr, &chain_head_ptr), (b1, Some(b2)) if b1.number >= (b2.number - n)) -} - -#[test] -fn test_close_to_chain_head() { - let offset = 1; - - let block_0 = BlockPtr::try_from(( - "bd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f", - 0, - )) - .unwrap(); - let block_1 = BlockPtr::try_from(( - "8511fa04b64657581e3f00e14543c1d522d5d7e771b54aa3060b662ade47da13", - 1, - )) - .unwrap(); - let block_2 = BlockPtr::try_from(( - "b98fb783b49de5652097a989414c767824dff7e7fd765a63b493772511db81c1", - 2, - )) - .unwrap(); - - assert!(!close_to_chain_head(&block_0, None, offset)); - assert!(!close_to_chain_head(&block_2, None, offset)); - - assert!(!close_to_chain_head( - &block_0, - Some(block_2.clone()), - offset - )); - - assert!(close_to_chain_head(&block_1, Some(block_2.clone()), offset)); - assert!(close_to_chain_head(&block_2, Some(block_2.clone()), offset)); -} diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs new file mode 100644 index 00000000000..026be3dfc27 --- /dev/null +++ b/core/src/subgraph/runner/mod.rs @@ -0,0 +1,1822 @@ +mod state; +mod trigger_runner; + +use crate::subgraph::context::IndexingContext; +use crate::subgraph::error::{ + ClassifyErrorHelper as _, DetailHelper as _, NonDeterministicErrorHelper as _, ProcessingError, + ProcessingErrorKind, +}; +use crate::subgraph::inputs::IndexingInputs; +use crate::subgraph::state::IndexingState; +use crate::subgraph::stream::new_block_stream; +use anyhow::Context as _; +use graph::blockchain::block_stream::{ + BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, +}; +use graph::blockchain::{ + Block, BlockTime, Blockchain, DataSource as _, SubgraphFilter, Trigger, TriggerFilter as _, + TriggerFilterWrapper, +}; +use graph::components::store::{ + EmptyStore, GetScope, ReadStore, SeqGenerator, StoredDynamicDataSource, +}; +use graph::components::subgraph::InstanceDSTemplate; +use graph::components::trigger_processor::RunnableTriggers; +use graph::components::{ + store::ModificationsAndCache, + subgraph::{MappingError, PoICausalityRegion, ProofOfIndexing, SharedProofOfIndexing}, +}; +use graph::data::store::scalar::Bytes; +use graph::data::subgraph::schema::SubgraphError; +use graph::data_source::{ + CausalityRegion, DataSource, DataSourceCreationError, TriggerData, offchain, +}; +use graph::env::EnvVars; +use graph::ext::futures::Cancelable; +use graph::futures03::stream::StreamExt; +use graph::prelude::{ + BlockNumber, BlockPtr, BlockState, CancelGuard, CancelHandle, CancelToken as _, + CheapClone as _, ENV_VARS, EntityCache, EntityModification, Error, InstanceDSTemplateInfo, + LogCode, RunnerMetrics, RuntimeHostBuilder, StopwatchMetrics, StoreError, StreamExtension, + UnfailOutcome, Value, anyhow, hex, retry, thiserror, +}; +use graph::schema::EntityKey; +use graph::slog::{Logger, debug, error, info, o, trace, warn}; +use graph::util::lfu_cache::EvictStats; +use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use std::vec; + +use self::state::{RestartReason, RunnerState, StopReason}; +use self::trigger_runner::TriggerRunner; + +const MINUTE: Duration = Duration::from_secs(60); + +const SKIP_PTR_UPDATES_THRESHOLD: Duration = Duration::from_secs(60 * 5); +const HANDLE_REVERT_SECTION_NAME: &str = "handle_revert"; +const PROCESS_BLOCK_SECTION_NAME: &str = "process_block"; +const PROCESS_TRIGGERS_SECTION_NAME: &str = "process_triggers"; +const HANDLE_CREATED_DS_SECTION_NAME: &str = "handle_new_data_sources"; + +pub struct SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + ctx: IndexingContext, + state: IndexingState, + inputs: Arc>, + logger: Logger, + pub metrics: RunnerMetrics, + cancel_handle: Option, + /// The current state in the runner's state machine. + /// This field drives the main loop of the runner. + runner_state: RunnerState, +} + +#[derive(Debug, thiserror::Error)] +pub enum SubgraphRunnerError { + #[error("subgraph runner terminated because a newer one was active")] + Duplicate, + + #[error(transparent)] + Unknown(#[from] Error), +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + pub fn new( + inputs: IndexingInputs, + ctx: IndexingContext, + logger: Logger, + metrics: RunnerMetrics, + env_vars: Arc, + ) -> Self { + Self { + inputs: Arc::new(inputs), + ctx, + state: IndexingState { + should_try_unfail_non_deterministic: true, + skip_ptr_updates_timer: Instant::now(), + backoff: ExponentialBackoff::with_jitter( + (MINUTE * 2).min(env_vars.subgraph_error_retry_ceil), + env_vars.subgraph_error_retry_ceil, + env_vars.subgraph_error_retry_jitter, + ), + entity_lfu_cache: LfuCache::new(), + cached_head_ptr: None, + postponed_indexes_created: false, + }, + logger, + metrics, + cancel_handle: None, + runner_state: RunnerState::Initializing, + } + } + + /// Revert the state to a previous block. When handling revert operations + /// or failed block processing, it is necessary to remove part of the existing + /// in-memory state to keep it constent with DB changes. + /// During block processing new dynamic data sources are added directly to the + /// IndexingContext of the runner. This means that if, for whatever reason, + /// the changes don;t complete then the remnants of that block processing must + /// be removed. The same thing also applies to the block cache. + /// This function must be called before continuing to process in order to avoid + /// duplicated host insertion and POI issues with dirty entity changes. + fn revert_state_to(&mut self, block_number: BlockNumber) { + self.state.entity_lfu_cache = LfuCache::new(); + + // 1. Revert all hosts(created by DDS) at a block higher than `block_number`. + // 2. Unmark any offchain data sources that were marked done on the blocks being removed. + // When no offchain datasources are present, 2. should be a noop. + self.ctx.revert_data_sources(block_number + 1); + } + + #[cfg(debug_assertions)] + pub fn context(&self) -> &IndexingContext { + &self.ctx + } + + #[cfg(debug_assertions)] + pub async fn run_for_test(self, break_on_restart: bool) -> Result { + self.run_inner(break_on_restart).await.map_err(Into::into) + } + + fn is_static_filters_enabled(&self) -> bool { + self.inputs.static_filters || self.ctx.hosts_len() > ENV_VARS.static_filters_threshold + } + + fn build_filter(&self) -> TriggerFilterWrapper { + let current_ptr = self.inputs.store.block_ptr(); + let static_filters = self.is_static_filters_enabled(); + + // Filter out data sources that have reached their end block + let end_block_filter = |ds: &&C::DataSource| match current_ptr.as_ref() { + // We filter out datasources for which the current block is at or past their end block. + Some(block) => ds.end_block().is_none_or(|end| block.number < end), + // If there is no current block, we keep all datasources. + None => true, + }; + + let data_sources = self.ctx.static_data_sources(); + + let subgraph_filter = data_sources + .iter() + .filter_map(|ds| ds.as_subgraph()) + .map(|ds| SubgraphFilter { + subgraph: ds.source.address(), + start_block: ds.source.start_block, + entities: ds + .mapping + .handlers + .iter() + .map(|handler| handler.entity.clone()) + .collect(), + manifest_idx: ds.manifest_idx, + }) + .collect::>(); + + // if static_filters is not enabled we just stick to the filter based on all the data sources. + if !static_filters { + return TriggerFilterWrapper::new( + C::TriggerFilter::from_data_sources( + self.ctx.onchain_data_sources().filter(end_block_filter), + ), + subgraph_filter, + ); + } + + // if static_filters is enabled, build a minimal filter with the static data sources and + // add the necessary filters based on templates. + // This specifically removes dynamic data sources based filters because these can be derived + // from templates AND this reduces the cost of egress traffic by making the payloads smaller. + + if !self.inputs.static_filters { + info!(self.logger, "forcing subgraph to use static filters.") + } + + let data_sources = self.ctx.static_data_sources(); + + let mut filter = C::TriggerFilter::from_data_sources( + data_sources + .iter() + .filter_map(|ds| ds.as_onchain()) + // Filter out data sources that have reached their end block if the block is final. + .filter(end_block_filter), + ); + + let templates = self.ctx.templates(); + + filter.extend_with_template(templates.iter().filter_map(|ds| ds.as_onchain()).cloned()); + + TriggerFilterWrapper::new(filter, subgraph_filter) + } + + #[cfg(debug_assertions)] + pub fn build_filter_for_test(&self) -> TriggerFilterWrapper { + self.build_filter() + } + + async fn start_block_stream(&mut self) -> Result>>, Error> { + let block_stream_canceler = CancelGuard::new(); + let block_stream_cancel_handle = block_stream_canceler.handle(); + // TriggerFilter needs to be rebuilt eveytime the blockstream is restarted + let filter = self.build_filter(); + + let block_stream = new_block_stream(&self.inputs, filter, &self.metrics.subgraph) + .await? + .cancelable(&block_stream_canceler); + + self.cancel_handle = Some(block_stream_cancel_handle); + + // Keep the stream's cancel guard around to be able to shut it down when the subgraph + // deployment is unassigned + self.ctx + .instances + .insert(self.inputs.deployment.id, block_stream_canceler); + + Ok(block_stream) + } + + fn is_canceled(&self) -> bool { + if let Some(ref cancel_handle) = self.cancel_handle { + cancel_handle.is_canceled() + } else { + false + } + } + + /// Initialize the runner by performing pre-loop setup. + /// + /// This method handles: + /// - Updating the deployment synced metric + /// - Attempting to unfail deterministic errors from the previous run + /// - Checking if the subgraph has already reached its max end block + /// + /// Returns the next state to transition to: + /// - `Restarting` to start the block stream (normal case) + /// - `Stopped` if the max end block was already reached + async fn initialize(&self) -> Result, SubgraphRunnerError> { + self.update_deployment_synced_metric(); + + // If a subgraph failed for deterministic reasons, before start indexing, we first + // revert the deployment head. It should lead to the same result since the error was + // deterministic. + if let Some(current_ptr) = self.inputs.store.block_ptr() { + if let Some(parent_ptr) = self + .inputs + .triggers_adapter + .parent_ptr(¤t_ptr) + .await? + { + // This reverts the deployment head to the parent_ptr if + // deterministic errors happened. + // + // There's no point in calling it if we have no current or parent block + // pointers, because there would be: no block to revert to or to search + // errors from (first execution). + // + // We attempt to unfail deterministic errors to mitigate deterministic + // errors caused by wrong data being consumed from the providers. It has + // been a frequent case in the past so this helps recover on a larger scale. + let _outcome = self + .inputs + .store + .unfail_deterministic_error(¤t_ptr, &parent_ptr) + .await?; + } + + // Stop subgraph when we reach maximum endblock. + if let Some(max_end_block) = self.inputs.max_end_block + && max_end_block <= current_ptr.block_number() + { + info!(self.logger, "Stopping subgraph as we reached maximum endBlock"; + "max_end_block" => max_end_block, + "current_block" => current_ptr.block_number()); + self.inputs.store.flush().await?; + return Ok(RunnerState::Stopped { + reason: StopReason::MaxEndBlockReached, + }); + } + } + + // Normal case: proceed to start the block stream + Ok(RunnerState::Restarting { + reason: RestartReason::StoreError, // Initial start uses the same path as restart + }) + } + + /// Await the next block stream event and transition to the appropriate state. + /// + /// This method waits for the next event from the block stream and determines + /// which state the runner should transition to: + /// - `ProcessingBlock` for new blocks to process + /// - `Reverting` for revert events + /// - `Stopped` when the stream ends or is canceled + /// - Returns back to `AwaitingBlock` for non-fatal errors that allow continuation + async fn await_block( + &mut self, + mut block_stream: Cancelable>>, + ) -> Result, SubgraphRunnerError> { + let event = { + let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); + block_stream.next().await + }; + + if self.is_canceled() { + return self.cancel(); + } + + match event { + Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { + Ok(RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + }) + } + Some(Ok(BlockStreamEvent::Revert(to_ptr, cursor))) => Ok(RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + }), + // Log and drop the errors from the block_stream + // The block stream will continue attempting to produce blocks + Some(Err(e)) => { + // Log error and continue waiting for blocks + debug!( + &self.logger, + "Block stream produced a non-fatal error"; + "error" => format!("{}", e), + ); + Ok(RunnerState::AwaitingBlock { block_stream }) + } + // If the block stream ends, that means that there is no more indexing to do. + None => Ok(RunnerState::Stopped { + reason: StopReason::StreamEnded, + }), + } + } + + fn cancel(&mut self) -> Result, SubgraphRunnerError> { + if self.ctx.instances.contains(&self.inputs.deployment.id) { + warn!( + self.logger, + "Terminating the subgraph runner because a newer one is active. \ + Possible reassignment detected while the runner was in a non-cancellable pending state", + ); + return Err(SubgraphRunnerError::Duplicate); + } + warn!( + self.logger, + "Terminating the subgraph runner because subgraph was unassigned", + ); + Ok(RunnerState::Stopped { + reason: StopReason::Unassigned, + }) + } + + /// Construct a SubgraphError and mark the subgraph as failed in the store. + async fn fail_subgraph( + &mut self, + message: String, + block_ptr: Option, + deterministic: bool, + ) -> Result<(), SubgraphRunnerError> { + let error = SubgraphError { + subgraph_id: self.inputs.deployment.hash.clone(), + message, + block_ptr, + handler: None, + deterministic, + }; + self.inputs + .store + .fail_subgraph(error) + .await + .context("Failed to set subgraph status to `failed`")?; + Ok(()) + } + + /// Handle a restart by potentially restarting the store and starting a new block stream. + /// + /// This method handles: + /// - Restarting the store if there were errors (to clear error state) + /// - Reverting state to the last good block if the store was restarted + /// - Starting a new block stream with updated filters + /// + /// Returns the next state to transition to: + /// - `AwaitingBlock` with the new block stream (normal case) + async fn restart( + &mut self, + reason: RestartReason, + ) -> Result, SubgraphRunnerError> { + debug!(self.logger, "Starting or restarting subgraph"; "reason" => ?reason); + + // If restarting due to store error, try to restart the store + if matches!(reason, RestartReason::StoreError) { + let store = self.inputs.store.cheap_clone(); + if let Some(store) = store.restart().await? { + let last_good_block = store.block_ptr().map(|ptr| ptr.number).unwrap_or(0); + self.revert_state_to(last_good_block); + self.inputs = Arc::new(self.inputs.with_store(store)); + } + } + + let block_stream = self.start_block_stream().await?; + + debug!(self.logger, "Started block stream"); + self.metrics.subgraph.deployment_status.running(); + self.update_deployment_synced_metric(); + + Ok(RunnerState::AwaitingBlock { block_stream }) + } + + /// Finalize the runner when it reaches a terminal state. + /// + /// This method handles cleanup tasks when the runner stops: + /// - Flushing the store to ensure all changes are persisted + /// - Logging the stop reason + async fn finalize(self, reason: StopReason) -> Result { + match reason { + StopReason::MaxEndBlockReached => { + info!(self.logger, "Stopping subgraph - max end block reached"); + } + StopReason::Canceled => { + info!(self.logger, "Stopping subgraph - canceled"); + } + StopReason::Unassigned => { + info!(self.logger, "Stopping subgraph - unassigned"); + } + StopReason::StreamEnded => { + info!(self.logger, "Stopping subgraph - stream ended"); + } + } + + self.inputs.store.flush().await?; + Ok(self) + } + + pub async fn run(self) -> Result<(), SubgraphRunnerError> { + self.run_inner(false).await.map(|_| ()) + } + + /// Main state machine loop for the subgraph runner. + /// + /// This method drives the runner through its state machine, transitioning + /// between states based on events and actions. The state machine replaces + /// the previous nested loop structure with explicit state transitions. + /// + /// ## State Machine + /// + /// The runner starts in `Initializing` and transitions through states: + /// - `Initializing` → `Restarting` (or `Stopped` if max end block reached) + /// - `Restarting` → `AwaitingBlock` + /// - `AwaitingBlock` → `ProcessingBlock`, `Reverting`, or `Stopped` + /// - `ProcessingBlock` → `AwaitingBlock` or `Restarting` + /// - `Reverting` → `AwaitingBlock` or `Restarting` + /// - `Stopped` → terminal (returns) + async fn run_inner(mut self, break_on_restart: bool) -> Result { + // Start in Initializing state + self.runner_state = RunnerState::Initializing; + + // Track whether we've started processing blocks (not just initialized). + // This is used for break_on_restart logic - we should only stop on restart + // after we've actually started processing, not on the initial "restart" + // which is really the first start of the block stream. + let mut has_processed_blocks = false; + + loop { + self.runner_state = match std::mem::take(&mut self.runner_state) { + RunnerState::Initializing => self.initialize().await?, + + RunnerState::Restarting { reason } => { + if break_on_restart && has_processed_blocks { + // In test mode, stop on restart after first block processing + info!(self.logger, "Stopping subgraph on break"); + RunnerState::Stopped { + reason: StopReason::Canceled, + } + } else { + self.restart(reason).await? + } + } + + RunnerState::AwaitingBlock { block_stream } => { + self.await_block(block_stream).await? + } + + RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + } => { + has_processed_blocks = true; + self.process_block_state(block_stream, block, cursor) + .await? + } + + RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + } => self.handle_revert_state(block_stream, to_ptr, cursor).await, + + RunnerState::Stopped { reason } => { + return self.finalize(reason).await; + } + }; + } + } + + /// Process a block and determine the next state. + /// + /// This is the state machine wrapper around `process_block` that handles + /// the block processing action and determines state transitions. + async fn process_block_state( + &mut self, + block_stream: Cancelable>>, + block: BlockWithTriggers, + cursor: FirehoseCursor, + ) -> Result, SubgraphRunnerError> { + let block_ptr = block.ptr(); + self.metrics + .stream + .deployment_head + .set(block_ptr.number as f64); + + if block.trigger_count() > 0 { + self.metrics + .subgraph + .block_trigger_count + .observe(block.trigger_count() as f64); + } + + // Check if we should skip this block (optimization for blocks without triggers). + // Do not skip if max_end_block has been reached — fall through to process_block so the + // block pointer is persisted and the existing max_end_block check in handle_action fires. + let max_end_block_reached = self + .inputs + .max_end_block + .is_some_and(|max| block_ptr.number >= max); + if block.trigger_count() == 0 + && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD + && !self.inputs.store.is_deployment_synced() + && !close_to_chain_head(&block_ptr, &self.inputs.chain.chain_head_ptr().await?, 1000) + && !max_end_block_reached + { + // Skip this block and continue with the same stream + return Ok(RunnerState::AwaitingBlock { block_stream }); + } else { + self.state.skip_ptr_updates_timer = Instant::now(); + } + + let block_start = Instant::now(); + + let action = { + let stopwatch = &self.metrics.stream.stopwatch; + let _section = stopwatch.start_section(PROCESS_BLOCK_SECTION_NAME); + self.process_block(block, cursor).await + }; + + let action = self.handle_action(block_start, block_ptr, action).await?; + + self.update_deployment_synced_metric(); + + if self.is_canceled() { + return self.cancel(); + } + + self.metrics + .subgraph + .observe_block_processed(block_start.elapsed(), action.block_finished()); + + // Convert Action to RunnerState + match action { + Action::Continue => Ok(RunnerState::AwaitingBlock { block_stream }), + Action::Restart => Ok(RunnerState::Restarting { + reason: RestartReason::DynamicDataSourceCreated, + }), + Action::Stop => Ok(RunnerState::Stopped { + reason: StopReason::MaxEndBlockReached, + }), + } + } + + /// Handle a revert event and determine the next state. + /// + /// This is the state machine wrapper around `handle_revert` that handles + /// the revert action and determines state transitions. + async fn handle_revert_state( + &mut self, + block_stream: Cancelable>>, + revert_to_ptr: BlockPtr, + cursor: FirehoseCursor, + ) -> RunnerState { + let stopwatch = &self.metrics.stream.stopwatch; + let _section = stopwatch.start_section(HANDLE_REVERT_SECTION_NAME); + + let action = self.handle_revert(revert_to_ptr, cursor).await; + + match action { + Action::Continue => RunnerState::AwaitingBlock { block_stream }, + Action::Restart => RunnerState::Restarting { + reason: RestartReason::StoreError, + }, + Action::Stop => RunnerState::Stopped { + reason: StopReason::Canceled, + }, + } + } + + async fn transact_block_state( + &mut self, + logger: &Logger, + block_ptr: BlockPtr, + firehose_cursor: FirehoseCursor, + block_time: BlockTime, + block_state: BlockState, + proof_of_indexing: SharedProofOfIndexing, + offchain_mods: Vec, + processed_offchain_data_sources: Vec, + ) -> Result<(), ProcessingError> { + fn log_evict_stats(logger: &Logger, evict_stats: &EvictStats) { + trace!(logger, "Entity cache statistics"; + "weight" => evict_stats.new_weight, + "evicted_weight" => evict_stats.evicted_weight, + "count" => evict_stats.new_count, + "evicted_count" => evict_stats.evicted_count, + "stale_update" => evict_stats.stale_update, + "hit_rate" => format!("{:.0}%", evict_stats.hit_rate_pct()), + "accesses" => evict_stats.accesses, + "evict_time_ms" => evict_stats.evict_time.as_millis()); + } + + let BlockState { + deterministic_errors, + persisted_data_sources, + metrics: block_state_metrics, + mut entity_cache, + .. + } = block_state; + let first_error = deterministic_errors.first().cloned(); + let has_errors = first_error.is_some(); + + // Avoid writing to store if block stream has been canceled + if self.is_canceled() { + return Err(ProcessingError::Canceled); + } + + if let Some(proof_of_indexing) = proof_of_indexing.into_inner() { + update_proof_of_indexing( + proof_of_indexing, + block_time, + &self.metrics.host.stopwatch, + &mut entity_cache, + ) + .await + .non_deterministic()?; + } + + let section = self + .metrics + .host + .stopwatch + .start_section("as_modifications"); + let ModificationsAndCache { + modifications: mut mods, + entity_lfu_cache: cache, + evict_stats, + } = entity_cache + .as_modifications(block_ptr.number, &self.metrics.host.stopwatch) + .await + .classify()?; + section.end(); + + log_evict_stats(&self.logger, &evict_stats); + + mods.extend(offchain_mods); + + // Put the cache back in the state, asserting that the placeholder cache was not used. + assert!(self.state.entity_lfu_cache.is_empty()); + self.state.entity_lfu_cache = cache; + + if !mods.is_empty() { + info!(&logger, "Applying {} entity operation(s)", mods.len()); + } + + let err_count = deterministic_errors.len(); + for (i, e) in deterministic_errors.iter().enumerate() { + let message = format!("{:#}", e).replace('\n', "\t"); + error!(&logger, "Subgraph error {}/{}", i + 1, err_count; + "error" => message, + "code" => LogCode::SubgraphSyncingFailure + ); + } + + // Transact entity operations into the store and update the + // subgraph's block stream pointer + let _section = self.metrics.host.stopwatch.start_section("transact_block"); + let start = Instant::now(); + + // If a deterministic error has happened, make the PoI to be the only entity that'll be stored. + if has_errors && self.inputs.errors_are_fatal() { + let is_poi_entity = + |entity_mod: &EntityModification| entity_mod.key().entity_type.is_poi(); + mods.retain(is_poi_entity); + // Confidence check + assert!( + mods.len() == 1, + "There should be only one PoI EntityModification" + ); + } + + let is_caught_up = self.is_caught_up(&block_ptr).await.non_deterministic()?; + + if !self.state.postponed_indexes_created + && close_to_chain_head( + &block_ptr, + &self.state.cached_head_ptr, + ENV_VARS.postpone_indexes_creation_threshold, + ) + { + self.state.postponed_indexes_created = true; + self.inputs + .store + .create_postponed_indexes() + .await + .non_deterministic()?; + } + + self.inputs + .store + .transact_block_operations( + block_ptr.clone(), + block_time, + firehose_cursor, + mods, + &self.metrics.host.stopwatch, + persisted_data_sources, + deterministic_errors, + processed_offchain_data_sources, + self.inputs.errors_are_non_fatal(), + is_caught_up, + ) + .await + .classify() + .detail("Failed to transact block operations")?; + + // For subgraphs with `nonFatalErrors` feature disabled, we consider + // any error as fatal. + // + // So we do an early return to make the subgraph stop processing blocks. + // + // In this scenario the only entity that is stored/transacted is the PoI, + // all of the others are discarded. + if has_errors && self.inputs.errors_are_fatal() { + if let Err(e) = self.inputs.store.flush().await { + error!(logger, "Failed to flush store after fatal errors"; "error" => format!("{:#}", e)); + } + // Only the first error is reported. + return Err(ProcessingError::Deterministic(Box::new( + first_error.unwrap(), + ))); + } + + let elapsed = start.elapsed().as_secs_f64(); + self.metrics + .subgraph + .block_ops_transaction_duration + .observe(elapsed); + + block_state_metrics + .flush_metrics_to_store(logger, block_ptr, self.inputs.deployment.id) + .non_deterministic()?; + + if has_errors { + self.maybe_cancel().await?; + } + + Ok(()) + } + + /// Cancel the subgraph if `disable_fail_fast` is not set and it is not + /// synced + async fn maybe_cancel(&self) -> Result<(), ProcessingError> { + // To prevent a buggy pending version from replacing a current version, if errors are + // present the subgraph will be unassigned. + let store = &self.inputs.store; + if !ENV_VARS.disable_fail_fast && !store.is_deployment_synced() { + store + .pause_subgraph() + .await + .map_err(|e| ProcessingError::Unknown(e.into()))?; + + // Use `Canceled` to avoiding setting the subgraph health to failed, an error was + // just transacted so it will be already be set to unhealthy. + Err(ProcessingError::Canceled) + } else { + Ok(()) + } + } + + async fn match_and_decode_many<'a, F>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: Vec>, + hosts_filter: F, + ) -> Result>, MappingError> + where + F: Fn(&TriggerData) -> Box + Send + 'a>, + { + let triggers = triggers.into_iter().map(|t| match t { + Trigger::Chain(t) => TriggerData::Onchain(t), + Trigger::Subgraph(t) => TriggerData::Subgraph(t), + }); + + self.ctx + .decoder + .match_and_decode_many( + logger, + block, + triggers, + hosts_filter, + &self.metrics.subgraph, + ) + .await + } + + // ========================================================================= + // Pipeline Stage Methods + // ========================================================================= + // + // The following methods implement the block processing pipeline stages. + // Each stage handles a specific phase of block processing: + // + // 1. match_triggers: Match and decode triggers against hosts + // 2. execute_triggers: Execute the matched triggers + // 3. process_dynamic_data_sources: Handle dynamically created data sources + // 4. (process_offchain_triggers): Existing handle_offchain_triggers method + // 5. (persist_block_state): Existing transact_block_state method + // + // ========================================================================= + + /// Pipeline Stage 1: Match triggers to hosts and decode them. + /// + /// Takes raw triggers from a block and matches them against all registered + /// hosts, returning runnable triggers ready for execution. + async fn match_triggers<'a>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: Vec>, + ) -> Result>, MappingError> { + let hosts_filter = |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger); + self.match_and_decode_many(logger, block, triggers, hosts_filter) + .await + } + + /// Pipeline Stage 2: Execute matched triggers. + /// + /// Takes runnable triggers and executes them using the TriggerRunner, + /// accumulating state changes in the block state. + async fn execute_triggers( + &self, + block: &Arc, + runnables: Vec>, + block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + let trigger_runner = TriggerRunner::new( + self.ctx.trigger_processor.as_ref(), + &self.logger, + &self.metrics.subgraph, + &self.inputs.debug_fork, + self.inputs.instrument, + ); + trigger_runner + .execute( + block, + runnables, + block_state, + proof_of_indexing, + causality_region, + ) + .await + } + + /// Pipeline Stage 3: Process dynamically created data sources. + /// + /// This loop processes data sources created during trigger execution: + /// 1. Instantiate the created data sources + /// 2. Reprocess triggers from this block that match the new data sources + /// 3. Repeat until no more data sources are created + /// + /// Note: This algorithm processes data sources spawned on the same block + /// _breadth first_ on the tree implied by the parent-child relationship + /// between data sources. + async fn process_dynamic_data_sources( + &mut self, + logger: &Logger, + block: &Arc, + firehose_cursor: &FirehoseCursor, + mut block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + fn log_triggers_found(logger: &Logger, triggers: &[Trigger]) { + if triggers.len() == 1 { + info!(logger, "1 trigger found in this block"); + } else if triggers.len() > 1 { + info!(logger, "{} triggers found in this block", triggers.len()); + } + } + + let _section = self + .metrics + .stream + .stopwatch + .start_section(HANDLE_CREATED_DS_SECTION_NAME); + + while block_state.has_created_data_sources() { + // Instantiate dynamic data sources, removing them from the block state. + let (data_sources, runtime_hosts) = + self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; + + let filter = &Arc::new(TriggerFilterWrapper::new( + C::TriggerFilter::from_data_sources( + data_sources.iter().filter_map(DataSource::as_onchain), + ), + vec![], + )); + + // TODO: We have to pass a reference to `block` to + // `refetch_block`, otherwise the call to + // handle_offchain_triggers below gets an error that `block` + // has moved. That is extremely fishy since it means that + // `handle_offchain_triggers` uses the non-refetched block + // + // It's also not clear why refetching needs to happen inside + // the loop; will firehose really return something diffrent + // each time even though the cursor doesn't change? + let block = self.refetch_block(logger, block, firehose_cursor).await?; + + // Reprocess the triggers from this block that match the new data sources + let block_with_triggers = self + .inputs + .triggers_adapter + .triggers_in_block(logger, block.as_ref().clone(), filter) + .await + .non_deterministic()?; + + let triggers = block_with_triggers.trigger_data; + log_triggers_found::(logger, &triggers); + + // Add entity operations for the new data sources to the block state + // and add runtimes for the data sources to the subgraph instance. + self.persist_dynamic_data_sources(&mut block_state, data_sources); + + // Process the triggers in each host in the same order the + // corresponding data sources have been created. + let hosts_filter = |_: &'_ TriggerData| -> Box + Send> { + Box::new(runtime_hosts.iter().map(Arc::as_ref)) + }; + let runnables = self + .match_and_decode_many(logger, &block, triggers, hosts_filter) + .await; + + let trigger_runner = TriggerRunner::new( + self.ctx.trigger_processor.as_ref(), + &self.logger, + &self.metrics.subgraph, + &self.inputs.debug_fork, + self.inputs.instrument, + ); + let res = match runnables { + Ok(runnables) => { + trigger_runner + .execute( + &block, + runnables, + block_state, + proof_of_indexing, + causality_region, + ) + .await + } + Err(e) => Err(e), + }; + + block_state = res.map_err(|e| { + // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. + // This can cause an unnecessary subgraph failure, to fix it we need to figure out a + // way to revert the effect of `create_dynamic_data_sources` so we may return a + // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. + match e { + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { + ProcessingError::Unknown(e) + } + } + })?; + } + + Ok(block_state) + } + + /// Pipeline Stage 4: Process offchain triggers. + /// + /// Retrieves ready offchain events and processes them, returning entity + /// modifications and processed data sources to be included in the transaction. + async fn process_offchain_triggers( + &mut self, + block: &Arc, + block_state: &mut BlockState, + ) -> Result<(Vec, Vec), ProcessingError> { + let offchain_events = self + .ctx + .offchain_monitor + .ready_offchain_events() + .non_deterministic()?; + + let vid_gen = block_state.seq_gen(); + let (offchain_mods, processed_offchain_data_sources, persisted_off_chain_data_sources) = + self.handle_offchain_triggers(offchain_events, block, vid_gen) + .await + .non_deterministic()?; + + block_state + .persisted_data_sources + .extend(persisted_off_chain_data_sources); + + Ok((offchain_mods, processed_offchain_data_sources)) + } + + /// Processes a block and returns the updated context and a boolean flag indicating + /// whether new dynamic data sources have been added to the subgraph. + /// + /// ## Pipeline Stages + /// + /// Block processing follows a pipeline of stages: + /// 1. **match_triggers**: Match and decode triggers against hosts + /// 2. **execute_triggers**: Execute the matched triggers via TriggerRunner + /// 3. **process_dynamic_data_sources**: Handle dynamically created data sources + /// 4. **process_offchain_triggers**: Process offchain events + /// 5. **persist_block**: Persist block state to the store + async fn process_block( + &mut self, + block: BlockWithTriggers, + firehose_cursor: FirehoseCursor, + ) -> Result { + let triggers = block.trigger_data; + let block = Arc::new(block.block); + let block_ptr = block.ptr(); + + let logger = self.logger.new(o!( + "block_number" => format!("{:?}", block_ptr.number), + "block_hash" => format!("{}", block_ptr.hash) + )); + + info!(logger, "Start processing block"; + "triggers" => triggers.len()); + + let proof_of_indexing = + SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); + + // Causality region for onchain triggers. + let causality_region = PoICausalityRegion::from_network(&self.inputs.network); + + let vid_gen = SeqGenerator::new(block_ptr.number); + let mut block_state = BlockState::new( + self.inputs.store.clone(), + std::mem::take(&mut self.state.entity_lfu_cache), + vid_gen, + ); + + let _section = self + .metrics + .stream + .stopwatch + .start_section(PROCESS_TRIGGERS_SECTION_NAME); + + // Stage 1: Match triggers to hosts and decode + let runnables = self.match_triggers(&logger, &block, triggers).await; + + // Stage 2: Execute triggers + let res = match runnables { + Ok(runnables) => { + self.execute_triggers( + &block, + runnables, + block_state, + &proof_of_indexing, + &causality_region, + ) + .await + } + Err(e) => Err(e), + }; + + match res { + // Triggers processed with no errors or with only deterministic errors. + Ok(state) => block_state = state, + + // Some form of unknown or non-deterministic error ocurred. + Err(MappingError::Unknown(e)) => return Err(ProcessingError::Unknown(e)), + + // Possible blockchain reorg detected - signal restart via ProcessingError::PossibleReorg. + // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. + Err(MappingError::PossibleReorg(e)) => return Err(ProcessingError::PossibleReorg(e)), + } + + // Check if there are any datasources that have expired in this block. ie: the end_block + // of that data source is equal to the block number of the current block. + let has_expired_data_sources = self.inputs.end_blocks.contains(&block_ptr.number); + + // If new onchain data sources have been created, and static filters are not in use, it is necessary + // to restart the block stream with the new filters. + let created_data_sources_needs_restart = + !self.is_static_filters_enabled() && block_state.has_created_on_chain_data_sources(); + + // Determine if the block stream needs to be restarted due to newly created on-chain data sources + // or data sources that have reached their end block. + let needs_restart = created_data_sources_needs_restart || has_expired_data_sources; + + // Checkpoint before dynamic DS processing for potential rollback scenarios. + // This captures the current state so it can be restored if dynamic data source + // processing fails in a way that requires partial rollback. + let _checkpoint = block_state.checkpoint(); + + // Stage 3: Process dynamic data sources + block_state = self + .process_dynamic_data_sources( + &logger, + &block, + &firehose_cursor, + block_state, + &proof_of_indexing, + &causality_region, + ) + .await?; + + // Stage 4: Process offchain triggers + let (offchain_mods, processed_offchain_data_sources) = self + .process_offchain_triggers(&block, &mut block_state) + .await?; + + // Stage 5: Persist block state + self.transact_block_state( + &logger, + block_ptr, + firehose_cursor, + block.timestamp(), + block_state, + proof_of_indexing, + offchain_mods, + processed_offchain_data_sources, + ) + .await?; + + match needs_restart { + true => Ok(Action::Restart), + false => Ok(Action::Continue), + } + } + + /// Refetch the block if it that is needed. Otherwise return the block as is. + async fn refetch_block( + &mut self, + logger: &Logger, + block: &Arc, + firehose_cursor: &FirehoseCursor, + ) -> Result, ProcessingError> { + if !self.inputs.chain.is_refetch_block_required() { + return Ok(block.cheap_clone()); + } + + let cur = firehose_cursor.clone(); + let log = logger.cheap_clone(); + let chain = self.inputs.chain.cheap_clone(); + let block = retry( + "refetch firehose block after dynamic datasource was added", + logger, + ) + .limit(5) + .no_timeout() + .run(move || { + let cur = cur.clone(); + let log = log.cheap_clone(); + let chain = chain.cheap_clone(); + async move { chain.refetch_firehose_block(&log, cur).await } + }) + .await + .non_deterministic()?; + Ok(Arc::new(block)) + } + + fn create_dynamic_data_sources( + &mut self, + created_data_sources: Vec, + ) -> Result<(Vec>, Vec>), ProcessingError> { + let mut data_sources = vec![]; + let mut runtime_hosts = vec![]; + + for info in created_data_sources { + let manifest_idx = info + .template + .manifest_idx() + .ok_or_else(|| anyhow!("Expected template to have an idx")) + .non_deterministic()?; + let created_ds_template = self + .inputs + .templates + .iter() + .find(|t| t.manifest_idx() == manifest_idx) + .ok_or_else(|| anyhow!("Expected to find a template for this dynamic data source")) + .non_deterministic()?; + + // Try to instantiate a data source from the template + let data_source = { + let res = match info.template { + InstanceDSTemplate::Onchain(_) => { + C::DataSource::from_template_info(info, created_ds_template) + .map(DataSource::Onchain) + .map_err(DataSourceCreationError::from) + } + InstanceDSTemplate::Offchain(_) => offchain::DataSource::from_template_info( + info, + self.ctx.causality_region_next_value(), + ) + .map(DataSource::Offchain), + }; + match res { + Ok(ds) => ds, + Err(e @ DataSourceCreationError::Ignore(..)) => { + warn!(self.logger, "{}", e.to_string()); + continue; + } + Err(DataSourceCreationError::Unknown(e)) => return Err(e).non_deterministic(), + } + }; + + // Try to create a runtime host for the data source + let host = self + .ctx + .add_dynamic_data_source(&self.logger, data_source.clone()) + .non_deterministic()?; + + match host { + Some(host) => { + data_sources.push(data_source); + runtime_hosts.push(host); + } + None => { + warn!( + self.logger, + "no runtime host created, there is already a runtime host instantiated for \ + this data source"; + "name" => &data_source.name(), + "address" => &data_source.address() + .map(hex::encode) + .unwrap_or("none".to_string()), + ) + } + } + } + + Ok((data_sources, runtime_hosts)) + } + + async fn handle_action( + &mut self, + start: Instant, + block_ptr: BlockPtr, + action: Result, + ) -> Result { + self.state.skip_ptr_updates_timer = Instant::now(); + + let elapsed = start.elapsed().as_secs_f64(); + self.metrics + .subgraph + .block_processing_duration + .observe(elapsed); + + match action { + Ok(action) => { + // Keep trying to unfail subgraph for everytime it advances block(s) until it's + // health is not Failed anymore. + if self.state.should_try_unfail_non_deterministic { + // If the deployment head advanced, we can unfail + // the non-deterministic error (if there's any). + let outcome = self + .inputs + .store + .unfail_non_deterministic_error(&block_ptr) + .await?; + + // Stop trying unless we're still behind the error block. + if outcome != UnfailOutcome::BehindErrorBlock { + self.state.should_try_unfail_non_deterministic = false; + + if outcome == UnfailOutcome::Unfailed { + self.metrics.subgraph.deployment_status.running(); + self.state.backoff.reset(); + } + } + } + + if let Some(stop_block) = self.inputs.stop_block + && block_ptr.number >= stop_block + { + info!(self.logger, "Stop block reached for subgraph"); + return Ok(Action::Stop); + } + + if let Some(max_end_block) = self.inputs.max_end_block + && block_ptr.number >= max_end_block + { + info!( + self.logger, + "Stopping subgraph as maximum endBlock reached"; + "max_end_block" => max_end_block, + "current_block" => block_ptr.number + ); + return Ok(Action::Stop); + } + + Ok(action) + } + // Handle errors based on their kind using the unified error classification. + // + // Error handling invariants: + // - Deterministic: Stop processing, persist PoI only, fail subgraph + // - NonDeterministic: Retry with backoff, may succeed on retry + // - PossibleReorg: Restart cleanly without persisting (don't fail subgraph) + // - Canceled: Clean shutdown, no error recording + Err(e) => match e.kind() { + ProcessingErrorKind::Canceled => { + debug!(self.logger, "Subgraph block stream shut down cleanly"); + Ok(Action::Stop) + } + + ProcessingErrorKind::PossibleReorg => { + // Possible reorg detected - restart the block stream cleanly. + // Don't persist anything and don't mark subgraph as failed. + // The block stream restart will allow detection of the actual reorg. + info!(self.logger, + "Possible reorg detected, restarting block stream"; + "error" => format!("{:#}", e), + ); + + // Revert in-memory state to last good block but don't touch the store + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + Ok(Action::Restart) + } + + ProcessingErrorKind::Deterministic => { + // Deterministic error - fail the subgraph permanently. + self.metrics.subgraph.deployment_status.failed(); + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + let message = format!("{:#}", e).replace('\n', "\t"); + let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); + + self.fail_subgraph(message, Some(block_ptr), true).await?; + + Err(err) + } + + ProcessingErrorKind::NonDeterministic => { + // Non-deterministic error - retry with backoff. + self.metrics.subgraph.deployment_status.failed(); + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + let message = format!("{:#}", e).replace('\n', "\t"); + + error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; + "attempt" => self.state.backoff.attempt, + "retry_delay_s" => self.state.backoff.delay().as_secs()); + + // Shouldn't fail subgraph if it's already failed for non-deterministic + // reasons. + // + // If we don't do this check we would keep adding the same error to the + // database. + if !self.inputs.store.health().await?.is_failed() { + self.fail_subgraph(message, Some(block_ptr), false).await?; + } + + // Sleep before restarting. + self.state.backoff.sleep_async().await; + + self.state.should_try_unfail_non_deterministic = true; + + // And restart the subgraph. + Ok(Action::Restart) + } + }, + } + } + + fn persist_dynamic_data_sources( + &mut self, + block_state: &mut BlockState, + data_sources: Vec>, + ) { + if !data_sources.is_empty() { + debug!( + self.logger, + "Creating {} dynamic data source(s)", + data_sources.len() + ); + } + + // Add entity operations to the block state in order to persist + // the dynamic data sources + for data_source in data_sources.iter() { + debug!( + self.logger, + "Persisting data_source"; + "name" => &data_source.name(), + "address" => &data_source.address().map(hex::encode).unwrap_or("none".to_string()), + ); + block_state.persist_data_source(data_source.as_stored_dynamic_data_source()); + } + } + + /// We consider a subgraph caught up when it's at most 10 blocks behind the chain head. + async fn is_caught_up(&mut self, block_ptr: &BlockPtr) -> Result { + const CAUGHT_UP_DISTANCE: BlockNumber = 10; + + // Ensure that `state.cached_head_ptr` has a value since it could be `None` on the first + // iteration of loop. If the deployment head has caught up to the `cached_head_ptr`, update + // it so that we are up to date when checking if synced. + let cached_head_ptr = self.state.cached_head_ptr.cheap_clone(); + if cached_head_ptr.is_none() + || close_to_chain_head(block_ptr, &cached_head_ptr, CAUGHT_UP_DISTANCE) + { + self.state.cached_head_ptr = self.inputs.chain.chain_head_ptr().await?; + } + let is_caught_up = + close_to_chain_head(block_ptr, &self.state.cached_head_ptr, CAUGHT_UP_DISTANCE); + if is_caught_up { + // Stop recording time-to-sync metrics. + self.metrics.stream.stopwatch.disable(); + } + Ok(is_caught_up) + } +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn handle_offchain_triggers( + &mut self, + triggers: Vec, + block: &Arc, + vid_gen: SeqGenerator, + ) -> Result< + ( + Vec, + Vec, + Vec, + ), + Error, + > { + let mut mods = vec![]; + let mut processed_data_sources = vec![]; + let mut persisted_data_sources = vec![]; + + for trigger in triggers { + // Using an `EmptyStore` and clearing the cache for each trigger is a makeshift way to + // get causality region isolation. + let schema = ReadStore::input_schema(&self.inputs.store); + let mut block_state = BlockState::new( + EmptyStore::new(schema), + LfuCache::new(), + vid_gen.cheap_clone(), + ); + + // PoI ignores offchain events. + // See also: poi-ignores-offchain + let proof_of_indexing = SharedProofOfIndexing::ignored(); + let causality_region = ""; + + let trigger = TriggerData::Offchain(trigger); + let process_res = { + let hosts = self.ctx.instance.hosts_for_trigger(&trigger); + let triggers_res = self.ctx.decoder.match_and_decode( + &self.logger, + block, + trigger, + hosts, + &self.metrics.subgraph, + ); + match triggers_res { + Ok(runnable) => { + self.ctx + .trigger_processor + .process_trigger( + &self.logger, + runnable.hosted_triggers, + block, + block_state, + &proof_of_indexing, + causality_region, + &self.inputs.debug_fork, + &self.metrics.subgraph, + self.inputs.instrument, + ) + .await + } + Err(e) => Err(e), + } + }; + match process_res { + Ok(state) => block_state = state, + Err(err) => { + let err = match err { + // Ignoring `PossibleReorg` isn't so bad since the subgraph will retry + // non-deterministic errors. + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => e, + }; + return Err(err.context("failed to process trigger".to_string())); + } + } + + anyhow::ensure!( + !block_state.has_created_on_chain_data_sources(), + "Attempted to create on-chain data source in offchain data source handler. This is not yet supported.", + ); + + let (data_sources, _) = + self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; + + // Add entity operations for the new data sources to the block state + // and add runtimes for the data sources to the subgraph instance. + self.persist_dynamic_data_sources(&mut block_state, data_sources); + + // This propagates any deterministic error as a non-deterministic one. Which might make + // sense considering offchain data sources are non-deterministic. + if let Some(err) = block_state.deterministic_errors.into_iter().next() { + return Err(anyhow!("{}", err)); + } + + mods.extend( + block_state + .entity_cache + .as_modifications(block.number(), &self.metrics.subgraph.stopwatch) + .await? + .modifications, + ); + processed_data_sources.extend(block_state.processed_data_sources); + persisted_data_sources.extend(block_state.persisted_data_sources) + } + + Ok((mods, processed_data_sources, persisted_data_sources)) + } + + fn update_deployment_synced_metric(&self) { + self.metrics + .subgraph + .deployment_synced + .record(self.inputs.store.is_deployment_synced()); + } +} + +#[derive(Debug)] +enum Action { + Continue, + Stop, + Restart, +} + +impl Action { + /// Return `true` if the action indicates that we are done with a block + fn block_finished(&self) -> bool { + match self { + Action::Restart => false, + Action::Continue | Action::Stop => true, + } + } +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn handle_revert(&mut self, revert_to_ptr: BlockPtr, cursor: FirehoseCursor) -> Action { + // Current deployment head in the database / WritableAgent Mutex cache. + // + // Safe unwrap because in a Revert event we're sure the subgraph has + // advanced at least once. + let subgraph_ptr = self.inputs.store.block_ptr().unwrap(); + if revert_to_ptr.number >= subgraph_ptr.number { + info!(&self.logger, "Block to revert is higher than subgraph pointer, nothing to do"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); + return Action::Continue; + } + + info!(&self.logger, "Reverting block to get back to main chain"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); + + if let Err(e) = self + .inputs + .store + .revert_block_operations(revert_to_ptr.clone(), cursor) + .await + { + error!(&self.logger, "Could not revert block. Retrying"; "error" => %e); + + // Exit inner block stream consumption loop and go up to loop that restarts subgraph + return Action::Restart; + } + + self.metrics + .stream + .reverted_blocks + .set(subgraph_ptr.number as f64); + self.metrics + .stream + .deployment_head + .set(subgraph_ptr.number as f64); + + self.revert_state_to(revert_to_ptr.number); + + let needs_restart: bool = self.needs_restart(revert_to_ptr, subgraph_ptr); + + if needs_restart { + Action::Restart + } else { + Action::Continue + } + } + + /// Determines if the subgraph needs to be restarted. + /// Currently returns true when there are data sources that have reached their end block + /// in the range between `revert_to_ptr` and `subgraph_ptr`. + fn needs_restart(&self, revert_to_ptr: BlockPtr, subgraph_ptr: BlockPtr) -> bool { + self.inputs + .end_blocks + .range(revert_to_ptr.number..=subgraph_ptr.number) + .next() + .is_some() + } +} + +impl From for SubgraphRunnerError { + fn from(err: StoreError) -> Self { + Self::Unknown(err.into()) + } +} + +/// Transform the proof of indexing changes into entity updates that will be +/// inserted when as_modifications is called. +async fn update_proof_of_indexing( + proof_of_indexing: ProofOfIndexing, + block_time: BlockTime, + stopwatch: &StopwatchMetrics, + entity_cache: &mut EntityCache, +) -> Result<(), Error> { + // Helper to store the digest as a PoI entity in the cache + async fn store_poi_entity( + entity_cache: &mut EntityCache, + key: EntityKey, + digest: Bytes, + block_time: BlockTime, + ) -> Result<(), Error> { + let digest_name = entity_cache.schema.poi_digest(); + let mut data = vec![ + ( + graph::data::store::ID.clone(), + Value::from(key.entity_id.to_string()), + ), + (digest_name, Value::from(digest)), + ]; + if entity_cache.schema.has_aggregations() { + let block_time = Value::Int8(block_time.as_secs_since_epoch()); + data.push((entity_cache.schema.poi_block_time(), block_time)); + } + let poi = entity_cache.make_entity(data)?; + entity_cache.set(key, poi, None).await + } + + let _section_guard = stopwatch.start_section("update_proof_of_indexing"); + + let mut proof_of_indexing = proof_of_indexing.take(); + + for (causality_region, stream) in proof_of_indexing.drain() { + // Create the special POI entity key specific to this causality_region + // There are two things called causality regions here, one is the causality region for + // the poi which is a string and the PoI entity id. The other is the data source + // causality region to which the PoI belongs as an entity. Currently offchain events do + // not affect PoI so it is assumed to be `ONCHAIN`. + // See also: poi-ignores-offchain + let entity_key = entity_cache + .schema + .poi_type() + .key_in(causality_region, CausalityRegion::ONCHAIN); + + // Grab the current digest attribute on this entity + let poi_digest = entity_cache.schema.poi_digest().clone(); + let prev_poi = entity_cache + .get(&entity_key, GetScope::Store) + .await + .map_err(Error::from)? + .map(|entity| match entity.get(poi_digest.as_str()) { + Some(Value::Bytes(b)) => b.clone(), + _ => panic!("Expected POI entity to have a digest and for it to be bytes"), + }); + + // Finish the POI stream, getting the new POI value. + let updated_proof_of_indexing = stream.pause(prev_poi.as_deref()); + let updated_proof_of_indexing: Bytes = (&updated_proof_of_indexing[..]).into(); + + // Put this onto an entity with the same digest attribute + // that was expected before when reading. + store_poi_entity( + entity_cache, + entity_key, + updated_proof_of_indexing, + block_time, + ) + .await?; + } + + Ok(()) +} + +/// Checks if the Deployment BlockPtr is within N blocks of the chain head or ahead. +fn close_to_chain_head( + deployment_head_ptr: &BlockPtr, + chain_head_ptr: &Option, + n: BlockNumber, +) -> bool { + matches!((deployment_head_ptr, &chain_head_ptr), (b1, Some(b2)) if b1.number >= (b2.number - n)) +} + +#[test] +fn test_close_to_chain_head() { + let offset = 1; + + let block_0 = BlockPtr::try_from(( + "bd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f", + 0, + )) + .unwrap(); + let block_1 = BlockPtr::try_from(( + "8511fa04b64657581e3f00e14543c1d522d5d7e771b54aa3060b662ade47da13", + 1, + )) + .unwrap(); + let block_2 = BlockPtr::try_from(( + "b98fb783b49de5652097a989414c767824dff7e7fd765a63b493772511db81c1", + 2, + )) + .unwrap(); + + assert!(!close_to_chain_head(&block_0, &None, offset)); + assert!(!close_to_chain_head(&block_2, &None, offset)); + + assert!(!close_to_chain_head( + &block_0, + &Some(block_2.clone()), + offset + )); + + assert!(close_to_chain_head( + &block_1, + &Some(block_2.clone()), + offset + )); + assert!(close_to_chain_head( + &block_2, + &Some(block_2.clone()), + offset + )); +} diff --git a/core/src/subgraph/runner/state.rs b/core/src/subgraph/runner/state.rs new file mode 100644 index 00000000000..d6401842522 --- /dev/null +++ b/core/src/subgraph/runner/state.rs @@ -0,0 +1,96 @@ +//! State machine types for SubgraphRunner. +//! +//! This module defines the explicit state machine that controls the runner's lifecycle, +//! replacing the previous nested loop structure with clear state transitions. + +use graph::blockchain::Blockchain; +use graph::blockchain::block_stream::{BlockStream, BlockWithTriggers, FirehoseCursor}; +use graph::ext::futures::Cancelable; +use graph::prelude::BlockPtr; + +/// The current state of the SubgraphRunner's lifecycle. +/// +/// The runner transitions through these states as it processes blocks, +/// handles reverts, and responds to errors or cancellation signals. +/// +/// ## State Transitions +/// +/// ```text +/// Initializing ───────────────────────────────────┐ +/// │ │ +/// v │ +/// AwaitingBlock ◄────────────────────────────────┤ +/// │ │ +/// ├── ProcessBlock event ──► ProcessingBlock │ +/// │ │ │ +/// │ ├── success ┼──► AwaitingBlock +/// │ │ │ +/// │ └── restart ┼──► Restarting +/// │ │ +/// ├── Revert event ──────────► Reverting ────┤ +/// │ │ +/// ├── Error ─────────────────► Restarting ───┤ +/// │ │ +/// └── Cancel/MaxBlock ───────► Stopped │ +/// │ +/// Restarting ─────────────────────────────────────┘ +/// ``` +#[derive(Default)] +pub enum RunnerState { + /// Initial state, ready to start block stream. + #[default] + Initializing, + + /// Block stream active, waiting for next event. + AwaitingBlock { + block_stream: Cancelable>>, + }, + + /// Processing a block through the pipeline. + /// The block stream is kept alive to continue processing after this block. + ProcessingBlock { + block_stream: Cancelable>>, + block: BlockWithTriggers, + cursor: FirehoseCursor, + }, + + /// Handling a revert event. + /// The block stream is kept alive to continue processing after the revert. + Reverting { + block_stream: Cancelable>>, + to_ptr: BlockPtr, + cursor: FirehoseCursor, + }, + + /// Restarting block stream (new filters, store restart, etc.). + Restarting { reason: RestartReason }, + + /// Terminal state. + Stopped { reason: StopReason }, +} + +/// Reasons for restarting the block stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RestartReason { + /// New dynamic data source was created that requires filter updates. + DynamicDataSourceCreated, + /// Store error occurred and store needs to be restarted. + StoreError, + /// Possible reorg detected, need to restart to detect it. + /// NOTE: Currently unused but reserved for future error handling consolidation (Phase 5). + #[allow(dead_code)] + PossibleReorg, +} + +/// Reasons for stopping the runner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StopReason { + /// The maximum end block was reached. + MaxEndBlockReached, + /// The runner was canceled (unassigned or shutdown). + Canceled, + /// The subgraph was unassigned while this runner was active. + Unassigned, + /// The block stream ended (typically in tests). + StreamEnded, +} diff --git a/core/src/subgraph/runner/trigger_runner.rs b/core/src/subgraph/runner/trigger_runner.rs new file mode 100644 index 00000000000..6c89052e5dc --- /dev/null +++ b/core/src/subgraph/runner/trigger_runner.rs @@ -0,0 +1,79 @@ +use std::sync::Arc; + +use graph::blockchain::Blockchain; +use graph::components::store::SubgraphFork; +use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; +use graph::components::trigger_processor::RunnableTriggers; +use graph::prelude::{BlockState, RuntimeHostBuilder, SubgraphInstanceMetrics, TriggerProcessor}; +use graph::slog::Logger; + +/// Handles the execution of triggers against runtime hosts, accumulating state. +/// +/// This component unifies the trigger processing loop that was previously duplicated +/// for initial triggers and dynamically created data source triggers. +pub struct TriggerRunner<'a, C: Blockchain, T: RuntimeHostBuilder> { + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a Arc, + debug_fork: &'a Option>, + instrument: bool, +} + +impl<'a, C, T> TriggerRunner<'a, C, T> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// Create a new TriggerRunner with the given dependencies. + pub fn new( + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a Arc, + debug_fork: &'a Option>, + instrument: bool, + ) -> Self { + Self { + processor, + logger, + metrics, + debug_fork, + instrument, + } + } + + /// Execute a sequence of runnable triggers, accumulating state changes. + /// + /// Processes each trigger in order. If any trigger fails with a non-deterministic + /// error, processing stops and the error is returned. Deterministic errors are + /// accumulated in the block state. + pub async fn execute( + &self, + block: &Arc, + runnables: Vec>, + block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + let mut state = block_state; + + for runnable in runnables { + state = self + .processor + .process_trigger( + self.logger, + runnable.hosted_triggers, + block, + state, + proof_of_indexing, + causality_region, + self.debug_fork, + self.metrics, + self.instrument, + ) + .await + .map_err(|e| e.add_trigger_context(&runnable.trigger))?; + } + + Ok(state) + } +} diff --git a/core/src/subgraph/state.rs b/core/src/subgraph/state.rs index 727cd8d45e6..c1da4b43199 100644 --- a/core/src/subgraph/state.rs +++ b/core/src/subgraph/state.rs @@ -1,15 +1,11 @@ use graph::{ - prelude::{BlockPtr, Entity}, - schema::EntityKey, - util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}, + components::store::EntityLfuCache, prelude::BlockPtr, util::backoff::ExponentialBackoff, }; use std::time::Instant; pub struct IndexingState { /// `true` -> `false` on the first run pub should_try_unfail_non_deterministic: bool, - /// `false` -> `true` once it reaches chain head - pub synced: bool, /// Backoff used for the retry mechanism on non-deterministic errors pub backoff: ExponentialBackoff, /// Related to field above `backoff` @@ -18,6 +14,9 @@ pub struct IndexingState { /// - The time THRESHOLD is passed /// - Or the subgraph has triggers for the block pub skip_ptr_updates_timer: Instant, - pub entity_lfu_cache: LfuCache>, + pub entity_lfu_cache: EntityLfuCache, pub cached_head_ptr: Option, + /// Set to `true` once postponed indexes have been created. This + /// ensures we only trigger index creation once per subgraph run. + pub postponed_indexes_created: bool, } diff --git a/core/src/subgraph/stream.rs b/core/src/subgraph/stream.rs index c1d767e3fcf..5547543f13d 100644 --- a/core/src/subgraph/stream.rs +++ b/core/src/subgraph/stream.rs @@ -1,13 +1,13 @@ use crate::subgraph::inputs::IndexingInputs; use anyhow::bail; use graph::blockchain::block_stream::{BlockStream, BufferedBlockStream}; -use graph::blockchain::Blockchain; +use graph::blockchain::{Blockchain, TriggerFilterWrapper}; use graph::prelude::{CheapClone, Error, SubgraphInstanceMetrics}; use std::sync::Arc; pub async fn new_block_stream( inputs: &IndexingInputs, - filter: &C::TriggerFilter, + filter: TriggerFilterWrapper, metrics: &SubgraphInstanceMetrics, ) -> Result>, Error> { let is_firehose = inputs.chain.chain_client().is_firehose(); @@ -18,6 +18,7 @@ pub async fn new_block_stream( inputs.deployment.clone(), inputs.store.cheap_clone(), inputs.start_blocks.clone(), + inputs.source_subgraph_stores.clone(), Arc::new(filter.clone()), inputs.unified_api_version.clone(), ) diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs index 61fa585e416..c3123e87268 100644 --- a/core/src/subgraph/trigger_processor.rs +++ b/core/src/subgraph/trigger_processor.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; -use graph::blockchain::{Block, Blockchain}; +use graph::blockchain::{Block, Blockchain, DecoderHook as _}; use graph::cheap_clone::CheapClone; use graph::components::store::SubgraphFork; use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; -use graph::data_source::{MappingTrigger, TriggerData, TriggerWithHandler}; +use graph::components::trigger_processor::{HostedTrigger, RunnableTriggers}; +use graph::data_source::TriggerData; use graph::prelude::tokio::time::Instant; use graph::prelude::{ BlockState, RuntimeHost, RuntimeHostBuilder, SubgraphInstanceMetrics, TriggerProcessor, }; use graph::slog::Logger; +use std::marker::PhantomData; use std::sync::Arc; pub struct SubgraphTriggerProcessor {} @@ -22,47 +24,28 @@ where async fn process_trigger<'a>( &'a self, logger: &Logger, - hosts: Box + Send + 'a>, + triggers: Vec>, block: &Arc, - trigger: &TriggerData, - mut state: BlockState, + mut state: BlockState, proof_of_indexing: &SharedProofOfIndexing, causality_region: &str, debug_fork: &Option>, subgraph_metrics: &Arc, instrument: bool, - ) -> Result, MappingError> { + ) -> Result { let error_count = state.deterministic_errors.len(); - let mut host_mapping: Vec<(&T::Host, TriggerWithHandler>)> = vec![]; - - { - let _section = subgraph_metrics.stopwatch.start_section("match_and_decode"); - - for host in hosts { - let mapping_trigger = match host.match_and_decode(trigger, block, logger)? { - // Trigger matches and was decoded as a mapping trigger. - Some(mapping_trigger) => mapping_trigger, - - // Trigger does not match, do not process it. - None => continue, - }; - - host_mapping.push((host, mapping_trigger)); - } - } - - if host_mapping.is_empty() { + if triggers.is_empty() { return Ok(state); } - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } + proof_of_indexing.start_handler(causality_region); - for (host, mapping_trigger) in host_mapping { + for HostedTrigger { + host, + mapping_trigger, + } in triggers + { let start = Instant::now(); state = host .process_mapping_trigger( @@ -86,18 +69,113 @@ where } } - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing.write_deterministic_error(logger, causality_region); } Ok(state) } } + +/// A helper for taking triggers as `TriggerData` (usually from the block +/// stream) and turning them into `HostedTrigger`s that are ready to run. +/// +/// The output triggers will be run in the order in which they are returned. +pub struct Decoder +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + hook: C::DecoderHook, + _builder: PhantomData, +} + +impl Decoder +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + pub fn new(hook: C::DecoderHook) -> Self { + Decoder { + hook, + _builder: PhantomData, + } + } +} + +impl> Decoder { + fn match_and_decode_inner<'a>( + &'a self, + logger: &Logger, + block: &Arc, + trigger: &TriggerData, + hosts: Box + Send + 'a>, + subgraph_metrics: &Arc, + ) -> Result>, MappingError> { + let mut host_mapping = vec![]; + + { + let _section = subgraph_metrics.stopwatch.start_section("match_and_decode"); + + for host in hosts { + let mapping_trigger = match host.match_and_decode(trigger, block, logger)? { + // Trigger matches and was decoded as a mapping trigger. + Some(mapping_trigger) => mapping_trigger, + + // Trigger does not match, do not process it. + None => continue, + }; + + host_mapping.push(HostedTrigger { + host, + mapping_trigger, + }); + } + } + Ok(host_mapping) + } + + pub(crate) fn match_and_decode<'a>( + &'a self, + logger: &Logger, + block: &Arc, + trigger: TriggerData, + hosts: Box + Send + 'a>, + subgraph_metrics: &Arc, + ) -> Result, MappingError> { + self.match_and_decode_inner(logger, block, &trigger, hosts, subgraph_metrics) + .map_err(|e| e.add_trigger_context(&trigger)) + .map(|hosted_triggers| RunnableTriggers { + trigger, + hosted_triggers, + }) + } + + pub(crate) async fn match_and_decode_many<'a, F>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: impl Iterator>, + hosts_filter: F, + metrics: &Arc, + ) -> Result>, MappingError> + where + F: Fn(&TriggerData) -> Box + Send + 'a>, + { + let mut runnables = vec![]; + for trigger in triggers { + let hosts = hosts_filter(&trigger); + match self.match_and_decode(logger, block, trigger, hosts, metrics) { + Ok(runnable_triggers) => runnables.push(runnable_triggers), + Err(e) => return Err(e), + } + } + self.hook + .after_decode(logger, &block.ptr(), runnables, metrics) + .await + } +} diff --git a/core/src/subgraph_manifest.rs b/core/src/subgraph_manifest.rs new file mode 100644 index 00000000000..ddc01e0bc69 --- /dev/null +++ b/core/src/subgraph_manifest.rs @@ -0,0 +1,93 @@ +use graph::{ + cheap_clone::CheapClone as _, + components::{ + link_resolver::{LinkResolver, LinkResolverContext}, + store::{StoreError, SubgraphStore}, + }, + data::subgraph::DeploymentHash, +}; +use slog::{Logger, debug}; + +pub(super) async fn load_raw_subgraph_manifest( + logger: &Logger, + subgraph_store: &dyn SubgraphStore, + link_resolver: &dyn LinkResolver, + hash: &DeploymentHash, +) -> Result { + if let Some(raw_manifest) = + subgraph_store + .raw_manifest(hash) + .await + .map_err(|e| Error::LoadManifest { + hash: hash.cheap_clone(), + source: anyhow::Error::from(e), + })? + { + debug!(logger, "Loaded raw manifest from the subgraph store"); + return Ok(raw_manifest); + } + + debug!(logger, "Loading raw manifest using link resolver"); + + let link_resolver = + link_resolver + .for_manifest(&hash.to_string()) + .map_err(|e| Error::CreateLinkResolver { + hash: hash.cheap_clone(), + source: e, + })?; + + let file_bytes = link_resolver + .cat( + &LinkResolverContext::new(hash, logger), + &hash.to_ipfs_link(), + ) + .await + .map_err(|e| Error::LoadManifest { + hash: hash.cheap_clone(), + source: e, + })?; + + let raw_manifest: serde_yaml::Mapping = + serde_yaml::from_slice(&file_bytes).map_err(|e| Error::ParseManifest { + hash: hash.cheap_clone(), + source: e, + })?; + + subgraph_store + .set_raw_manifest_once(hash, &raw_manifest) + .await + .map_err(|e| Error::StoreManifest { + hash: hash.cheap_clone(), + source: e, + })?; + + Ok(raw_manifest) +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum Error { + #[error("failed to create link resolver for '{hash}': {source:#}")] + CreateLinkResolver { + hash: DeploymentHash, + source: anyhow::Error, + }, + + #[error("failed to load manifest for '{hash}': {source:#}")] + LoadManifest { + hash: DeploymentHash, + source: anyhow::Error, + }, + + #[error("failed to parse manifest for '{hash}': {source:#}")] + ParseManifest { + hash: DeploymentHash, + source: serde_yaml::Error, + }, + + #[error("failed to store manifest for '{hash}': {source:#}")] + StoreManifest { + hash: DeploymentHash, + source: StoreError, + }, +} diff --git a/core/src/subgraph_provider.rs b/core/src/subgraph_provider.rs new file mode 100644 index 00000000000..7e479dd9495 --- /dev/null +++ b/core/src/subgraph_provider.rs @@ -0,0 +1,346 @@ +use std::{collections::HashMap, sync::Arc, time::Instant}; + +use graph::{ + amp, + cheap_clone::CheapClone as _, + components::{ + link_resolver::LinkResolver, + metrics::subgraph::SubgraphCountMetric, + store::{DeploymentLocator, SubgraphStore}, + subgraph::SubgraphInstanceManager, + }, + log::factory::LoggerFactory, +}; +use itertools::Itertools as _; +use parking_lot::RwLock; +use slog::{debug, error}; +use tokio_util::sync::CancellationToken; + +use super::subgraph_manifest; + +/// Starts and stops subgraph deployments. +/// +/// For each subgraph deployment, checks the subgraph processing kind +/// and finds the appropriate subgraph instance manager to handle the +/// processing of the subgraph deployment. +/// +/// This is required to support both trigger-based subgraphs and Amp-powered subgraphs, +/// which have separate runners. +pub struct SubgraphProvider { + logger_factory: LoggerFactory, + count_metrics: Arc, + subgraph_store: Arc, + link_resolver: Arc, + + /// Stops active subgraph start request tasks. + /// + /// When a subgraph deployment start request is processed, a background task is created + /// to load the subgraph manifest and determine the subgraph processing kind. The processing + /// kind is then used to find the appropriate subgraph instance manager. This token stops + /// all tasks that are still loading manifests or waiting for subgraphs to start. + cancel_token: CancellationToken, + + /// Contains the enabled subgraph instance managers. + /// + /// Only subgraphs for which there is an appropriate instance manager will be started. + instance_managers: SubgraphInstanceManagers, + + /// Maintains a list of started subgraphs with their processing kinds. + /// + /// Used to forward subgraph deployment stop requests to the appropriate subgraph instance manager. + assignments: SubgraphAssignments, +} + +impl SubgraphProvider { + /// Creates a new subgraph provider. + /// + /// # Arguments + /// - `logger_factory`: Creates loggers for each subgraph deployment start/stop request + /// - `count_metrics`: Tracks the number of started subgraph deployments + /// - `subgraph_store`: Loads subgraph manifests to determine the subgraph processing kinds + /// - `link_resolver`: Loads subgraph manifests to determine the subgraph processing kinds + /// - `cancel_token`: Stops active subgraph start request tasks + /// - `instance_managers`: Contains the enabled subgraph instance managers + pub fn new( + logger_factory: &LoggerFactory, + count_metrics: Arc, + subgraph_store: Arc, + link_resolver: Arc, + cancel_token: CancellationToken, + instance_managers: SubgraphInstanceManagers, + ) -> Self { + let logger = logger_factory.component_logger("SubgraphProvider", None); + let logger_factory = logger_factory.with_parent(logger.cheap_clone()); + + debug!(logger, "Creating subgraph provider"; + "enabled_subgraph_processing_kinds" => instance_managers.0.keys().join(", ") + ); + + Self { + logger_factory, + count_metrics, + subgraph_store, + link_resolver, + cancel_token, + instance_managers, + assignments: SubgraphAssignments::new(), + } + } + + /// Starts a subgraph deployment with the appropriate subgraph instance manager. + /// + /// Loads the subgraph manifest for the specified deployment locator, determines + /// the subgraph processing kind, finds the required instance manager, and forwards + /// the start request to that instance manager. Keeps the subgraph processing kind + /// in memory for handling the stop requests. + async fn assign_and_start_subgraph( + &self, + loc: DeploymentLocator, + stop_block: Option, + ) -> Result<(), Error> { + let logger = self.logger_factory.subgraph_logger(&loc); + + let raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*self.subgraph_store, + &*self.link_resolver, + &loc.hash, + ) + .await + .map_err(|e| Error::LoadManifest { + loc: loc.cheap_clone(), + source: e, + })?; + + let subgraph_kind = SubgraphProcessingKind::from_manifest(&raw_manifest); + self.assignments.set_subgraph_kind(&loc, subgraph_kind); + + let Some(instance_manager) = self.instance_managers.get(&subgraph_kind) else { + return Err(Error::GetManager { loc, subgraph_kind }); + }; + + instance_manager.start_subgraph(loc, stop_block).await; + Ok(()) + } +} + +#[async_trait::async_trait] +impl SubgraphInstanceManager for SubgraphProvider { + async fn start_subgraph(self: Arc, loc: DeploymentLocator, stop_block: Option) { + let logger = self + .logger_factory + .subgraph_logger(&loc) + .new(slog::o!("method" => "start_subgraph")); + + if self.assignments.is_assigned(&loc) { + debug!(logger, "Subgraph is already started"); + return; + } + + self.count_metrics.deployment_count.inc(); + + let handle = tokio::spawn({ + let provider = self.cheap_clone(); + let loc = loc.cheap_clone(); + let start_instant = Instant::now(); + + async move { + debug!(logger, "Starting subgraph"); + + let fut = provider.assign_and_start_subgraph(loc, stop_block); + match provider.cancel_token.run_until_cancelled(fut).await { + Some(Ok(())) => { + debug!(logger, "Subgraph started"; + "duration_ms" => start_instant.elapsed().as_millis() + ); + } + Some(Err(e)) => { + error!(logger, "Subgraph failed to start"; + "e" => ?e + ); + } + None => { + debug!(logger, "Subgraph start cancelled"); + } + } + } + }); + + self.assignments.add( + loc, + SubgraphAssignment { + handle, + subgraph_kind: None, + }, + ) + } + + async fn stop_subgraph(&self, loc: DeploymentLocator) { + let logger = self + .logger_factory + .subgraph_logger(&loc) + .new(slog::o!("method" => "stop_subgraph")); + + debug!(logger, "Stopping subgraph"); + + let Some(SubgraphAssignment { + handle, + subgraph_kind, + }) = self.assignments.take(&loc) + else { + debug!(logger, "Subgraph is not started"); + return; + }; + + handle.abort(); + self.count_metrics.deployment_count.dec(); + + let Some(subgraph_kind) = subgraph_kind else { + debug!(logger, "Unknown subgraph kind"); + return; + }; + + let Some(instance_manager) = self.instance_managers.get(&subgraph_kind) else { + debug!(logger, "Missing instance manager"); + return; + }; + + instance_manager.stop_subgraph(loc).await; + debug!(logger, "Subgraph stopped"); + } +} + +/// Enumerates all possible errors of the subgraph provider. +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("failed to load manifest for '{loc}': {source:#}")] + LoadManifest { + loc: DeploymentLocator, + source: subgraph_manifest::Error, + }, + + #[error("failed to get instance manager for '{loc}' with kind '{subgraph_kind}'")] + GetManager { + loc: DeploymentLocator, + subgraph_kind: SubgraphProcessingKind, + }, +} + +/// Contains a mapping of enabled subgraph instance managers by subgraph processing kinds. +/// +/// Before starting a subgraph, its processing kind is determined from the subgraph manifest. +/// Then, the appropriate instance manager is loaded from this mapping. +#[derive(Default)] +pub struct SubgraphInstanceManagers( + HashMap>, +); + +impl SubgraphInstanceManagers { + /// Creates a new empty subgraph instance manager mapping. + pub fn new() -> Self { + Self(HashMap::new()) + } + + /// Adds a new subgraph instance manager for all subgraphs of the specified processing kind. + pub fn add( + &mut self, + subgraph_kind: SubgraphProcessingKind, + instance_manager: Arc, + ) { + self.0.insert(subgraph_kind, instance_manager); + } + + /// Returns the subgraph instance manager for the specified processing kind. + pub fn get( + &self, + subgraph_kind: &SubgraphProcessingKind, + ) -> Option> { + self.0 + .get(subgraph_kind) + .map(|instance_manager| instance_manager.cheap_clone()) + } +} + +/// Enumerates the supported subgraph processing kinds. +/// +/// Subgraphs may have different processing requirements, and this enum helps to map them +/// to the appropriate instance managers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::Display)] +#[strum(serialize_all = "snake_case")] +pub enum SubgraphProcessingKind { + /// Represents trigger-based subgraphs. + Trigger, + + /// Represents Amp-powered subgraphs. + Amp, +} + +impl SubgraphProcessingKind { + /// Determines the subgraph processing kind from the subgraph manifest. + fn from_manifest(raw_manifest: &serde_yaml::Mapping) -> Self { + use serde_yaml::Value; + + let is_amp_manifest = raw_manifest + .get("dataSources") + .and_then(Value::as_sequence) + .and_then(|seq| { + seq.iter() + .filter_map(Value::as_mapping) + .filter_map(|map| map.get("kind")) + .filter_map(Value::as_str) + .find(|kind| *kind == amp::manifest::DataSource::KIND) + }) + .is_some(); + + if is_amp_manifest { + return Self::Amp; + } + + Self::Trigger + } +} + +/// Maintains a list of started subgraph deployments with details required for stopping them. +struct SubgraphAssignments(RwLock>); + +impl SubgraphAssignments { + /// Creates a new empty list of started subgraph deployments. + fn new() -> Self { + Self(RwLock::new(HashMap::new())) + } + + /// Adds a new subgraph deployment to the list of started subgraph deployments. + fn add(&self, loc: DeploymentLocator, subgraph_assignment: SubgraphAssignment) { + self.0.write().insert(loc, subgraph_assignment); + } + + /// Updates the started subgraph deployment with the specified subgraph processing kind. + fn set_subgraph_kind(&self, loc: &DeploymentLocator, subgraph_kind: SubgraphProcessingKind) { + if let Some(subgraph_assignment) = self.0.write().get_mut(loc) { + subgraph_assignment.subgraph_kind = Some(subgraph_kind); + } + } + + /// Checks if the subgraph deployment is started. + fn is_assigned(&self, loc: &DeploymentLocator) -> bool { + self.0.read().contains_key(loc) + } + + /// Removes the subgraph deployment from the list of started subgraph deployments and returns its details. + fn take(&self, loc: &DeploymentLocator) -> Option { + self.0.write().remove(loc) + } +} + +/// Contains the details of a started subgraph deployment. +struct SubgraphAssignment { + /// The handle to the background task that starts this subgraph deployment. + handle: tokio::task::JoinHandle<()>, + + /// The subgraph processing kind of this subgraph deployment. + /// + /// Used to get the appropriate subgraph instance manager to forward the stop request to. + /// + /// Set to `None` until the subgraph manifest is loaded and parsed. + subgraph_kind: Option, +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 0044c6b7812..29d8e142db0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ # by running something like the following # docker build --target STAGE -f docker/Dockerfile . -FROM golang:bullseye as envsubst +FROM golang:bookworm AS envsubst # v1.2.0 ARG ENVSUBST_COMMIT_SHA=16035fe3571ad42c7796bf554f978bb2df64231b @@ -13,7 +13,7 @@ ARG ENVSUBST_COMMIT_SHA=16035fe3571ad42c7796bf554f978bb2df64231b RUN go install github.com/a8m/envsubst/cmd/envsubst@$ENVSUBST_COMMIT_SHA \ && strip -g /go/bin/envsubst -FROM rust:bullseye as graph-node-build +FROM rust:bookworm AS graph-node-build ARG COMMIT_SHA=unknown ARG REPO_NAME=unknown @@ -22,20 +22,18 @@ ARG TAG_NAME=unknown ADD . /graph-node -RUN apt-get update \ - && apt-get install -y cmake protobuf-compiler && \ - cd /graph-node && \ - RUSTFLAGS="-g" cargo build --release --package graph-node \ +RUN apt-get update && apt-get install -y cmake protobuf-compiler + +RUN cd /graph-node \ + && RUSTFLAGS="-g -C force-frame-pointers=yes" cargo build --release --package graph-node \ && cp target/release/graph-node /usr/local/bin/graph-node \ && cp target/release/graphman /usr/local/bin/graphman \ - # Reduce the size of the layer by removing unnecessary files. && cargo clean \ - && objcopy --only-keep-debug /usr/local/bin/graph-node /usr/local/bin/graph-node.debug \ - && strip -g /usr/local/bin/graph-node \ - && strip -g /usr/local/bin/graphman \ - && cd /usr/local/bin \ - && objcopy --add-gnu-debuglink=graph-node.debug graph-node \ - && echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ + && cp /usr/local/bin/graph-node /usr/local/bin/graph-node-debug \ + && cp /usr/local/bin/graphman /usr/local/bin/graphman-debug \ + && strip -g /usr/local/bin/graph-node /usr/local/bin/graphman + +RUN echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ && echo "TAG_NAME='$TAG_NAME'" >> /etc/image-info \ && echo "BRANCH_NAME='$BRANCH_NAME'" >> /etc/image-info \ && echo "COMMIT_SHA='$COMMIT_SHA'" >> /etc/image-info \ @@ -43,49 +41,41 @@ RUN apt-get update \ && echo "RUST_VERSION='$(rustc --version)'" >> /etc/image-info \ && echo "CARGO_DEV_BUILD='$CARGO_DEV_BUILD'" >> /etc/image-info -# Debug image to access core dumps -FROM graph-node-build as graph-node-debug -RUN apt-get update \ - && apt-get install -y curl gdb postgresql-client - -COPY docker/Dockerfile /Dockerfile -COPY docker/bin/* /usr/local/bin/ - # The graph-node runtime image with only the executable -FROM debian:bullseye-slim as graph-node -ENV RUST_LOG "" -ENV GRAPH_LOG "" -ENV EARLY_LOG_CHUNK_SIZE "" -ENV ETHEREUM_RPC_PARALLEL_REQUESTS "" -ENV ETHEREUM_BLOCK_CHUNK_SIZE "" - -ENV postgres_host "" -ENV postgres_user "" -ENV postgres_pass "" -ENV postgres_db "" -ENV postgres_args "sslmode=prefer" +FROM debian:bookworm-20241111-slim AS graph-node +ENV RUST_LOG="" +ENV GRAPH_LOG="" +ENV EARLY_LOG_CHUNK_SIZE="" +ENV ETHEREUM_RPC_PARALLEL_REQUESTS="" +ENV ETHEREUM_BLOCK_CHUNK_SIZE="" + +ENV postgres_host="" +ENV postgres_user="" +ENV postgres_pass="" +ENV postgres_db="" +ENV postgres_args="sslmode=prefer" # The full URL to the IPFS node -ENV ipfs "" +ENV ipfs="" # The etherum network(s) to connect to. Set this to a space-separated # list of the networks where each entry has the form NAME:URL -ENV ethereum "" +ENV ethereum="" # The role the node should have, one of index-node, query-node, or # combined-node -ENV node_role "combined-node" +ENV node_role="combined-node" # The name of this node -ENV node_id "default" +ENV node_id="default" # The ethereum network polling interval (in milliseconds) -ENV ethereum_polling_interval "" +ENV ethereum_polling_interval="" # The location of an optional configuration file for graph-node, as # described in ../docs/config.md # Using a configuration file is experimental, and the file format may # change in backwards-incompatible ways -ENV GRAPH_NODE_CONFIG "" +ENV GRAPH_NODE_CONFIG="" # Disable core dumps; this is useful for query nodes with large caches. Set # this to anything to disable coredumps (via 'ulimit -c 0') -ENV disable_core_dumps "" +ENV disable_core_dumps="" # HTTP port EXPOSE 8000 @@ -97,7 +87,7 @@ EXPOSE 8020 EXPOSE 8030 RUN apt-get update \ - && apt-get install -y libpq-dev ca-certificates netcat + && apt-get install -y libpq-dev ca-certificates netcat-openbsd ADD docker/wait_for docker/start /usr/local/bin/ COPY --from=graph-node-build /usr/local/bin/graph-node /usr/local/bin/graphman /usr/local/bin/ @@ -105,3 +95,13 @@ COPY --from=graph-node-build /etc/image-info /etc/image-info COPY --from=envsubst /go/bin/envsubst /usr/local/bin/ COPY docker/Dockerfile /Dockerfile CMD ["start"] + +# Debug and profiling image +# Build with: docker build --target graph-node-debug -f docker/Dockerfile . +FROM graph-node AS graph-node-debug +COPY --from=graph-node-build /usr/local/bin/graph-node-debug /usr/local/bin/graph-node +COPY --from=graph-node-build /usr/local/bin/graphman-debug /usr/local/bin/graphman +RUN apt-get update \ + && apt-get install -y curl gdb postgresql-client linux-perf procps binutils +COPY docker/Dockerfile /Dockerfile +COPY docker/bin/* /usr/local/bin/ diff --git a/docker/README.md b/docker/README.md index af95f4bbd52..6ea02f70b0f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,34 +1,9 @@ -# Graph Node Docker Image +# Running prebuilt `graph-node` images -Preconfigured Docker image for running a Graph Node. - -## Usage - -```sh -docker run -it \ - -e postgres_host= \ - -e postgres_port= \ - -e postgres_user= \ - -e postgres_pass= \ - -e postgres_db= \ - -e ipfs=: \ - -e ethereum=: \ - graphprotocol/graph-node:latest -``` - -### Example usage - -```sh -docker run -it \ - -e postgres_host=host.docker.internal \ - -e postgres_port=5432 \ - -e postgres_user=graph-node \ - -e postgres_pass=oh-hello \ - -e postgres_db=graph-node \ - -e ipfs=host.docker.internal:5001 \ - -e ethereum=mainnet:http://localhost:8545/ \ - graphprotocol/graph-node:latest -``` +You can run the `graph-node` docker image either in a [complete +setup](#docker-compose) controlled by Docker Compose, or, if you already +have an IPFS and Postgres server, [by +itself](#running-with-existing-ipfs-and-postgres). ## Docker Compose @@ -77,3 +52,17 @@ docker rmi graphprotocol/graph-node:latest # Tag the newly created image docker tag graph-node graphprotocol/graph-node:latest ``` + +## Running with existing IPFS and Postgres + +```sh +docker run -it \ + -e postgres_host= \ + -e postgres_port= \ + -e postgres_user= \ + -e postgres_pass= \ + -e postgres_db= \ + -e ipfs=: \ + -e ethereum=: \ + graphprotocol/graph-node:latest +``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 701fc68dacf..c78c2eb2194 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,13 +22,13 @@ services: ethereum: 'mainnet:http://host.docker.internal:8545' GRAPH_LOG: info ipfs: - image: ipfs/kubo:v0.14.0 + image: ipfs/kubo:v0.17.0 ports: - '5001:5001' volumes: - - ./data/ipfs:/data/ipfs + - ./data/ipfs:/data/ipfs:Z postgres: - image: postgres:14 + image: postgres ports: - '5432:5432' command: @@ -47,4 +47,4 @@ services: PGDATA: "/var/lib/postgresql/data" POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" volumes: - - ./data/postgres:/var/lib/postgresql/data + - ./data/postgres:/var/lib/postgresql/data:Z diff --git a/docker/tag.sh b/docker/tag.sh index 032ab54417b..1abafa95afa 100644 --- a/docker/tag.sh +++ b/docker/tag.sh @@ -25,4 +25,7 @@ tag_and_push "$SHORT_SHA" # Builds for tags vN.N.N become the 'latest' [[ "$TAG_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && tag_and_push latest +# If the build is from the master branch, tag it as 'nightly' +[ "$BRANCH_NAME" = "master" ] && tag_and_push nightly + exit 0 diff --git a/docs/aggregations.md b/docs/aggregations.md index 77f49b9bceb..3e90d7ee62f 100644 --- a/docs/aggregations.md +++ b/docs/aggregations.md @@ -1,10 +1,5 @@ # Timeseries and aggregations -**This feature is experimental. We very much encourage users to try this -out, but we might still need to make material changes to what's described -here in a manner that is not backwards compatible. That might require -deleting and redeploying any subgraph that uses the features here.** - _This feature is available from spec version 1.1.0 onwards_ ## Overview @@ -16,13 +11,13 @@ data points are to be aggregated. A very simple aggregation can be declared like ```graphql type Data @entity(timeseries: true) { id: Int8! - timestamp: Int8! + timestamp: Timestamp! price: BigDecimal! } type Stats @aggregation(intervals: ["hour", "day"], source: "Data") { id: Int8! - timestamp: Int8! + timestamp: Timestamp! sum: BigDecimal! @aggregate(fn: "sum", arg: "price") } ``` @@ -33,9 +28,11 @@ populate the `Stats` aggregations whenever a given hour or day ends. The type for the raw data points is defined with an `@entity(timeseries: true)` annotation. Timeseries types are immutable, and must have an `id` -field and a `timestamp` field. The `timestamp` is set automatically by -`graph-node` to the timestamp of the current block; if mappings set this -field, it is silently overridden when the entity is saved. +field and a `timestamp` field. The `id` must be of type `Int8` and is set +automatically so that ids are increasing in insertion order. The `timestamp` +is set automatically by `graph-node` to the timestamp of the current block; +if mappings set this field, it is silently overridden when the entity is +saved. Aggregations are declared with an `@aggregation` annotation instead of an `@entity` annotation. They must have an `id` field and a `timestamp` field. @@ -46,8 +43,6 @@ for example, to the beginning of the hour for an hourly aggregation. The the aggregation. Which one is chosen is not specified and should not be relied on. -**TODO**: add a `Timestamp` type and use that for `timestamp` - **TODO**: figure out whether we should just automatically add `id` and `timestamp` and have validation just check that these fields don't exist @@ -63,7 +58,7 @@ type Token @entity { .. } # Raw data points type TokenData @entity(timeseries: true) { id: Bytes! - timestamp: Int8! + timestamp: Timestamp! token: Token! amount: BigDecimal! priceUSD: BigDecimal! @@ -72,11 +67,11 @@ type TokenData @entity(timeseries: true) { # Aggregations over TokenData type TokenStats @aggregation(intervals: ["hour", "day"], source: "TokenData") { id: Int8! - timestamp: Int8! + timestamp: Timestamp! token: Token! totalVolume: BigDecimal! @aggregate(fn: "sum", arg: "amount") priceUSD: BigDecimal! @aggregate(fn: "last", arg: "priceUSD") - count: Int8! @aggregate(fn: "count") + count: Int8! @aggregate(fn: "count", cumulative: true) } ``` @@ -85,31 +80,25 @@ _dimensions_, and fields with the `@aggregate` directive are called _aggregates_. A timeseries type really represents many timeseries, one for each combination of values for the dimensions. -**TODO** As written, this supports buckets that start at zero with every new -hour/day. We also want to support cumulative statistics, i.e., snapshotting -of time series where a new bucket starts with the values of the previous -bucket. - -**TODO** Since average is a little more complicated to handle for cumulative -aggregations, and it doesn't seem like it used in practice, we won't -initially support it. (same for variance, stddev etc.) - -**TODO** The timeseries type can be simplified for some situations if -aggregations can be done over expressions, for example over `priceUSD * -amount` to track `totalVolumeUSD` +The same timeseries can be used for multiple aggregations. For example, the +`Stats` aggregation could also be formed by aggregating over the `TokenData` +timeseries. Since `Stats` doesn't have a `token` dimension, all aggregates +will be formed across all tokens. -**TODO** It might be necessary to allow `@aggregate` fields that are only -used for some intervals. We could allow that with syntax like -`@aggregate(fn: .., arg: .., interval: "day")` +Each `@aggregate` by default starts at 0 for each new bucket and therefore +just aggregates over the time interval for the bucket. The `@aggregate` +directive also accepts a boolean flag `cumulative` that indicates whether +the aggregation should be cumulative. Cumulative aggregations aggregate over +the entire timeseries up to the end of the time interval for the bucket. ## Specification ### Timeseries A timeseries is an entity type with the annotation `@entity(timeseries: -true)`. It must have an `id` attribute and a `timestamp` attribute of type -`Int8`. It must not also be annotated with `immutable: false` as timeseries -are always immutable. +true)`. It must have an `id` attribute of type `Int8` and a `timestamp` +attribute of type `Timestamp`. It must not also be annotated with +`immutable: false` as timeseries are always immutable. ### Aggregations @@ -121,15 +110,18 @@ must have two arguments: - `source`: the name of a timeseries type. Aggregates are computed based on the attributes of the timeseries type. -The aggregation type must have an `id` attribute and a `timestamp` attribute -of type `Int8`. +The aggregation type must have an `id` attribute of type `Int8` and a +`timestamp` attribute of type `Timestamp`. The aggregation type must have at least one attribute with the `@aggregate` annotation. These attributes must be of a numeric type (`Int`, `Int8`, `BigInt`, or `BigDecimal`) The annotation must have two arguments: - `fn`: the name of an aggregation function -- `arg`: the name of an attribute in the timeseries type +- `arg`: the name of an attribute in the timeseries type, or an expression + using only constants and attributes of the timeseries type + +#### Aggregation functions The following aggregation functions are currently supported: @@ -142,15 +134,42 @@ The following aggregation functions are currently supported: | `first` | First value | | `last` | Last value | -## Querying +The `first` and `last` aggregation function calculate the first and last +value in an interval by sorting the data by `id`; `graph-node` enforces +correctness here by automatically setting the `id` for timeseries entities. + +#### Aggregation expressions -_This section is not implemented yet, and will require a bit more thought -about details_ +The `arg` can be the name of any attribute in the timeseries type, or an +expression using only constants and attributes of the timeseries type such +as `price * amount` or `greatest(amount0, amount1)`. Expressions use SQL +syntax and support a subset of builtin SQL functions, operators, and other +constructs. -**TODO** As written, timeseries points like `TokenData` can be queried like -any other entity. It would be nice to restrict how these data points can be -queried, maybe even forbid it, as that would give us more latitude in how we -store that data. +Supported operators are `+`, `-`, `*`, `/`, `%`, `^`, `=`, `!=`, `<`, `<=`, +`>`, `>=`, `<->`, `and`, `or`, and `not`. In addition the operators `is +[not] {null|true|false}`, and `is [not] distinct from` are supported. + +The supported SQL functions are the [math +functions](https://www.postgresql.org/docs/current/functions-math.html) +`abs`, `ceil`, `ceiling`, `div`, `floor`, `gcd`, `lcm`, `mod`, `power`, +`sign`, and the [conditional +functions](https://www.postgresql.org/docs/current/functions-conditional.html) +`coalesce`, `nullif`, `greatest`, and `least`. + +The +[statement](https://www.postgresql.org/docs/current/functions-conditional.html#FUNCTIONS-CASE) +`case when .. else .. end` is also supported. + +Some examples of valid expressions, assuming the underlying timeseries +contains the mentioned fields: + +- Aggregate the value of a token: `@aggregate(fn: "sum", arg: "priceUSD * amount")` +- Aggregate the maximum positive amount of two different amounts: + `@aggregate(fn: "max", arg: "greatest(amount0, amount1, 0)")` +- Conditionally sum an amount: `@aggregate(fn: "sum", arg: "case when amount0 > amount1 then amount0 else 0 end")` + +## Querying We create a toplevel query field for each aggregation. That query field accepts the following arguments: @@ -159,16 +178,18 @@ accepts the following arguments: dimension - A mandatory `interval` - An optional `current` to indicate whether to include the current, - partially filled bucket in the response. Can be either `ignore` (the + partially filled bucket in the response. Can be either `exclude` (the default) or `include` -- Optional `timestamp_{gte|gt|lt|lte|eq}` filters to restrict the range of - timestamps to return -- Timeseries are always sorted by the dimensions in the order in which they - are declared in the schema and the `timestamp` in descending order +- Optional `timestamp_{gte|gt|lt|lte|eq|in}` filters to restrict the range + of timestamps to return. The timestamp to filter by must be a string + containing microseconds since the epoch. The value `"1704164640000000"` + corresponds to `2024-01-02T03:04Z` +- Timeseries are sorted by `timestamp` and `id` in descending order by + default ```graphql token_stats(interval: "hour", - current: ignore, + current: exclude, where: { token: "0x1234", timestamp_gte: 1234567890, @@ -181,5 +202,52 @@ token_stats(interval: "hour", } ``` -**TODO**: what about time-travel? Is it possible to include a block -constraint? +### Current Bucket + +By default, aggregation queries return only completed, rolled-up buckets +(`current: exclude`). These are buckets whose time interval has ended and +whose data has been fully aggregated by `graph-node`'s rollup process. + +Setting `current: include` adds an additional, partially filled bucket that +is computed on-the-fly from the unrolled timeseries data that has been +inserted since the last rollup. This current bucket aggregates raw data +points from the source timeseries table that have not yet been rolled up +into the aggregation table. It covers the time period from the end of the +last completed bucket up to the most recent data point. + +- `current: exclude` (default) — return only completed, rolled-up buckets +- `current: include` — also return the in-progress bucket computed from + unrolled source data + +The current bucket is useful when you need near-real-time aggregation data +without waiting for the next rollup cycle to complete. + +#### Nested Aggregation Queries + +The `current` argument also works on nested aggregation fields accessed +through a parent entity. For example, if a `Token` entity has a derived +aggregation field `tokenStats`, you can query the current bucket for each +token: + +```graphql +{ + tokens { + id + name + tokenStats(interval: "hour", current: include) { + timestamp + totalVolume + } + } +} +``` + +This returns both the completed rolled-up hourly buckets and the current +in-progress bucket for each token's stats. + +#### Limitations + +Current bucket support for nested aggregation fields is only available when +the field references a single aggregation type. It is not supported when the +aggregation field is accessed through an interface with multiple +implementations. diff --git a/docs/amp-powered-subgraphs.md b/docs/amp-powered-subgraphs.md new file mode 100644 index 00000000000..60bccdde449 --- /dev/null +++ b/docs/amp-powered-subgraphs.md @@ -0,0 +1,582 @@ +# Amp-powered subgraphs + +> [!WARNING] +> This feature is experimental and may change in future releases. + +> [!NOTE] +> This feature is available starting from spec version `1.5.0` + +Amp-powered subgraphs are a new kind of subgraphs with SQL data sources that query and index data from the Amp servers. +They are significantly more efficient than the standard subgraphs, and the indexing time can be reduced from days and weeks, +to minutes and hours in most cases. + +## Prerequisites + +To enable Amp-powered subgraphs, the `GRAPH_AMP_FLIGHT_SERVICE_ADDRESS` ENV variable must be set to a valid Amp Flight gRPC service address. + +Additionally, if authentication is required for the Amp Flight gRPC service, the `GRAPH_AMP_FLIGHT_SERVICE_TOKEN` ENV variable must contain a valid authentication token. + +## Subgraph manifest + +Amp-powered subgraphs introduce a new structure for defining Amp subgraph data sources within the manifest. + +### Spec version + +The minimum spec version for Amp-powered subgraphs is `1.5.0`. + +

+Example YAML: + +```diff ++ specVersion: 1.5.0 + dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Data source structure + +### `kind` + +Every Amp data source must have the `kind` set to `amp`, and Amp-powered subgraphs must contain only Amp data sources. +This is used to assign the subgraph to the appropriate indexing process. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: ++ - kind: amp + name: Transfers + network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `name` + +Every Amp data source must have the `name` set to a non-empty string, containing only numbers, letters, hypens, or underscores. +This name is used for observability purposes and to identify progress and potential errors produced by the data source. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp ++ name: Transfers + network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `network` + +Every Amp data source must have the `network` field set to a valid network name. +This is used to validate that the SQL queries for this data source produce results for the expected network. + +> [!NOTE] +> Currently, the SQL queries are required to produce results for a single network in order to maintain compatibility with non-Amp subgraphs. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers ++ network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `source` + +Every Amp data source must have a valid `source` that describes the behavior of SQL queries from this data source. + +### `source.dataset` + +Contains the name of the dataset that can be queried by SQL queries in this data source. +This is used to validate that the SQL queries for this data source only query the expected dataset. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `source.tables` + +Contains the names of the tables that can be queried by SQL queries in this data source. +This is used to validate that the SQL queries for this data source only query the expected tables. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: + dataset: edgeandnode/ethereum_mainnet ++ tables: ++ - blocks ++ - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Optional `source.address` + +Contains the contract address with which SQL queries in the data source interact. + +Enables SQL query reuse through `sg_source_address()` calls instead of hard-coding the contract address. +SQL queries resolve `sg_source_address()` calls to this contract address. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ address: "0xc944E90C64B2c07662A292be6244BDf05Cda44a7" + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Optional `source.startBlock` + +Contains the minimum block number that SQL queries in the data source can query. +This is used as a starting point for the indexing process. + +_When not provided, defaults to block number `0`._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ startBlock: 11446769 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Optional `source.endBlock` + +Contains the maximum block number that SQL queries in the data source can query. +Reaching this block number will complete the indexing process. + +_When not provided, defaults to the maximum possible block number._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `transformer` + +Every Amp data source must have a valid `transformer` that describes the transformations of source tables indexed by the Amp-powered subgraph. + +### `transformer.apiVersion` + +Represents the version of this transformer. Each version may contain a different set of features. + +> [!NOTE] +> Currently, only the version `0.0.1` is available. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: ++ apiVersion: 0.0.1 + tables: + - name: Transfers + file: +``` + +
+ +### Optional `transformer.abis` + +Contains a list of ABIs that SQL queries can reference to extract event signatures. + +Enables the use of `sg_event_signature('CONTRACT_NAME', 'EVENT_NAME')` calls in the +SQL queries which are resolved to full event signatures based on this list. + +_When not provided, defaults to an empty list._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: ++ abis: ++ - name: ERC721 # The name of the contract ++ file: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `transformer.tables` + +Contains a list of transformed tables that extract data from source tables into subgraph entities. + +### Transformer table structure + +### `transformer.tables[i].name` + +Represents the name of the transformed table. Must reference a valid entity name from the subgraph schema. + +
+Example: + +**GraphQL schema:** + +```graphql +type Block @entity(immutable: true) { + # .. entity fields ... +} +``` + +**YAML manifest:** + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Blocks + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: + apiVersion: 0.0.1 ++ tables: ++ - name: Block + file: +``` + +
+ +### `transformer.tables[i].query` + +Contains an inline SQL query that executes on the Amp server. +This is useful for simple SQL queries like `SELECT * FROM "edgeandnode/ethereum_mainnet".blocks;`. +For more complex cases, a separate file containing the SQL query can be used in the `file` field. + +The data resulting from this SQL query execution transforms into subgraph entities. + +_When not provided, the `file` field is used instead._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Blocks + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: + apiVersion: 0.0.1 ++ tables: + - name: Block ++ query: SELECT * FROM "edgeandnode/ethereum_mainnet".blocks; +``` + +
+ +### `transformer.tables[i].file` + +Contains the IPFS link to the SQL query that executes on the Amp server. + +The data resulting from this SQL query execution transforms into subgraph entities. + +_Ignored when the `query` field is provided._ +_When not provided, the `query` field is used instead._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Blocks + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: + apiVersion: 0.0.1 ++ tables: + - name: Block ++ file: +``` + +
+ +### Amp-powered subgraph examples + +Complete examples on how to create, deploy and query Amp-powered subgraphs are available in a separate repository: +https://github.com/edgeandnode/amp-subgraph-examples + +## SQL query requirements + +### Names + +The names of tables, columns, and aliases must not start with `amp_` as this +prefix is reserved for internal use. + +### Block numbers + +Every SQL query in Amp-powered subgraphs must return the block number for every row. +This is required because subgraphs rely on this information for storing subgraph entities. + +Graph-node will look for block numbers in the following columns: +`_block_num`, `block_num`, `blockNum`, `block`, `block_number`, `blockNumber`. + +Example SQL query: `SELECT _block_num, /* .. other projections .. */ FROM "edgeandnode/ethereum_mainnet".blocks;` + +### Block hashes + +Every SQL query in Amp-powered subgraphs is expected to return the block hash for every row. +This is required because subgraphs rely on this information for storing subgraph entities. + +When a SQL query does not have the block hash projection, graph-node will attempt to get it from the +source tables specified in the subgraph manifest. + +Graph-node will look for block hashes in the following columns: +`hash`, `block_hash`, `blockHash`. + +Example SQL query: `SELECT hash, /* .. other projections .. */ FROM "edgeandnode/ethereum_mainnet".blocks;` + +> [!NOTE] +> If a table does not contain the block hash column, it can be retrieved by joining that table with another that contains the column on the `_block_num` column. + +### Block timestamps + +> [!NOTE] +> Only required for Amp-powered subgraphs that use subgraph aggregations. + +Every SQL query in Amp-powered subgraphs is expected to return the block timestamps for every row. +This is required because subgraphs rely on this information for storing subgraph entities. + +When a SQL query does not have the block timestamps projection, graph-node will attempt to get it from the +source tables specified in the subgraph manifest. + +Graph-node will look for block timestamps in the following columns: +`timestamp`, `block_timestamp`, `blockTimestamp`. + +Example SQL query: `SELECT timestamp, /* .. other projections .. */ FROM "edgeandnode/ethereum_mainnet".blocks;` + +> [!NOTE] +> If a table does not contain the block timestamp column, it can be retrieved by joining that table with another that contains the column on the `_block_num` column. + +## Type conversions + +Amp core SQL data types are converted intuitively to compatible subgraph entity types. + +## Schema generation + +Amp-powered subgraphs support the generation of GraphQL schemas based on the schemas of SQL queries referenced in the subgraph manifest. +This is useful when indexing entities that do not rely on complex relationships, such as contract events. + +The generated subgraph entities are immutable. + +To enable schema generation, simply remove the `schema` field from the subgraph manifest. + +> [!NOTE] +> For more flexibility and control over the schema, a manually created GraphQL schema is preferred. + +## Aggregations + +Amp-powered subgraphs fully support the subgraph aggregations feature. +This allows having complex aggregations on top of data indexed from the Amp servers. + +For more information on using the powerful subgraph aggregations feature, +refer to the [documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/aggregations.md). + +## Composition + +Amp-powered subgraphs fully support the subgraph composition feature. +This allows applying complex subgraph mappings on top of data indexed from the Amp servers. + +For more information on using the powerful subgraph composition feature, +refer to the [documentation](https://github.com/graphprotocol/example-composable-subgraph). + +## ENV variables + +Amp-powered subgraphs feature introduces the following new ENV variables: + +- `GRAPH_AMP_FLIGHT_SERVICE_ADDRESS` – The address of the Amp Flight gRPC service. _Defaults to `None`, which disables support for Amp-powered subgraphs._ +- `GRAPH_AMP_FLIGHT_SERVICE_TOKEN` – Token used to authenticate Amp Flight gRPC service requests. _Defaults to `None`, which disables authentication._ +- `GRAPH_AMP_BUFFER_SIZE` – Maximum number of response batches to buffer in memory per stream for each SQL query. _Defaults to `1,000`._ +- `GRAPH_AMP_BLOCK_RANGE` – Maximum number of blocks to request per stream for each SQL query. _Defaults to `100,000`._ +- `GRAPH_AMP_QUERY_RETRY_MIN_DELAY_SECONDS` – Minimum time to wait before retrying a failed SQL query to the Amp server. _Defaults to `1` second._ +- `GRAPH_AMP_QUERY_RETRY_MAX_DELAY_SECONDS` – Maximum time to wait before retrying a failed SQL query to the Amp server. _Defaults to `600` seconds._ + +## Metrics + +In addition to reporting updates to the existing `deployment_status`, `deployment_head`, `deployment_synced` and `deployment_blocks_processed_count` +metrics, Amp-powered subgraphs feature introduces the following new metrics: + +- `deployment_target` – Tracks the maximum block number currently available for indexing within a deployment. +- `deployment_indexing_duration_seconds` – Tracks the total duration in seconds of deployment indexing. + +Additionally, the `deployment_sync_secs` is extended with new sections specific to the Amp indexing process. diff --git a/docs/config.md b/docs/config.md index 0a6c3536da4..db5fc083a5f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -5,12 +5,11 @@ CLI. The location of the file is passed with the `--config` command line switch. configuration file, it is not possible to use the options `--postgres-url`, `--postgres-secondary-hosts`, and `--postgres-host-weights`. -The TOML file consists of four sections: +The TOML file consists of three sections: -- `[chains]` sets the endpoints to blockchain clients. -- `[store]` describes the available databases. -- `[ingestor]` sets the name of the node responsible for block ingestion. -- `[deployment]` describes how to place newly deployed subgraphs. +- [`[chains]`](#configuring-chains) lists the available chains and how to access them. +- [`[store]`](#configuring-multiple-databases) describes the available databases. +- [`[deployment]`](#controlling-deployment) describes how to place newly deployed subgraphs. Some of these sections support environment variable expansion out of the box, most notably Postgres connection strings. The official `graph-node` Docker image @@ -100,48 +99,162 @@ time the configuration is changed to make sure that the connection pools are what is expected. Here, `$all_nodes` should be a list of all the node names that will use this configuration file. -## Configuring Ethereum Providers +## Configuring Chains -The `[chains]` section controls the ethereum providers that `graph-node` +The `[chains]` section controls the providers that `graph-node` connects to, and where blocks and other metadata for each chain are -stored. The section consists of the name of the node doing block ingestion -(currently not used), and a list of chains. The configuration for a chain -`name` is specified in the section `[chains.]`, and consists of the -`shard` where chain data is stored and a list of providers for that -chain. For each provider, the following information must be given: - -- `label`: a label that is used when logging information about that - provider (not implemented yet) +stored. The section consists of the name of the node responsible for block +ingestion and a list of chains. Block ingestion only runs on the node +whose `--node-id` matches the `ingestor` value. The +`--disable-block-ingestor` flag (or `DISABLE_BLOCK_INGESTOR` env var) +acts as a hard override that always prevents ingestion regardless of +the config. + +The section-level setting `cache_size` controls the default number of +blocks from the chain head for which block data is kept cached. Individual +chains can override this value. The default is 500. When the environment +variable `GRAPH_STORE_IGNORE_BLOCK_CACHE` is set, blocks older than +`cache_size` are treated as if they have no data. The value must be greater +than the reorg threshold. + +The configuration for a chain `name` is specified in the section +`[chains.]`, with the following: + +- `shard`: where chain data is stored +- `protocol`: the protocol type being indexed, default `ethereum` + (alternatively `near`, `cosmos`,`arweave`,`starknet`) +- `amp`: the network name used by AMP for this chain; defaults to the chain name. + Set this when AMP uses a different name than graph-node (e.g., `amp = "ethereum-mainnet"` + on a chain named `mainnet`). +- `cache_size`: number of blocks from the chain head for which to keep + block data cached. Defaults to the section-level `cache_size`. +- `provider`: a list of providers for that chain + +Additionally, Ethereum chains support per-chain RPC tuning settings. When +omitted, each setting falls back to its corresponding environment variable +default (see [environment-variables.md](environment-variables.md) for +details): + +- `polling_interval`: block ingestor polling interval in milliseconds. + Default: `ETHEREUM_POLLING_INTERVAL` (1000ms). +- `json_rpc_timeout`: timeout for JSON-RPC requests in seconds. + Default: `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT` (180s). +- `request_retries`: number of times to retry failed JSON-RPC requests. + Default: `GRAPH_ETHEREUM_REQUEST_RETRIES` (10). +- `max_block_range_size`: maximum number of blocks to scan for triggers per + request. Default: `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE` (1000). +- `block_batch_size`: number of blocks to request in parallel. + Default: `ETHEREUM_BLOCK_BATCH_SIZE` (10). +- `block_ptr_batch_size`: number of block pointers to request in parallel. + Default: `ETHEREUM_BLOCK_PTR_BATCH_SIZE` (100). +- `max_event_only_range`: maximum range for `eth_getLogs` requests that + don't filter on contract address. + Default: `GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE` (500). +- `target_triggers_per_block_range`: ideal number of triggers per batch. + Default: `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE` (100). +- `get_logs_max_contracts`: maximum contracts per `eth_getLogs` call. + Default: `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS` (2000). +- `block_ingestor_max_concurrent_json_rpc_calls`: maximum concurrent + JSON-RPC calls for transaction receipts during block ingestion. + Default: `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS` (1000). +- `genesis_block_number`: genesis block number for this chain. + Default: `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER` (0). + +A `provider` is an object with the following characteristics: + +- `label`: the name of the provider, which will appear in logs +- `details`: provider details + +`details` includes the following: + +- `type`: one of `web3` (default), `firehose`, or `web3call` - `transport`: one of `rpc`, `ws`, and `ipc`. Defaults to `rpc`. - `url`: the URL for the provider - `features`: an array of features that the provider supports, either empty - or any combination of `traces` and `archive` + or any combination of the following for Web3 providers: + - `traces`: provider supports `debug_traceBlockByNumber` for call tracing + - `archive`: provider is an archive node with full historical state + - `no_eip1898`: provider doesn't support EIP-1898 (block parameter by hash/number object) + - `no_eip2718`: provider doesn't return the `type` field in transaction receipts + (pre-EIP-2718 chains). When set, receipts are patched to add + `"type": "0x0"` for legacy transaction compatibility. + - `compression/`: provider supports compression for RPC requests, + where `` is the compression method supported by the provider, + one of `gzip`, `brotli`, or `deflate`. The default is no compression. + + For Firehose providers: `compression` and `filters` + - `headers`: HTTP headers to be added on every request. Defaults to none. - `limit`: the maximum number of subgraphs that can use this provider. Defaults to unlimited. At least one provider should be unlimited, otherwise `graph-node` might not be able to handle all subgraphs. The tracking for this is approximate, and a small amount of deviation from this value should be expected. The deviation will be less than 10. +- `token`: bearer token, for Firehose providers +- `key`: API key for Firehose providers when using key-based authentication + +Note that for backwards compatibility, Web3 provider `details` can be specified at the "top level" of +the `provider`. -The following example configures two chains, `mainnet` and `kovan`, where -blocks for `mainnet` are stored in the `vip` shard and blocks for `kovan` +The following example configures three chains, `mainnet`, `sepolia` and `near-mainnet`, where +blocks for `mainnet` are stored in the `vip` shard and blocks for `sepolia` are stored in the primary shard. The `mainnet` chain can use two different -providers, whereas `kovan` only has one provider. +providers, whereas `sepolia` only has one provider. The `near-mainnet` chain expects data from +the `near` protocol via a Firehose, where the Firehose offers the `compression` and `filters` +optimisations. ```toml [chains] ingestor = "block_ingestor_node" +cache_size = 500 [chains.mainnet] shard = "vip" +amp = "ethereum-mainnet" +# Per-chain RPC tuning (all optional — omitted fields use env var defaults) +json_rpc_timeout = 300 +request_retries = 15 +max_block_range_size = 2000 provider = [ { label = "mainnet1", url = "http://..", features = [], headers = { Authorization = "Bearer foo" } }, { label = "mainnet2", url = "http://..", features = [ "archive", "traces" ] } ] -[chains.kovan] +[chains.sepolia] shard = "primary" -provider = [ { label = "kovan", url = "http://..", features = [] } ] +provider = [ { label = "sepolia", url = "http://..", features = [] } ] + +[chains.near-mainnet] +shard = "blocks_b" +protocol = "near" +provider = [ { label = "near", details = { type = "firehose", url = "https://..", key = "", features = ["compression", "filters"] } } ] ``` +### Block ingestor failover + +When the block ingestor's `do_poll()` call fails (after all internal per-request retries are +exhausted), `graph-node` automatically attempts to switch to a healthier provider. The logic +is: + +1. **Probe the current provider first.** `do_poll()` can fail for reasons unrelated to RPC + availability (e.g. a database error or a chain reorg). If the current provider still + responds to `eth_blockNumber`, the failure was not caused by the provider — no switch + occurs. +2. **Probe all alternatives in parallel.** If the current provider is unreachable, all other + validated providers are probed simultaneously via `eth_blockNumber` to minimise wait time + when providers are timing out. +3. **Switch to the first reachable provider.** The first provider to respond successfully + to the probe is selected as the new provider for the ingestor. + The remaining probes are cancelled at this point. +4. **If all providers are unreachable**, the ingestor stays on the current provider and + re-probes on the next `do_poll()` failure. + +There is no automatic return to the original provider. Once the ingestor switches, it stays on +the new provider until that provider fails, at which point the same probe-and-switch logic +applies. + +Only validated providers are eligible as failover candidates. A provider in a temporary failure +state (e.g. unreachable at startup, pending re-validation) is excluded until it passes +validation again. + ### Controlling the number of subgraphs using a provider **This feature is experimental and might be removed in a future release** diff --git a/docs/dump.md b/docs/dump.md new file mode 100644 index 00000000000..c8a3d32372b --- /dev/null +++ b/docs/dump.md @@ -0,0 +1,311 @@ +## Dump Format + +The `graphman dump` command exports all entity data and metadata for a +single subgraph deployment into a self-contained directory of Parquet files +and JSON metadata. The resulting dump can be used to restore the deployment +into a different `graph-node` instance via `graphman restore`. Dumps are +consistent snapshots of the deployment's state at a specific point in time. + +**WARNING**: The dump and restore commands are experimental and can not +replace proper database backups at this point. In particular, there is no +guarantee that a dump will be restorable. Having said that, we encourage +users to try out the dump and restore commands in non-production +environments and report any issues they encounter. + +**WARNING**: Dumping happens in a single transaction and can put significant +load on the database for large subgraphs. Use with caution on production +instances. + +**WARNING**: Restoring a dump will currently create all the default indexes +that a new deployment gets, and ignores the indexes that might have been +carefully curated for the original deployment, even though they are recorded +in the dump. This can lead to very long restore times for large subgraphs. +The restore process will be optimized in the future to only create indexes +that are present in the dump's metadata. + +### Usage + +#### Dumping a deployment + +```bash +graphman dump +``` + +`` identifies the subgraph deployment to dump. It can be a +subgraph name, a deployment hash (`Qm...`), or a database namespace +(`sgdNNN`). `` is the path where the dump will be written; it +will be created if it does not exist. + +Running `graphman dump` against an existing dump directory performs an +**incremental dump**: only rows added since the last dump are exported, and +new chunk files are appended rather than rewriting existing ones. + +```bash +# Full dump +graphman dump my-subgraph /backups/my-subgraph + +# Incremental update of the same dump +graphman dump my-subgraph /backups/my-subgraph +``` + +#### Restoring a deployment + +```bash +graphman restore [options] +``` + +`` is the path to a dump previously created with `graphman +dump`. + +| Option | Description | +| ----------- | --------------------------------------------------------------------------------------------------- | +| `--shard` | Target database shard. Uses deployment rules (or primary shard) when omitted. Required with `--add` | +| `--name` | Subgraph name for deployment rule matching and node assignment. Falls back to an existing name | +| `--replace` | Drop and recreate if the deployment already exists in the target shard | +| `--add` | Create a copy in a shard that doesn't already have this deployment (requires `--shard`) | +| `--force` | Replace if the deployment exists in the target shard, add if it doesn't | + +`--replace`, `--add`, and `--force` are mutually exclusive. When none is +given, restore fails if the deployment already exists in the target shard. + +```bash +# Restore into the default shard +graphman restore /backups/my-subgraph + +# Restore into a specific shard, replacing if it already exists +graphman restore /backups/my-subgraph --shard shard1 --replace + +# Force-restore (replace or add as needed) +graphman restore /backups/my-subgraph --force +``` + +### Directory layout + +A dump directory has the following structure: + +``` +/ + metadata.json -- deployment metadata + per-table state + schema.graphql -- raw GraphQL schema text + subgraph.yaml -- raw subgraph manifest YAML (optional) + / + chunk_000000.parquet -- rows ordered by vid + chunk_000001.parquet -- incremental append (future chunks) + ... + data_sources$/ + chunk_000000.parquet -- dynamic data sources +``` + +Each entity type defined in the GraphQL schema gets its own subdirectory, +named after the entity type exactly as it appears in the schema (e.g. +`Token/`, `Pool/`). The Proof of Indexing appears as a regular entity +directory name `Poi$`. The special `data_sources$` directory holds dynamic +data sources created at runtime. + +Within each directory, data is stored in numbered chunk files +(`chunk_000000.parquet`, `chunk_000001.parquet`, ...). A fresh dump +produces a single `chunk_000000.parquet` per table. Incremental dumps +append new chunks rather than rewriting existing ones. + +The GraphQL schema and subgraph manifest are stored as separate plain-text +files `schema.graphql` and `subgraph.yaml`. + +### metadata.json + +The top-level `metadata.json` contains everything needed to reconstruct the +deployment's table structure, plus diagnostic information captured at dump +time. Its structure is: + +```json +{ + "version": 1, + "deployment": "Qm...", + "network": "mainnet", + + "manifest": { + "spec_version": "1.0.0", + "description": "Optional subgraph description", + "repository": "https://github.com/...", + "features": ["..."], + "entities_with_causality_region": ["EntityType1"], + "history_blocks": 2147483647 + }, + + "earliest_block_number": 12345, + "start_block": { "number": 12345, "hash": "0xabc..." }, + "head_block": { "number": 99999, "hash": "0xdef..." }, + "entity_count": 150000, + + "graft_base": null, + "graft_block": null, + "debug_fork": null, + + "health": { + "failed": false, + "health": "healthy", + "fatal_error": null, + "non_fatal_errors": [] + }, + + "indexes": { + "token": [ + "CREATE INDEX CONCURRENTLY IF NOT EXISTS attr_0_0_id ON sgd.token USING btree (id)" + ] + }, + + "tables": { + "Token": { + "immutable": true, + "has_causality_region": false, + "chunks": [ + { + "file": "Token/chunk_000000.parquet", + "min_vid": 0, + "max_vid": 50000, + "row_count": 50000 + } + ], + "max_vid": 50000 + }, + "data_sources$": { + "immutable": false, + "has_causality_region": true, + "chunks": [ + { + "file": "data_sources$/chunk_000000.parquet", + "min_vid": 0, + "max_vid": 100, + "row_count": 100 + } + ], + "max_vid": 100 + } + } +} +``` + +**Field descriptions:** + +| Field | Description | +| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `version` | Format version. Must be `1`. | +| `deployment` | Deployment hash (`Qm...`). | +| `network` | The blockchain network (e.g. `mainnet`, `goerli`). | +| `manifest` | Manifest metadata extracted from `subgraphs.subgraph_manifest`. | +| `manifest.spec_version` | Subgraph API version. Required to parse `schema.graphql`. | +| `manifest.entities_with_causality_region` | Entity types that have a `causality_region` column. | +| `manifest.history_blocks` | How many blocks of entity version history are retained. | +| `earliest_block_number` | Earliest block for which data exists (accounts for pruning). | +| `start_block` | The block where indexing started. Null if not set. | +| `head_block` | The latest indexed block at dump time. | +| `entity_count` | Total entity count across all tables. | +| `graft_base` | Deployment hash of the graft base, if any. | +| `graft_block` | Block pointer of the graft point, if any. | +| `debug_fork` | Debug fork deployment hash, if any. | +| `health` | Point-in-time health snapshot. Not used during restore. | +| `indexes` | Point-in-time index definitions as SQL. Not used during restore (indexes are auto-created by `Layout::create_relational_schema()`). | +| `tables` | Per-table metadata keyed by entity type name (or `data_sources$`). | + +Each entry in `tables` contains: + +| Field | Description | +| ---------------------- | ------------------------------------------------------------------------------ | +| `immutable` | Whether the entity type is immutable (uses `block$` instead of `block_range`). | +| `has_causality_region` | Whether rows have a `causality_region` column. | +| `chunks` | Ordered list of Parquet chunk files for this table. | +| `chunks[].file` | Relative path from the dump directory. | +| `chunks[].min_vid` | Minimum `vid` value in this chunk. | +| `chunks[].max_vid` | Maximum `vid` value in this chunk. | +| `chunks[].row_count` | Number of rows in this chunk. | +| `max_vid` | Maximum `vid` across all chunks. `-1` if the table is empty. | + +### Parquet schema: entity tables + +Each entity table's Parquet file uses an Arrow schema derived from the +entity's GraphQL definition. Columns are ordered as follows: + +1. **System columns** (always present, in this order): + - `vid` (Int64, non-nullable) -- row version ID + - Block tracking (one of): + - Immutable entities: `block$` (Int32, non-nullable) + - Mutable entities: `block_range_start` (Int32, non-nullable), + `block_range_end` (Int32, nullable -- null means unbounded/current) + - `causality_region` (Int32, non-nullable) -- only if the entity has one + +2. **Data columns** in GraphQL declaration order, skipping fulltext + (`TSVector`) columns which are generated and rebuilt on restore. + +The PostgreSQL `int4range` type used for `block_range` is decomposed into +two scalar columns (`block_range_start`, `block_range_end`) in the Parquet +representation. This avoids the need for a custom range type in Arrow. + +#### Type mapping + +GraphQL/PostgreSQL column types map to Arrow data types as follows: + +| ColumnType | Arrow DataType | Notes | +| --------------- | ------------------------------ | -------------------------------------------------------------- | +| `Boolean` | `Boolean` | | +| `Int` | `Int32` | | +| `Int8` | `Int64` | | +| `Bytes` | `Binary` | Raw bytes, no hex encoding | +| `BigInt` | `Utf8` | Stored as decimal string for arbitrary precision | +| `BigDecimal` | `Utf8` | Stored as decimal string for arbitrary precision | +| `Timestamp` | `Timestamp(Microsecond, None)` | Microseconds since epoch, no timezone | +| `String` | `Utf8` | | +| `Enum(...)` | `Utf8` | Enum variant as string (cast from PG enum to text during dump) | +| `TSVector(...)` | _skipped_ | Fulltext index columns are generated; rebuilt on restore | + +**Array columns:** A GraphQL list field (e.g. `tags: [String!]!`) is +stored as `List` where `T` is the base Arrow type from the table +above. Whether a column is a list is determined by the GraphQL field type, +not by `ColumnType`. For example, `[String!]!` becomes `List` and +`[Int!]` becomes `List`. + +**Nullability** follows the GraphQL schema: non-null fields produce +non-nullable Arrow columns; optional fields produce nullable columns. List +elements within list columns are always marked nullable in the Arrow schema. + +### Parquet schema: data_sources$ + +The `data_sources$` table has a fixed schema independent of the GraphQL +definition: + +| Column | Arrow DataType | Nullable | Description | +| ------------------- | -------------- | -------- | -------------------------------------------------- | +| `vid` | `Int64` | no | Row version ID | +| `block_range_start` | `Int32` | no | Lower bound of `block_range` | +| `block_range_end` | `Int32` | yes | Upper bound (null = unbounded) | +| `causality_region` | `Int32` | no | Causality region | +| `manifest_idx` | `Int32` | no | Index into the manifest's data source list | +| `parent` | `Int32` | yes | Self-referencing parent data source | +| `id` | `Binary` | yes | Data source identifier | +| `param` | `Binary` | yes | Data source parameter | +| `context` | `Utf8` | yes | JSON context | +| `done_at` | `Int32` | yes | Block number where the data source was marked done | + +### Compression + +All Parquet files use ZSTD compression (default level). + +### Row ordering + +Within each Parquet chunk file, rows are ordered by `vid` (ascending). +This matches the primary key ordering in PostgreSQL and enables efficient +sequential reads during restore. + +### Incremental dumps + +An incremental dump reads the existing `metadata.json`, determines the +`max_vid` for each table, and queries only rows with `vid > max_vid`. New +rows are written to new chunk files (e.g. `chunk_000001.parquet`) and the +metadata is updated atomically (write to a temp file, then rename). + +### Atomicity + +The `metadata.json` file is always written atomically: the dump writes to +`metadata.json.tmp` first, then renames it to `metadata.json`. This +ensures that a reader never sees a partially-written metadata file. If the +dump process crashes mid-write, the previous `metadata.json` remains +intact. The Parquet chunk files are written before `metadata.json` is +updated, so chunk files referenced by `metadata.json` are always complete. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index b50fd598ca5..7a601e32e64 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -10,10 +10,15 @@ those. ## JSON-RPC configuration for EVM chains +> **Note**: Many of these settings can now be overridden per-chain in the TOML +> configuration file under `[chains.]`. When a per-chain value is set in +> the config file it takes precedence over the environment variable. See +> [config.md](config.md#configuring-chains) for details. + - `ETHEREUM_REORG_THRESHOLD`: Maximum expected reorg size, if a larger reorg happens, subgraphs might process inconsistent data. Defaults to 250. - `ETHEREUM_POLLING_INTERVAL`: how often to poll Ethereum for new blocks (in ms, - defaults to 500ms) + defaults to 1000ms) - `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE`: The ideal amount of triggers to be processed in a batch. If this is too small it may cause too many requests to the ethereum node, if it is too large it may cause unreasonably expensive @@ -53,6 +58,13 @@ those. be used if the store uses more than one shard. - `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER`: Specify genesis block number. If the flag is not set, the default value will be `0`. +- `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS`: Maximum number of contracts to query in a single `eth_getLogs` request. + Defaults to 2000. + +## Firehose configuration + +- `GRAPH_NODE_FIREHOSE_MAX_DECODE_SIZE`: Maximum size of a message that can be + decoded by the firehose. Defaults to 25MB. ## Running mapping handlers @@ -79,8 +91,17 @@ those. may use (in bytes, defaults to 256MB). - `GRAPH_MAX_IPFS_CACHE_SIZE`: maximum number of files cached (defaults to 50). - `GRAPH_MAX_IPFS_CACHE_FILE_SIZE`: maximum size of each cached file (in bytes, defaults to 1MiB). -- `GRAPH_IPFS_REQUEST_LIMIT`: Limits the number of requests per second to IPFS for file data sources. - Defaults to 100. +- `GRAPH_IPFS_REQUEST_LIMIT`: Limits the number of requests per second to IPFS for file data sources. Defaults to 100. +- `GRAPH_IPFS_MAX_ATTEMPTS`: This limits the IPFS retry requests in case of a + file not found or logical issue working as a safety mechanism to + prevent infinite spamming of IPFS servers and network congestion + (default: 100 000). +- `GRAPH_IPFS_CACHE_LOCATION`: When set, files retrieved from IPFS will be + cached in that location; future accesses to the same file will be served + from cache rather than IPFS. This can either be a URL starting with + `redis://`, in which case there must be a Redis instance running at that + URL, or an absolute file system path which must be a directory writable + by the `graph-node` process (experimental) ## GraphQL @@ -105,18 +126,10 @@ those. result is checked while the response is being constructed, so that execution does not take more memory than what is configured. The default value for both is unlimited. -- `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`: maximum number of GraphQL - operations per WebSocket connection. Any operation created after the limit - will return an error to the client. Default: 1000. - `GRAPH_GRAPHQL_HTTP_PORT` : Port for the GraphQL HTTP server -- `GRAPH_GRAPHQL_WS_PORT` : Port for the GraphQL WebSocket server - `GRAPH_SQL_STATEMENT_TIMEOUT`: the maximum number of seconds an individual SQL query is allowed to take during GraphQL execution. Default: unlimited -- `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`: disables the internal - mechanism that is used to trigger updates on GraphQL subscriptions. When - this variable is set to any value, `graph-node` will still accept GraphQL - subscriptions, but they won't receive any updates. - `ENABLE_GRAPHQL_VALIDATIONS`: enables GraphQL validations, based on the GraphQL specification. This will validate and ensure every query executes follows the execution rules. Default: `false` @@ -142,6 +155,10 @@ those. - `GRAPH_QUERY_CACHE_BLOCKS`: How many recent blocks per network should be kept in the query cache. This should be kept small since the lookup time and the cache memory usage are proportional to this value. Set to 0 to disable the cache. Defaults to 1. - `GRAPH_QUERY_CACHE_MAX_MEM`: Maximum total memory to be used by the query cache, in MB. The total amount of memory used for caching will be twice this value - once for recent blocks, divided evenly among the `GRAPH_QUERY_CACHE_BLOCKS`, and once for frequent queries against older blocks. The default is plenty for most loads, particularly if `GRAPH_QUERY_CACHE_BLOCKS` is kept small. Defaults to 1000, which corresponds to 1GB. - `GRAPH_QUERY_CACHE_STALE_PERIOD`: Number of queries after which a cache entry can be considered stale. Defaults to 100. +- `GRAPH_QUERY_CACHE_MAX_ENTRY_RATIO`: Limits the maximum size of a cache + entry. Query results larger than the size of a cache shard divided by this + value will not be cached. The default is 3. A value of 0 means that there + is no limit on the size of a cache entry. ## Miscellaneous @@ -174,11 +191,10 @@ those. query, and the `query_id` of the GraphQL query that caused the SQL query. These SQL queries are marked with `component: GraphQlRunner` There are additional SQL queries that get logged when `sql` is given. These are - queries caused by mappings when processing blocks for a subgraph, and - queries caused by subscriptions. If `cache` is present in addition to - `gql`, also logs information for each toplevel GraphQL query field - whether that could be retrieved from cache or not. Defaults to no - logging. + queries caused by mappings when processing blocks for a subgraph. If + `cache` is present in addition to `gql`, also logs information for each + toplevel GraphQL query field whether that could be retrieved from cache + or not. Defaults to no logging. - `GRAPH_LOG_TIME_FORMAT`: Custom log time format.Default value is `%b %d %H:%M:%S%.3f`. More information [here](https://docs.rs/chrono/latest/chrono/#formatting-and-parsing). - `STORE_CONNECTION_POOL_SIZE`: How many simultaneous connections to allow to the store. Due to implementation details, this value may not be strictly adhered to. Defaults to 10. @@ -205,6 +221,14 @@ those. decisions. Set to `true` to turn simulation on, defaults to `false` - `GRAPH_STORE_CONNECTION_TIMEOUT`: How long to wait to connect to a database before assuming the database is down in ms. Defaults to 5000ms. +- `GRAPH_STORE_SETUP_TIMEOUT`: Timeout for database setup operations + (migrations, schema creation) in milliseconds. Defaults to 30000ms (30s). + Setup operations can legitimately take longer than normal runtime operations. +- `GRAPH_STORE_CONNECTION_UNAVAILABLE_RETRY`: When a database shard is marked + unavailable due to connection timeouts, this controls how often to allow a + single probe request through to check if the database has recovered. Only one + request per interval will attempt a connection; all others fail instantly. + Value is in seconds and defaults to 2s. - `EXPERIMENTAL_SUBGRAPH_VERSION_SWITCHING_MODE`: default is `instant`, set to `synced` to only switch a named subgraph to a new deployment once it has synced, making the new deployment the "Pending" version. @@ -221,6 +245,18 @@ those. copying or grafting should take. This limits how long transactions for such long running operations will be, and therefore helps control bloat in other tables. Value is in seconds and defaults to 180s. +- `GRAPH_STORE_BATCH_TIMEOUT`: How long a batch operation during copying, + grafting, or pruning is allowed to take at most. This is meant to guard + against batches that are catastrophically big and should be set to a + small multiple of `GRAPH_STORE_BATCH_TARGET_DURATION`, like 10 times that + value, and needs to be at least 2 times that value when set. If this + timeout is hit, the batch size is reset to 1 so we can be sure that + batches stay below `GRAPH_STORE_BATCH_TARGET_DURATION` and the smaller + batch is retried. Value is in seconds and defaults to unlimited. +- `GRAPH_STORE_BATCH_WORKERS`: The number of workers to use for batch + operations. If there are idle connectiosn, each subgraph copy operation + will use up to this many workers to copy tables in parallel. Defaults + to 1 and must be at least 1 - `GRAPH_START_BLOCK`: block hash:block number where the forked subgraph will start indexing at. - `GRAPH_FORK_BASE`: api url for where the graph node will fork from, use `https://api.thegraph.com/subgraphs/id/` for the hosted service. @@ -247,3 +283,87 @@ those. - `GRAPH_STORE_WRITE_BATCH_SIZE`: how many changes to accumulate during syncing in kilobytes before a write has to happen. The default is 10_000 which corresponds to 10MB. Setting this to 0 disables write batching. +- `GRAPH_MIN_HISTORY_BLOCKS`: Specifies the minimum number of blocks to + retain for subgraphs with historyBlocks set to auto. The default value is 2 times the reorg threshold. +- `GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT`: Timeout for checking + `eth_getBlockReceipts` support during chain startup, if this times out + individual transaction receipts will be fetched instead. Defaults to 10s. +- `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION`: During the coping of a subgraph + postponing creation of certain indexes (btree, attribute based ones), would + speed up syncing +- `GRAPH_STORE_INSERT_EXTRA_COLS`: Makes it possible to work around bugs in + the subgraph writing code that manifest as Postgres errors saying 'number + of parameters must be between 0 and 65535' Such errors are always + graph-node bugs, but since it is hard to work around them, setting this + variable to something like 10 makes it possible to work around such a bug + while it is being fixed (default: 0) +- `GRAPH_ENABLE_SQL_QUERIES`: Enable the experimental [SQL query + interface](implementation/sql-interface.md). + (default: false) +- `GRAPH_STORE_ACCOUNT_LIKE_SCAN_INTERVAL_HOURS`: If set, enables an experimental job that + periodically scans for entity tables that may benefit from an [account-like optimization](https://thegraph.com/docs/en/indexing/tooling/graph-node/#account-like-optimisation) and marks them with an + account-like flag. The value is the interval in hours at which the job + should run. The job reads data from the `info.table_stats` materialized view, which refreshes every six hours. + Expects an integer value, e.g., 24. Requires also setting + `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT` and `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`. +- `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT`: Sets the minimum total number of versions a table must have + to be considered for account-like flagging. Expects a positive integer value. No default value. +- `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`: Sets the maximum unique entities to version ratio + (e.g., 0.01 ≈ 1:100 entity-to-version ratio). +- `GRAPH_STORE_DISABLE_CALL_CACHE`: Disables storing or reading `eth_call` results from the store call cache. + This option may be useful for indexers who are running their own RPC nodes. + Disabling the store call cache may significantly impact performance; the actual impact depends on + the average execution time of an `eth_call` compared to the cost of a database lookup for a cached result. + (default: false) + +## Log Store Configuration + +`graph-node` supports storing and querying subgraph logs through multiple backends: Elasticsearch, Loki, local files, or disabled. + +**For complete log store documentation**, including detailed configuration, querying examples, and choosing the right backend, see the **[Log Store Guide](log-store.md)**. + +### Quick Reference + +**Backend selection:** +- `GRAPH_LOG_STORE_BACKEND`: `disabled` (default), `elasticsearch`, `loki`, or `file` + +**Elasticsearch:** +- `GRAPH_LOG_STORE_ELASTICSEARCH_URL`: Elasticsearch endpoint URL (required) +- `GRAPH_LOG_STORE_ELASTICSEARCH_USER`: Username (optional) +- `GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD`: Password (optional) +- `GRAPH_LOG_STORE_ELASTICSEARCH_INDEX`: Index name (default: `subgraph`) + +**Loki:** +- `GRAPH_LOG_STORE_LOKI_URL`: Loki endpoint URL (required) +- `GRAPH_LOG_STORE_LOKI_TENANT_ID`: Tenant ID (optional) + +**File-based:** +- `GRAPH_LOG_STORE_FILE_DIR`: Log directory (required) +- `GRAPH_LOG_STORE_FILE_MAX_SIZE`: Max file size in bytes (default: 104857600 = 100MB) +- `GRAPH_LOG_STORE_FILE_RETENTION_DAYS`: Retention period (default: 30) + +**Deprecated variables** (will be removed in future versions): +- `GRAPH_ELASTICSEARCH_URL` → use `GRAPH_LOG_STORE_ELASTICSEARCH_URL` +- `GRAPH_ELASTICSEARCH_USER` → use `GRAPH_LOG_STORE_ELASTICSEARCH_USER` +- `GRAPH_ELASTICSEARCH_PASSWORD` → use `GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD` +- `GRAPH_ELASTIC_SEARCH_INDEX` → use `GRAPH_LOG_STORE_ELASTICSEARCH_INDEX` + +### Example: File-based Logs for Local Development + +```bash +mkdir -p ./graph-logs +export GRAPH_LOG_STORE_BACKEND=file +export GRAPH_LOG_STORE_FILE_DIR=./graph-logs + +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 +``` + +See the **[Log Store Guide](log-store.md)** for: +- Detailed configuration for all backends +- How log stores work internally +- GraphQL query examples +- Choosing the right backend for your use case +- Best practices and troubleshooting diff --git a/docs/getting-started.md b/docs/getting-started.md deleted file mode 100644 index 13c2c8786d4..00000000000 --- a/docs/getting-started.md +++ /dev/null @@ -1,486 +0,0 @@ -# Getting Started -> **Note:** This project is heavily a WIP, and until it reaches v1.0, the API is subject to change in breaking ways without notice. - -## 0 Introduction - -This page explains everything you need to know to run a local Graph Node, including links to other reference pages. First, we describe what The Graph is and then explain how to get started. - -### 0.1 What Is The Graph? - -The Graph is a decentralized protocol for indexing and querying data from blockchains, which makes it possible to query for data that is difficult or impossible to do directly. Currently, we only work with Ethereum. - -For example, with the popular Cryptokitties decentralized application (dApp) that implements the [ERC-721 Non-Fungible Token (NFT)](https://github.com/ethereum/eips/issues/721) standard, it is relatively straightforward to ask the following questions: -> *How many cryptokitties does a specific Ethereum account own?* -> *When was a particular cryptokitty born?* - -These read patterns are directly supported by the methods exposed by the [contract](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyCore.sol): the [`balanceOf`](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyOwnership.sol#L64) and [`getKitty`](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyCore.sol#L91) methods for these two examples. - -However, other questions are more difficult to answer: -> *Who are the owners of the cryptokitties born between January and February of 2018?* - -To answer this question, you need to process all [`Birth` events](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyBase.sol#L15) and then call the [`ownerOf` method](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyOwnership.sol#L144) for each cryptokitty born. An alternate approach could involve processing all (`Transfer` events) and filtering based on the most recent transfer for each cryptokitty. - -Even for this relatively simple question, it would take hours or even days for a dApp running in a browser to find an answer. Indexing and caching data off blockchains is hard. There are also edge cases around finality, chain reorganizations, uncled blocks, etc., which make it even more difficult to display deterministic data to the end user. - -The Graph solves this issue by providing an open source node implementation, [Graph Node](../README.md), which handles indexing and caching of blockchain data. The entire community can contribute to and utilize this tool. In the current implementation, it exposes functionality through a GraphQL API for end users. - -### 0.2 How Does It Work? - -The Graph must be run alongside a running IPFS node, Ethereum node, and a store (Postgres, in this initial implementation). - -![Data Flow Diagram](images/TheGraph_DataFlowDiagram.png) - -The high-level dataflow for a dApp using The Graph is as follows: -1. The dApp creates/modifies data on Ethereum through a transaction to a smart contract. -2. The smart contract emits one or more events (logs) while processing this transaction. -3. The Graph Node listens for specific events and triggers handlers in a user-defined mapping. -4. The mapping is a WASM module that runs in a WASM runtime. It creates one or more store transactions in response to Ethereum events. -5. The store is updated along with the indexes. -6. The dApp queries the Graph Node for data indexed from the blockchain using the node's [GraphQL endpoint](https://graphql.org/learn/). The Graph Node, in turn, translates the GraphQL queries into queries for its underlying store to fetch this data. This makes use of the store's indexing capabilities. -7. The dApp displays this data in a user-friendly format, which an end-user leverages when making new transactions against the Ethereum blockchain. -8. And, this cycle repeats. - -### 0.3 What's Needed to Build a Graph Node? -Three repositories are relevant to building on The Graph: -1. [Graph Node](../README.md) – A server implementation for indexing, caching, and serving queries against data from Ethereum. -2. [Graph CLI](https://github.com/graphprotocol/graph-tooling/tree/main/packages/cli) – A CLI for building and compiling projects that are deployed to the Graph Node. -3. [Graph TypeScript Library](https://github.com/graphprotocol/graph-tooling/tree/main/packages/ts) – TypeScript/AssemblyScript library for writing subgraph mappings to be deployed to The Graph. - -### 0.4 Getting Started Overview -Below, we outline the required steps to build a subgraph from scratch, which will serve queries from a GraphQL endpoint. The three major steps are: - -1. [Define the subgraph](#1-define-the-subgraph) - 1. [Define the data sources and create a manifest](#11-define-the-data-sources-and-create-a-manifest) - - 2. [Create the GraphQL schema](#12-create-the-graphql-schema-for-the-data-source) - - 3. [Create a subgraph project and generate types](#13-create-a-subgraph-project-and-generate-types) - - 4. [Write the mappings](#14-writing-mappings) -2. Deploy the subgraph - 1. [Start up an IPFS node](#21-start-up-ipfs) - - 2. [Create the Postgres database](#22-create-the-postgres-db) - - 3. [Start the Graph Node and Connect to an Ethereum node](#23-starting-the-graph-node-and-connecting-to-an-ethereum-node) - - 4. [Deploy the subgraph](#24-deploying-the-subgraph) -3. Query the subgraph - 1. [Query the newly deployed GraphQL API](#3-query-the-local-graph-node) - -Now, let's dig in! - -## 1 Define the Subgraph -When we refer to a subgraph, we reference the entire project that is indexing a chosen set of data. - -To start, create a repository for this project. - -### 1.1 Define the Data Sources and Create a Manifest - -When building a subgraph, you must first decide what blockchain data you want the Graph Node to index. These are known as `dataSources`, which are datasets derived from a blockchain, i.e., an Ethereum smart contract. - -The subgraph is defined by a YAML file known as the **subgraph manifest**. This file should always be named `subgraph.yaml`. View the full specification for the subgraph manifest [here](subgraph-manifest.md). It contains a schema, data sources, and mappings that are used to deploy the GraphQL endpoint. - -Let's go through an example to display what a subgraph manifest looks like. In this case, we use the common ERC721 contract and look at the `Transfer` event because it is familiar to many developers. Below, we define a subgraph manifest with one contract under `dataSources`, which is a smart contract implementing the ERC721 interface: -```yaml -specVersion: 0.0.1 -description: ERC-721 Example -repository: https://github.com//erc721-example -schema: - file: ./schema.graphql -dataSources: -- kind: ethereum/contract - name: MyERC721Contract - network: mainnet - source: - address: "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" - abi: ERC721 - mapping: - kind: ethereum/events - apiVersion: 0.0.1 - language: wasm/assemblyscript - entities: - - Token - abis: - - name: ERC721 - file: ./abis/ERC721ABI.json - eventHandlers: - - event: Transfer(address,address,uint256) - handler: handleTransfer - file: ./mapping.ts -``` -We point out a few important facts from this example to supplement the [subgraph manifest spec](subgraph-manifest.md): - -* The name `ERC721` under `source > abi` must match the name displayed underneath `abis > name`. -* The event `Transfer(address,address,uint256)` under `eventHandlers` must match what is in the ABI. The name `handleTransfer` under `eventHandlers > handler` must match the name of the mapping function, which we explain in section 1.4. -* Ensure that you have the correct contract address under `source > address`. This is also the case when indexing testnet contracts as well because you might switch back and forth. -* You can define multiple data sources under dataSources. Within a datasource, you can also have multiple `entities` and `events`. See [this subgraph](https://github.com/graphprotocol/decentraland-subgraph/blob/master/subgraph.yaml) for an example. -* If at any point the Graph CLI outputs 'Failed to copy subgraph files', it probably means you have a typo in the manifest. - -#### 1.1.1 Obtain the Contract ABIs -The ABI JSON file must contain the correct ABI to source all the events or any contract state you wish to ingest into the Graph Node. There are a few ways to obtain an ABI for the contract: -* If you are building your own project, you likely have access to your most current ABIs of your smart contracts. -* If you are building a subgraph for a public project, you can download that project to your computer and generate the ABI by using [`truffle compile`](https://truffleframework.com/docs/truffle/overview) or `solc` to compile. This creates the ABI files that you can then transfer to your subgraph `/abi` folder. -* Sometimes, you can also find the ABI on [Etherscan](https://etherscan.io), but this is not always reliable because the uploaded ABI may be out of date. Make sure you have the correct ABI. Otherwise, you will not be able to start a Graph Node. - -If you run into trouble here, double-check the ABI and ensure that the event signatures exist *exactly* as you expect them by examining the smart contract code you are sourcing. Also, note with the ABI, you only need the array for the ABI. Compiling the contracts locally results in a `.json` file that contains the complete ABI nested within the `.json` file under the key `abi`. - -An example `abi` for the `Transfer` event is shown below and would be stored in the `/abi` folder with the name `ERC721ABI.json`: - -```json - [{ - "anonymous": false, - "inputs": [ - { - "indexed": true, - "name": "_from", - "type": "address" - }, - { - "indexed": true, - "name": "_to", - "type": "address" - }, - { - "indexed": true, - "name": "_tokenId", - "type": "uint256" - } - ], - "name": "Transfer", - "type": "event" - }] - ``` - -Once you create this `subgraph.yaml` file, move to the next section. - -### 1.2 Create the GraphQL Schema for the Data Source -GraphQL schemas are defined using the GraphQL interface definition language (IDL). If you have never written a GraphQL schema, we recommend checking out a [quick primer](https://graphql.org/learn/schema/#type-language) on the GraphQL type system. - -With The Graph, rather than defining the top-level `Query` type, you simply define entity types. Then, the Graph Node will generate top-level fields for querying single instances and collections of that entity type. Each entity type is required to be annotated with an `@entity` directive. - -As you see in the example `subgraph.yaml` manifest above, it contains one entity named `Token`. Let's define what that would look like for the GraphQL schema: - -Define a Token entity type: -```graphql -type Token @entity { - id: ID! - currentOwner: Address! -} -``` - -This `entity` tracks a single ERC721 token on Ethereum by its ID and the current owner. The **`ID` field is required** and stores values of the ID type, which are strings. The `ID` must be a unique value so that it can be placed into the store. For an ERC721 token, the unique ID could be the token ID because that value is unique to that coin. - -The exclamation mark represents the fact that that field must be set when the entity is stored in the database, i.e., it cannot be `null`. See the [Schema API](https://github.com/graphprotocol/docs/blob/main/pages/en/querying/graphql-api.mdx#schema) for a complete reference on defining the schema for The Graph. - -When you complete the schema, add its path to the top-level `schema` key in the subgraph manifest. See the code below for an example: - -```yaml -specVersion: 0.0.1 -schema: - file: ./schema.graphql -``` - -### 1.3 Create a Subgraph Project and Generate Types -Once you have the `subgraph.yaml` manifest and the `./schema.graphql` file, you are ready to use the Graph CLI to set up the subgraph directory. The Graph CLI is a command-line tool that contains helpful commands for deploying the subgraphs. Before continuing with this guide, please go to the [Graph CLI README](https://github.com/graphprotocol/graph-tooling/tree/main/packages/cli) and follow the instructions up to Step 7 for setting up the subgraph directory. - -Once you run `yarn codegen` as outlined in the [Graph CLI README](https://github.com/graphprotocol/graph-tooling/tree/main/packages/cli), you are ready to create the mappings. - -`yarn codegen` looks at the contract ABIs defined in the subgraph manifest and generates TypeScript classes for the smart contracts the mappings script will interface with, which includes the types of public methods and events. In reality, the classes are AssemblyScript but more on that later. - -Classes are also generated based on the types defined in the GraphQL schema. These generated classes are incredibly useful for writing correct mappings. This allows you to autocomplete Ethererum events as well as improve developer productivity using the TypeScript language support in your favorite editor or IDE. - -### 1.4 Write the Mappings - -The mappings that you write will perform transformations on the Ethereum data you are sourcing, and it will dictate how this data is loaded into the Graph Node. Mappings can be very simple but can become complex. It depends on how much abstraction you want between the data and the underlying Ethereum contract. - -Mappings are written in a subset of TypeScript called AssemblyScript, which can be compiled down to WASM. AssemblyScript is stricter than normal TypeScript but follows the same backbone. A few TypeScript/JavaScript features that are not supported in AssemblyScript include plain old Javascript objects (POJOs), untyped arrays, untyped maps, union types, the `any` type, and variadic functions. In addition, `switch` statements also work differently. See the [AssemblyScript wiki](https://github.com/AssemblyScript/assemblyscript/wiki) for a full reference on AssemblyScript features. - -In the mapping file, create export functions named after the event handlers in the subgraph manifest. Each handler should accept a single parameter called `event` with a type corresponding to the name of the event that is being handled. This type was generated for you in the previous step, 1.3. - -```typescript -export function handleTransfer(event: Transfer): void { - // Event handler logic goes here -} -``` - -As mentioned, AssemblyScript does not have untyped maps or POJOs, so classes are generated to represent the types defined in the GraphQL schema. The generated type classes handle property type conversions for you, so AssemblyScript's requirement of strictly typed functions is satisfied without the extra work of converting each property explicitly. - -Let's look at an example. Continuing with our previous token example, let's write a mapping that tracks the owner of a particular ERC721 token. - -```typescript - -// This is an example event type generated by `graph-cli` -// from an Ethereum smart contract ABI -import { Transfer } from './types/abis/SomeContract' - -// This is an example of an entity type generated from a -// subgraph's GraphQL schema -import { Token } from './types/schema' - -export function handleTransfer(event: Transfer): void { - let tokenID = event.params.tokenID.toHex() - let token = new Token(tokenID) - token.currentOwner = event.params.to - - token.save() -} -``` -A few things to note from this code: -* We create a new entity named `token`, which is stored in the Graph Node database. -* We create an ID for that token, which must be unique, and then create an entity with `new Token(tokenID)`. We get the token ID from the event emitted by Ethereum, which was turned into an AssemblyScript type by the [Graph TypeScript Library](https://github.com/graphprotocol/graph-tooling/tree/main/packages/ts). We access it at `event.params.tokenId`. Note that you must set `ID` as a string and call `toHex()` on the `tokenID` to turn it into a hex string. -* This entity is updated by the `Transfer` event emitted by the ERC721 contract. -* The current owner is gathered from the event with `event.params.to`. It is set as an Address by the Token class. -* Event handlers functions always return `void`. -* `token.save()` is used to set the Token entity. `.save()` comes from `graph-ts` just like the entity type (`Token` in this example). It is used for setting the value(s) of a particular entity's attribute(s) in the store. There is also a `.load()` function, which will be explained in 1.4.1. - -#### 1.4.1 Use the `save`, `load`, and `remove` entity functions - -The only way that entities may be added to The Graph is by calling `.save()`, which may be called multiple times in an event handler. `.save()` will only set the entity attributes that have explicitly been set on the `entity`. Attributes that are not explicitly set or are unset by calling `Entity.unset()` will not be overwritten. This means you can safely update one field of an entity and not worry about overwriting other fields not referenced in the mapping. - -The definition for `.save()` is: - -```typescript -entity.save() // Entity is representative of the entity type being updated. In our example above, it is Token. -``` - - `.load()` expects the entity type and ID of the entity. Use `.load()` to retrieve information previously added with `.save()`. - -The definition for `.load()` is: - - ```typescript -entity.load() // Entity is representative of the entity type being updated. In our example above, it is Token. -``` - -Once again, all these functions come from the [Graph TypeScript Library](https://github.com/graphprotocol/graph-tooling/tree/main/packages/ts). - -Let's look at the ERC721 token as an example for using `token.load()`. Above, we showed how to use `token.save()`. Now, let's consider that you have another event handler that needs to retrieve the currentOwner of an ERC721 token. To do this within an event handler, you would write the following: - -```typescript - let token = token.load(tokenID.toHex()) - if (token !== null) { - let owner = token.currentOwner - } -``` - -You now have the `owner` data, and you can use that in the mapping to set the owner value to a new entity. - -There is also `.remove()`, which allows you to erase an entry that exists in the store. You simply pass the entity and ID: - -```typescript -entity.remove(ID) -``` - -#### 1.4.2 Call into the Contract Storage to Get Data - -You can also obtain data that is stored in one of the included ABI contracts. Any state variable that is marked `public` or any `view` function can be accessed. Below shows how you obtain the token -symbol of an ERC721 token, which is a state variable of the smart contract. You would add this inside of the event handler function. - -```typescript - let tokenContract = ERC721.bind(event.address); - let tokenSymbol = tokenContract.symbol(); -``` - -Note, we are using an ERC721 class generated from the ABI, which we call bind on. This is gathered from the subgraph manifest here: -```yaml - source: - address: "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" - abi: ERC721 -``` - -The class is imported from the ABI's TypeScript file generated via `yarn codegen`. - -## 2 Deploy the Subgraph - -### 2.1 Start Up an IPFS Node -To deploy the subgraph to the Graph Node, the subgraph will first need to be built and stored on IPFS, along with all linked files. - -To run an IPFS daemon locally, execute the following: -1. Download and install IPFS. -2. Run `ipfs init`. -3. Run `ipfs daemon`. - -If you encounter problems, follow the instructions from the [IPFS website](https://ipfs.io/docs/getting-started/). - -To confirm the subgraph is stored on IPFS, pass that subgraph ID into `ipfs cat` to view the subgraph manifest with file paths replaced by IPLD links. - -### 2.2 Create the Postgres database - -Ensure that you have Postgres installed. Navigate to a location where you want to save the `.postgres` folder. The desktop is fine since this folder can be used for many different subgraphs. Then, run the following commands: - -``` -initdb -D .postgres -pg_ctl -D .postgres -l logfile start -createdb -``` -Name the database something relevant to the project so that you always know how to access it. - -### 2.3 Start the Graph Node and Connect to an Ethereum Node - -When you start the Graph Node, you need to specify which Ethereum network it should connect to. There are three common ways to do this: - * Infura - * A local Ethereum node - * Ganache - -The Ethereum Network (Mainnet, Ropsten, Rinkeby, etc.) must be passed as a flag in the command that starts the Graph Node as laid out in the following subsections. - -#### 2.3.1 Infura - -[Infura](https://infura.io/) is supported and is the simplest way to connect to an Ethereum node because you do not have to set up your own geth or parity node. However, it does sync slower than being connected to your own node. The following flags are passed to start the Graph Node and indicate you want to use Infura: - -```sh -cargo run -p graph-node --release -- \ - --postgres-url postgresql://<:PASSWORD>@localhost:5432/ \ - --ethereum-rpc :https://mainnet.infura.io \ - --ipfs 127.0.0.1:5001 \ - --debug -``` - -Also, note that the Postgres database may not have a password at all. If that is the case, the Postgres connection URL can be passed as follows: - -` --postgres-url postgresql://@localhost:5432/ \ ` - -#### 2.3.2 Local Geth or Parity Node - -This is the speediest way to get mainnet or testnet data. The problem is that if you do not already have a synced [geth](https://geth.ethereum.org/docs/getting-started) or [parity](https://github.com/paritytech/parity-ethereum) node, you will have to sync one, which takes a very long time and takes up a lot of space. Additionally, note that geth `fast sync` works. So, if you are starting from scratch, this is the fastest way to get caught up, but expect at least 12 hours of syncing on a modern laptop with a good internet connection to sync geth. Normal mode geth or parity will take much longer. Use the following geth command to start syncing: - -`geth --syncmode "fast" --rpc --ws --wsorigins="*" --rpcvhosts="*" --cache 1024` - -Once you have the local node fully synced, run the following command: - -```sh -cargo run -p graph-node --release -- \ - --postgres-url postgresql://<:PASSWORD>@localhost:5432/ \ - --ethereum-rpc :127.0.0.1:8545 \ - --ipfs 127.0.0.1:5001 \ - --debug -``` - -This assumes the local node is on the default `8545` port. If you are on a different port, change it. - -Switching back and forth between sourcing data from Infura and your own local nodes is fine. The Graph Node picks up where it left off. - -#### 2.3.3 Ganache - -**IMPORTANT: Ganache fixed the [issue](https://github.com/trufflesuite/ganache/issues/907) that prevented things from working properly. However, it did not release the new version. Follow the steps in this [issue](https://github.com/graphprotocol/graph-node/issues/375) to run the fixed version locally.** - -[Ganache](https://github.com/trufflesuite/ganache-cli) can be used as well and is preferable for quick testing. This might be an option if you are simply testing out the contracts for quick iterations. Of course, if you close Ganache, then the Graph Node will no longer have any data to source. Ganache is best for short-term projects such as hackathons. Also, it is useful for testing to see that the schema and mappings are working properly before working on the mainnet. - -You can connect the Graph Node to Ganache the same way you connected to a local geth or parity node in the previous section, 2.3.2. Note, however, that Ganache normally runs on port `9545` instead of `8545`. - -#### 2.3.4 Local Parity Testnet - -To set up a local testnet that will allow you to rapidly test the project, download the parity software if you do not already have it. - -This command will work for a one-line install: - -`bash <(curl https://get.parity.io -L)` - -Next, you want to make an account that you can unlock and make transactions on for the parity dev chain. Run the following command: - -`parity account new --chain dev` - -Create a password that you will remember. Take note of the account that gets output. Now, you also have to make that password a text file and pass it into the next command. The desktop is a good location for it. If the password is `123`, only put the numbers in the text file. Do not include any quotes. - -Then, run this command: - -`parity --config dev --unsafe-expose --jsonrpc-cors="all" --unlock --password ~/Desktop/password.txt` - -The chain should start and will be accessible by default on `localhost:8545`. It is a chain with 0 block time and instant transactions, making testing very fast. Passing `unsafe-expose` and `--jsonrpc-cors="all"` as flags allows MetaMask to connect. The `unlock` flag gives parity the ability to send transactions with that account. You can also import the account to MetaMask, which allows you to interact with the test chain directly in your browser. With MetaMask, you need to import the account with the private testnet Ether. The base account that the normal configuration of parity gives you is -`0x00a329c0648769A73afAc7F9381E08FB43dBEA72`. - -The private key is: -``` -4d5db4107d237df6a3d58ee5f70ae63d73d7658d4026f2eefd2f204c81682cb7 (note this is the private key given along with the parity dev chain, so it is okay to share) -``` -Use MetaMask ---> import account ---> private key. - -All the extra information for customization of a parity dev chain is located [here](https://wiki.parity.io/Private-development-chain#customizing-the-development-chain). - -You now have an Ethereum account with a ton of Ether and should be able to set up the migrations on this network and use Truffle. Now, send some Ether to the previous account that was created and unlocked. This way, you can run `truffle migrate` with this account. - -#### 2.3.5 Syncing with a Public Testnet - -If you want to sync using a public testnet such as Kovan, Rinkeby, or Ropsten, just make sure the local node is a testnet node or that you are hitting the correct Infura testnet endpoint. - -### 2.4 Deploy the Subgraph - -When you deploy the subgraph to the Graph Node, it will start ingesting all the subgraph events from the blockchain, transforming that data with the subgraph mappings and storing it in the Graph Node. Note that a running subgraph can safely be stopped and restarted, picking up where it left off. - -Now that the infrastructure is set up, you can run `yarn create-subgraph` and then `yarn deploy` in the subgraph directory. These commands should have been added to `package.json` in section 1.3 when we took a moment to go through the set up for [Graph CLI documentation](https://github.com/graphprotocol/graph-tooling/tree/main/packages/cli). This builds the subgraph and creates the WASM files in the `dist/` folder. Next, it uploads the `dist/ -` files to IPFS and deploys it to the Graph Node. The subgraph is now fully running. - -The `watch` flag allows the subgraph to continually restart every time you save an update to the `manifest`, `schema`, or `mappings`. If you are making many edits or have a subgraph that has been syncing for a few hours, leave this flag off. - -Depending on how many events have been emitted by your smart contracts, it could take less than a minute to get fully caught up. If it is a large contract, it could take hours. For example, ENS takes about 12 to 14 hours to register every single ENS domain. - -## 3 Query the Local Graph Node -With the subgraph deployed to the locally running Graph Node, visit http://127.0.0.1:8000/ to open up a [GraphiQL](https://github.com/graphql/graphiql) interface where you can explore the deployed GraphQL API for the subgraph by issuing queries and viewing the schema. - -We provide a few simple examples below, but please see the [Query API](https://github.com/graphprotocol/docs/blob/main/pages/en/querying/graphql-api.mdx#queries) for a complete reference on how to query the subgraph's entities. - -Query the `Token` entities: -```graphql -{ - tokens(first: 100) { - id - currentOwner - } -} -``` -Notice that `tokens` is plural and that it will return at most 100 entities. - -Later, when you have deployed the subgraph with this entity, you can query for a specific value, such as the token ID: - -```graphql -{ - token(first: 100, id: "c2dac230ed4ced84ad0ca5dfb3ff8592d59cef7ff2983450113d74a47a12") { - currentOwner - } -} -``` - -You can also sort, filter, or paginate query results. The query below would organize all tokens by their ID and return the current owner of each token. - -```graphql -{ - tokens(first: 100, orderBy: id) { - currentOwner - } -} -``` - -GraphQL provides a ton of functionality. Once again, check out the [Query API](graphql-api.md#1-queries) to find out how to use all supported query features. - -## 4 Changing the Schema, Mappings, and Manifest, and Launching a New Subgraph - -When you first start building the subgraph, it is likely that you will make a few changes to the manifest, mappings, or schema. If you update any of them, rerun `yarn codegen` and `yarn deploy`. This will post the new files on IPFS and deploy the new subgraph. Note that the Graph Node can track multiple subgraphs, so you can do this as many times as you like. - -## 5 Common Patterns for Building Subgraphs - -### 5.1 Removing Elements of an Array in a Subgraph - -Using the AssemblyScript built-in functions for arrays is the way to go. Find the source code [here](https://github.com/AssemblyScript/assemblyscript/blob/18826798074c9fb02243dff76b1a938570a8eda7/std/assembly/array.ts). Using `.indexOf()` to find the element and then using `.splice()` is one way to do so. See this [file](https://github.com/graphprotocol/aragon-subgraph/blob/master/individual-dao-subgraph/mappings/ACL.ts) from the Aragon subgraph for a working implementation. - -### 5.2 Getting Data from Multiple Versions of Your Contracts - -If you have launched multiple versions of your smart contracts onto Ethereum, it is very easy to source data from all of them. This simply requires you to add all versions of the contracts to the `subgraph.yaml` file and handle the events from each contract. Design your schema to consider both versions, and handle any changes to the event signatures that are emitted from each version. See the [0x Subgraph](https://github.com/graphprotocol/0x-subgraph/tree/master/src/mappings) for an implementation of multiple versions of smart contracts being ingested by a subgraph. - -## 5 Example Subgraphs - -Here is a list of current subgraphs that we have open sourced: -* https://github.com/graphprotocol/ens-subgraph -* https://github.com/graphprotocol/decentraland-subgraph -* https://github.com/graphprotocol/adchain-subgraph -* https://github.com/graphprotocol/0x-subgraph -* https://github.com/graphprotocol/aragon-subgraph -* https://github.com/graphprotocol/dharma-subgraph -* https://github.com/daostack/subgraph -* https://github.com/graphprotocol/dydx-subgraph -* https://github.com/livepeer/livepeerjs/tree/master/packages/subgraph -* https://github.com/graphprotocol/augur-subgraph - -## Contributions - -All feedback and contributions in the form of issues and pull requests are welcome! - diff --git a/docs/graphman-graphql-api.md b/docs/graphman-graphql-api.md new file mode 100644 index 00000000000..486bee6090d --- /dev/null +++ b/docs/graphman-graphql-api.md @@ -0,0 +1,213 @@ +# Graphman GraphQL API + +The graphman API provides functionality to manage various aspects of `graph-node` through GraphQL operations. It is only +started when the environment variable `GRAPHMAN_SERVER_AUTH_TOKEN` is set. The token is used to authenticate graphman +GraphQL requests. Even with the token, the server should not be exposed externally as it provides operations that an +attacker can use to severely impede the functioning of an indexer. The server listens on the port `GRAPHMAN_PORT`, port +`8050` by default. + +Environment variables to control the graphman API: + +- `GRAPHMAN_SERVER_AUTH_TOKEN` - The token is used to authenticate graphman GraphQL requests. +- `GRAPHMAN_PORT` - The port for the graphman GraphQL server (Defaults to `8050`) + +## GraphQL playground + +When the graphman GraphQL server is running the GraphQL playground is available at the following +address: http://127.0.0.1:8050 + +**Note:** The port might be different. + +Please make sure to set the authorization header to be able to use the playground: + +```json +{ + "Authorization": "Bearer GRAPHMAN_SERVER_AUTH_TOKEN" +} +``` + +**Note:** There is a headers section at the bottom of the playground page. + +## Supported commands + +The playground is the best place to see the full schema, the latest available queries and mutations, and their +documentation. Below, we will briefly describe some supported commands and example queries. + +At the time of writing, the following graphman commands are available via the GraphQL API: + +### Deployment Info + +Returns the available information about one, multiple, or all deployments. + +**Example query:** + +```text +query { + deployment { + info(deployment: { hash: "Qm..." }) { + status { + isPaused + } + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "info": [ + { + "status": { + "isPaused": false + } + } + ] + } + } +} +``` + +### Pause Deployment + +Pauses a deployment that is not already paused. + +**Example query:** + +```text +mutation { + deployment { + pause(deployment: { hash: "Qm..." }) { + success + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "pause": { + "success": true + } + } + } +} +``` + +### Resume Deployment + +Resumes a deployment that has been previously paused. + +**Example query:** + +```text +mutation { + deployment { + resume(deployment: { hash: "Qm..." }) { + success + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "resume": { + "success": true + } + } + } +} +``` + +### Restart Deployment + +Pauses a deployment and resumes it after a delay. + +**Example query:** + +```text +mutation { + deployment { + restart(deployment: { hash: "Qm..." }) { + id + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "restart": { + "id": "UNIQUE_EXECUTION_ID" + } + } + } +} +``` + +This is a long-running command because the default delay before resuming the deployment is 20 seconds. Long-running +commands are executed in the background. For long-running commands, the GraphQL API will return a unique execution ID. + +The ID can be used to query the execution status and the output of the command: + +```text +query { + execution { + info(id: "UNIQUE_EXECUTION_ID") { + status + errorMessage + } + } +} +``` + +**Example response when execution is in-progress:** + +```json +{ + "data": { + "execution": { + "info": { + "status": "RUNNING", + "errorMessage": null + } + } + } +} +``` + +**Example response when execution is completed:** + +```json +{ + "data": { + "execution": { + "info": { + "status": "SUCCEEDED", + "errorMessage": null + } + } + } +} +``` + +## Other commands + +GraphQL support for other graphman commands will be added over time, so please make sure to check the GraphQL playground +for the full schema and the latest available queries and mutations. diff --git a/docs/graphman.md b/docs/graphman.md index 31353fbabc3..8c857703dda 100644 --- a/docs/graphman.md +++ b/docs/graphman.md @@ -371,21 +371,30 @@ Inspect all blocks after block `13000000`: Remove the call cache of the specified chain. -If block numbers are not mentioned in `--from` and `--to`, then all the call cache will be removed. +Either remove entries in the range `--from` and `--to`, remove stale contracts which have not been accessed for a specified duration `--ttl_days`, or remove the entire cache with `--remove-entire-cache`. Removing the entire cache can reduce indexing performance significantly and should generally be avoided. -USAGE: - graphman chain call-cache remove [OPTIONS] + Usage: graphman chain call-cache remove [OPTIONS] -OPTIONS: - -f, --from - Starting block number + Options: + --remove-entire-cache + Remove the entire cache + + --ttl-days + Remove stale contracts based on call_meta table - -h, --help - Print help information + --ttl-max-contracts + Limit the number of contracts to consider for stale contract removal + + -f, --from + Starting block number - -t, --to + -t, --to Ending block number + -h, --help + Print help (see a summary with '-h') + + ### DESCRIPTION Remove the call cache of a specified chain. @@ -404,6 +413,15 @@ the first block number will be used as the starting block number. The `to` option is used to specify the ending block number of the block range. In the absence of `to` option, the last block number will be used as the ending block number. +#### `--remove-entire-cache` +The `--remove-entire-cache` option is used to remove the entire call cache of the specified chain. + +#### `--ttl-days ` +The `--ttl-days` option is used to remove stale contracts based on the `call_meta.accessed_at` field. For example, if `--ttl-days` is set to 7, all calls to a contract that has not been accessed in the last 7 days will be removed from the call cache. + +#### `--ttl-max-contracts ` +The `--ttl-max-contracts` option is used to limit the maximum number of contracts to be removed when using the `--ttl-days` option. For example, if `--ttl-max-contracts` is set to 100, at most 100 contracts will be removed from the call cache even if more contracts meet the TTL criteria. + ### EXAMPLES Remove the call cache for all blocks numbered from 10 to 20: @@ -412,5 +430,12 @@ Remove the call cache for all blocks numbered from 10 to 20: Remove all the call cache of the specified chain: - graphman --config config.toml chain call-cache ethereum remove + graphman --config config.toml chain call-cache ethereum remove --remove-entire-cache + +Remove stale contracts from the call cache that have not been accessed in the last 7 days: + + graphman --config config.toml chain call-cache ethereum remove --ttl-days 7 + +Remove stale contracts from the call cache that have not been accessed in the last 7 days, limiting the removal to a maximum of 100 contracts: + graphman --config config.toml chain call-cache ethereum remove --ttl-days 7 --ttl-max-contracts 100 diff --git a/docs/implementation/README.md b/docs/implementation/README.md index 31d4eb694a6..d54a39babbe 100644 --- a/docs/implementation/README.md +++ b/docs/implementation/README.md @@ -10,3 +10,4 @@ the code should go into comments. * [SQL Query Generation](./sql-query-generation.md) * [Adding support for a new chain](./add-chain.md) * [Pruning](./pruning.md) +* [Dump Format](./dump.md) diff --git a/docs/implementation/add-chain.md b/docs/implementation/add-chain.md deleted file mode 100644 index f4af3371f25..00000000000 --- a/docs/implementation/add-chain.md +++ /dev/null @@ -1,279 +0,0 @@ -# Adding support for a new chain - -## Context - -`graph-node` started as a project that could only index EVM compatible chains, eg: `ethereum`, `xdai`, etc. - -It was known from the start that with growth we would like `graph-node` to be able to index other chains like `NEAR`, `Solana`, `Cosmos`, list goes on... - -However to do it, several refactors were necessary, because the code had a great amount of assumptions based of how Ethereum works. - -At first there was a [RFC](https://github.com/graphprotocol/rfcs/blob/10aaae30fdf82f0dd2ccdf4bbecf7ec6bbfb703b/rfcs/0005-multi-blockchain-support.md) for a design overview, then actual PRs such as: - -- https://github.com/graphprotocol/graph-node/pull/2272 -- https://github.com/graphprotocol/graph-node/pull/2292 -- https://github.com/graphprotocol/graph-node/pull/2399 -- https://github.com/graphprotocol/graph-node/pull/2411 -- https://github.com/graphprotocol/graph-node/pull/2453 -- https://github.com/graphprotocol/graph-node/pull/2463 -- https://github.com/graphprotocol/graph-node/pull/2755 - -All new chains, besides the EVM compatible ones, are integrated using [StreamingFast](https://www.streamingfast.io/)'s [Firehose](https://firehose.streamingfast.io/). The integration consists of chain specific `protobuf` files with the type definitions. - -## How to do it? - -The `graph-node` repository contains multiple Rust crates in it, this section will be divided in each of them that needs to be modified/created. - -> It's important to remember that this document is static and may not be up to date with the current implementation. Be aware too that it won't contain all that's needed, it's mostly listing the main areas that need change. - -### chain - -You'll need to create a new crate in the [chain folder](https://github.com/graphprotocol/graph-node/tree/1cd7936f9143f317feb51be1fc199122761fcbb1/chain) with an appropriate name and the same `version` as the rest of the other ones. - -> Note: you'll probably have to add something like `graph-chain-{{CHAIN_NAME}} = { path = "../chain/{{CHAIN_NAME}}" }` to the `[dependencies]` section of a few other `Cargo.toml` files - -It's here that you add the `protobuf` definitions with the specific types for the chain you're integrating with. Examples: - -- [Ethereum](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/chain/ethereum/proto/codec.proto) -- [NEAR](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/chain/near/proto/codec.proto) -- [Cosmos](https://github.com/graphprotocol/graph-node/blob/caa54c1039d3c282ac31bb0e96cb277dbf82f793/chain/cosmos/proto/type.proto) - -To compile those we use a crate called `tonic`, it will require a [`build.rs` file](https://doc.rust-lang.org/cargo/reference/build-scripts.html) like the one in the other folders/chains, eg: - -```rust -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) - .expect("Failed to compile Firehose CoolChain proto(s)"); -} -``` - -You'll also need a `src/codec.rs` to extract the data from the generated Rust code, much like [this one](https://github.com/graphprotocol/graph-node/blob/caa54c1039d3c282ac31bb0e96cb277dbf82f793/chain/cosmos/src/codec.rs). - -Besides this source file, there should also be a `TriggerFilter`, `NodeCapabilities` and `RuntimeAdapter`, here are a few empty examples: - -`src/adapter.rs` -```rust -use crate::capabilities::NodeCapabilities; -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::prelude::*; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter {} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, _data_sources: impl Iterator + Clone) {} - - fn node_capabilities(&self) -> NodeCapabilities { - NodeCapabilities {} - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - vec![] - } -} -``` - -`src/capabilities.rs` -```rust -use std::cmp::PartialOrd; -use std::fmt; -use std::str::FromStr; - -use anyhow::Error; -use graph::impl_slog_value; - -use crate::DataSource; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd)] -pub struct NodeCapabilities {} - -impl FromStr for NodeCapabilities { - type Err = Error; - - fn from_str(_s: &str) -> Result { - Ok(NodeCapabilities {}) - } -} - -impl fmt::Display for NodeCapabilities { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("{{CHAIN_NAME}}") - } -} - -impl_slog_value!(NodeCapabilities, "{}"); - -impl graph::blockchain::NodeCapabilities for NodeCapabilities { - fn from_data_sources(_data_sources: &[DataSource]) -> Self { - NodeCapabilities {} - } -} -``` - -`src/runtime/runtime_adapter.rs` -```rust -use crate::{Chain, DataSource}; -use anyhow::Result; -use blockchain::HostFn; -use graph::blockchain; - -pub struct RuntimeAdapter {} - -impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, _ds: &DataSource) -> Result> { - Ok(vec![]) - } -} -``` - -The chain specific type definitions should also be available for the `runtime`. Since it comes mostly from the `protobuf` files, there's a [generation tool](https://github.com/streamingfast/graph-as-to-rust) made by StreamingFast that you can use to create the `src/runtime/generated.rs`. - -You'll also have to implement `ToAscObj` for those types, that usually is made in a `src/runtime/abi.rs` file. - -Another thing that will be needed is the `DataSource` types for the [subgraph manifest](https://thegraph.com/docs/en/developer/create-subgraph-hosted/#the-subgraph-manifest). - -`src/data_source.rs` -```rust -#[derive(Clone, Debug)] -pub struct DataSource { - // example fields: - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, - /*...*/ -} - -impl blockchain::DataSource for DataSource { /*...*/ } - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { /*...*/ } - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { /*...*/ } - -impl blockchain::DataSourceTemplate for DataSourceTemplate { /*...*/ } -``` - -And at last, the type that will glue them all, the `Chain` itself. - -`src/chain.rs` -```rust -pub struct Chain { /*...*/ } - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::CoolChain; - - type Block = codec::...; - - type DataSource = DataSource; - - // ... - - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = NodeCapabilities; - - type RuntimeAdapter = RuntimeAdapter; -} - -pub struct TriggersAdapter { /*...*/ } - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { /*...*/ } - -pub struct FirehoseMapper { - endpoint: Arc, -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { /*...*/ } -``` - -### node - -The `src/main.rs` file should be able to handle the connection to the new chain via Firehose for the startup, similar to [this](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/node/src/main.rs#L255). - -### graph - -Two changes are required here: - -1. [BlockchainKind](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/graph/src/blockchain/mod.rs#L309) needs to have a new variant for the chain you're integrating with. -2. And the [IndexForAscTypeId](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/graph/src/runtime/mod.rs#L147) should have the new variants for the chain specific types of the `runtime`. - -### server - -You'll just have to handle the new `BlockchainKind` in the [index-node/src/resolver.rs](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/server/index-node/src/resolver.rs#L361). - -### core - -Just like in the `server` crate, you'll just have to handle the new `BlockchainKind` in the [SubgraphInstanceManager](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/core/src/subgraph/instance_manager.rs#L41). - -## Example Integrations (PRs) - -- NEAR by StreamingFast - - https://github.com/graphprotocol/graph-node/pull/2820 -- Cosmos by Figment - - https://github.com/graphprotocol/graph-node/pull/3212 - - https://github.com/graphprotocol/graph-node/pull/3543 -- Solana by StreamingFast - - https://github.com/graphprotocol/graph-node/pull/3210 - -## What else? - -Besides making `graph-node` support the new chain, [graph-cli](https://github.com/graphprotocol/graph-tooling/tree/main/packages/cli) and [graph-ts](https://github.com/graphprotocol/graph-tooling/tree/main/packages/ts) should also include the new types and enable the new functionality so that subgraph developers can use it. - -For now this document doesn't include how to do that integration, here are a few PRs that might help you with that: - -- NEAR - - `graph-cli` - - https://github.com/graphprotocol/graph-tooling/pull/760 - - https://github.com/graphprotocol/graph-tooling/pull/783 - - `graph-ts` - - https://github.com/graphprotocol/graph-ts/pull/210 - - https://github.com/graphprotocol/graph-ts/pull/217 -- Cosmos - - `graph-cli` - - https://github.com/graphprotocol/graph-tooling/pull/827 - - https://github.com/graphprotocol/graph-tooling/pull/851 - - https://github.com/graphprotocol/graph-toolingpull/888 - - `graph-ts` - - https://github.com/graphprotocol/graph-ts/pull/250 - - https://github.com/graphprotocol/graph-ts/pull/273 - -Also this document doesn't include the multi-blockchain part required for The Graph Network, which at this current moment is in progress, for now the network only supports Ethereum `mainnet`. diff --git a/docs/implementation/metadata.md b/docs/implementation/metadata.md index 562bc371e71..1cf3c189c6c 100644 --- a/docs/implementation/metadata.md +++ b/docs/implementation/metadata.md @@ -7,7 +7,7 @@ List of all known subgraph names. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. | Column | Type | Use | -|-------------------|--------------|-------------------------------------------| +| ----------------- | ------------ | ----------------------------------------- | | `id` | `text!` | primary key, UUID | | `name` | `text!` | user-chosen name | | `current_version` | `text` | `subgraph_version.id` for current version | @@ -18,13 +18,12 @@ List of all known subgraph names. Maintained in the primary, but there is a back The `id` is used by the hosted explorer to reference the subgraph. - ### `subgraphs.subgraph_version` Mapping of subgraph names from `subgraph` to IPFS hashes. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. | Column | Type | Use | -|---------------|--------------|-------------------------| +| ------------- | ------------ | ----------------------- | | `id` | `text!` | primary key, UUID | | `subgraph` | `text!` | `subgraph.id` | | `deployment` | `text!` | IPFS hash of deployment | @@ -32,15 +31,14 @@ Mapping of subgraph names from `subgraph` to IPFS hashes. Maintained in the prim | `vid` | `int8!` | unused | | `block_range` | `int4range!` | unused | - ## Managing a deployment Directory of all deployments. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. -### `deployment_schemas` +### `public.deployment_schemas` | Column | Type | Use | -|--------------|----------------|----------------------------------------------| +| ------------ | -------------- | -------------------------------------------- | | `id` | `int4!` | primary key | | `subgraph` | `text!` | IPFS hash of deployment | | `name` | `text!` | name of `sgdNNN` schema | @@ -52,49 +50,66 @@ Directory of all deployments. Maintained in the primary, but there is a backgrou There can be multiple copies of the same deployment, but at most one per shard. The `active` flag indicates which of these copies will be used for queries; `graph-node` makes sure that there is always exactly one for each IPFS hash. -### `subgraph_deployment` +### `subgraphs.head` + +Details about a deployment that change on every block. Maintained in the +shard alongside the deployment's data in `sgdNNN`. + +| Column | Type | Use | +| ----------------- | ---------- | -------------------------------------------- | +| `id` | `integer!` | primary key, same as `deployment_schemas.id` | +| `block_hash` | `bytea` | current subgraph head | +| `block_number` | `numeric` | | +| `entity_count` | `numeric!` | total number of entities | +| `firehose_cursor` | `text` | | + +The head block pointer in `block_number` and `block_hash` is the latest +block that has been fully processed by the deployment. It will be `null` +until the deployment is fully initialized, and only set when the deployment +processes the first block. For deployments that are grafted or being copied, +the head block pointer will be `null` until the graft/copy has finished +which can take considerable time. + +### `subgraphs.deployment` Details about a deployment to track sync progress etc. Maintained in the shard alongside the deployment's data in `sgdNNN`. The table should only -contain frequently changing data, but for historical reasons contains also -static data. - -| Column | Type | Use | -|--------------------------------------|------------|----------------------------------------------| -| `id` | `integer!` | primary key, same as `deployment_schemas.id` | -| `deployment` | `text!` | IPFS hash | -| `failed` | `boolean!` | | -| `synced` | `boolean!` | | -| `earliest_block_number` | `integer!` | earliest block for which we have data | -| `latest_ethereum_block_hash` | `bytea` | current subgraph head | -| `latest_ethereum_block_number` | `numeric` | | -| `entity_count` | `numeric!` | total number of entities | -| `graft_base` | `text` | IPFS hash of graft base | -| `graft_block_hash` | `bytea` | graft block | -| `graft_block_number` | `numeric` | | -| `reorg_count` | `integer!` | | -| `current_reorg_depth` | `integer!` | | -| `max_reorg_depth` | `integer!` | | -| `fatal_error` | `text` | | -| `non_fatal_errors` | `text[]` | | -| `health` | `health!` | | -| `last_healthy_ethereum_block_hash` | `bytea` | | -| `last_healthy_ethereum_block_number` | `numeric` | | -| `firehose_cursor` | `text` | | -| `debug_fork` | `text` | | +contain data that changes fairly infrequently, but for historical reasons +contains also static data. + +| Column | Type | Use | +| ------------------------------------ | ------------- | ---------------------------------------------------- | +| `id` | `integer!` | primary key, same as `deployment_schemas.id` | +| `subgraph` | `text!` | IPFS hash | +| `earliest_block_number` | `integer!` | earliest block for which we have data | +| `health` | `health!` | | +| `failed` | `boolean!` | | +| `fatal_error` | `text` | | +| `non_fatal_errors` | `text[]` | | +| `graft_base` | `text` | IPFS hash of graft base | +| `graft_block_hash` | `bytea` | graft block | +| `graft_block_number` | `numeric` | | +| `reorg_count` | `integer!` | | +| `current_reorg_depth` | `integer!` | | +| `max_reorg_depth` | `integer!` | | +| `last_healthy_ethereum_block_hash` | `bytea` | | +| `last_healthy_ethereum_block_number` | `numeric` | | +| `debug_fork` | `text` | | +| `synced_at` | `timestamptz` | time when deployment first reach chain head | +| `synced_at_block_number` | `integer` | block number where deployment first reach chain head | The columns `reorg_count`, `current_reorg_depth`, and `max_reorg_depth` are set during indexing. They are used to determine whether a reorg happened while a query was running, and whether that reorg could have affected the query. -### `subgraph_manifest` +### `subgraphs.subgraph_manifest` Details about a deployment that rarely change. Maintained in the shard alongside the deployment's data in `sgdNNN`. | Column | Type | Use | -|-------------------------|------------|------------------------------------------------------| +| ----------------------- | ---------- | ---------------------------------------------------- | | `id` | `integer!` | primary key, same as `deployment_schemas.id` | | `spec_version` | `text!` | | | `description` | `text` | | @@ -108,25 +123,25 @@ shard alongside the deployment's data in `sgdNNN`. | `on_sync` | `text` | Additional behavior when deployment becomes synced | | `history_blocks` | `int4!` | How many blocks of history to keep | -### `subgraph_deployment_assignment` +### `subgraphs.subgraph_deployment_assignment` Tracks which index node is indexing a deployment. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. | Column | Type | Use | -|---------|-------|---------------------------------------------| +| ------- | ----- | ------------------------------------------- | | id | int4! | primary key, ref to `deployment_schemas.id` | | node_id | text! | name of index node | This table could simply be a column on `deployment_schemas`. -### `dynamic_ethereum_contract_data_source` +### `subgraphs.dynamic_ethereum_contract_data_source` Stores the dynamic data sources for all subgraphs (will be turned into a table that lives in each subgraph's namespace `sgdNNN` soon) -### `subgraph_error` +### `subgraphs.subgraph_error` Stores details about errors that subgraphs encounter during indexing. @@ -147,7 +162,7 @@ should have the 'account-like' optimization turned on. Details about features that a deployment uses, Maintained in the primary. | Column | Type | Use | -|----------------|-----------|-------------| +| -------------- | --------- | ----------- | | `id` | `text!` | primary key | | `spec_version` | `text!` | | | `api_version` | `text` | | diff --git a/docs/implementation/sql-interface.md b/docs/implementation/sql-interface.md new file mode 100644 index 00000000000..6b90fe6da9c --- /dev/null +++ b/docs/implementation/sql-interface.md @@ -0,0 +1,89 @@ +# SQL Queries + +**This interface is extremely experimental. There is no guarantee that this +interface will ever be brought to production use. It's solely here to help +evaluate the utility of such an interface** + +**The interface is only available if the environment variable `GRAPH_ENABLE_SQL_QUERIES` is set to `true`** + +SQL queries can be issued by posting a JSON document to +`/subgraphs/sql`. The server will respond with a JSON response that +contains the records matching the query in JSON form. + +The body of the request must contain the following keys: + +* `deployment`: the hash of the deployment against which the query should + be run +* `query`: the SQL query +* `mode`: either `info` or `data`. When the mode is `info` only some + information of the response is reported, with a mode of `data` the query + result is sent in the response + +The SQL query can use all the tables of the given subgraph. Table and +attribute names for normal `@entity` types are snake-cased from their form +in the GraphQL schema, so that data for `SomeDailyStuff` is stored in a +table `some_daily_stuff`. For `@aggregation` types, the table can be +accessed as `()`, for example, `my_stats('hour')` for +`type MyStats @aggregation(..) { .. }` + +The query can use fairly arbitrary SQL, including aggregations and most +functions built into PostgreSQL. + +## Example + +For a subgraph whose schema defines an entity `Block`, the following query +```json +{ + "query": "select number, hash, parent_hash, timestamp from block order by number desc limit 2", + "deployment": "QmSoMeThInG", + "mode": "data" +} +``` + +might result in this response +```json +{ + "data": [ + { + "hash": "\\x5f91e535ee4d328725b869dd96f4c42059e3f2728dfc452c32e5597b28ce68d6", + "number": 5000, + "parent_hash": "\\x82e95c1ee3a98cd0646225b5ae6afc0b0229367b992df97aeb669c898657a4bb", + "timestamp": "2015-07-30T20:07:44+00:00" + }, + { + "hash": "\\x82e95c1ee3a98cd0646225b5ae6afc0b0229367b992df97aeb669c898657a4bb", + "number": 4999, + "parent_hash": "\\x875c9a0f8215258c3b17fd5af5127541121cca1f594515aae4fbe5a7fbef8389", + "timestamp": "2015-07-30T20:07:36+00:00" + } + ] +} +``` + +## Limitations/Ideas/Disclaimers + +Most of these are fairly easy to address: + +- bind variables/query parameters are not supported, only literal SQL + queries +* queries must finish within `GRAPH_SQL_STATEMENT_TIMEOUT` (unlimited by + default) +* queries are always executed at the subgraph head. It would be easy to add + a way to specify a block at which the query should be executed +* the interface right now pretty much exposes the raw SQL schema for a + subgraph, though system columns like `vid` or `block_range` are made + inaccessible. +* it is not possible to join across subgraphs, though it would be possible + to add that. Implenting that would require some additional plumbing that + hides the effects of sharding. +* JSON as the response format is pretty terrible, and we should change that + to something that isn't so inefficient +* the response contains data that's pretty raw; as the example shows, + binary data uses Postgres' notation for hex strings +* because of how broad the supported SQL is, it is pretty easy to issue + queries that take a very long time. It will therefore not be hard to take + down a `graph-node`, especially when no query timeout is set + +Most importantly: while quite a bit of effort has been put into making this +interface safe, in particular, making sure it's not possible to write +through this interface, there's no guarantee that this works without bugs. diff --git a/docs/log-store.md b/docs/log-store.md new file mode 100644 index 00000000000..8be1cddecd8 --- /dev/null +++ b/docs/log-store.md @@ -0,0 +1,853 @@ +# Log Store Configuration and Usage + +This guide explains how to configure subgraph indexing logs storage in graph-node. + +## Table of Contents + +- [Overview](#overview) +- [How Log Stores Work](#how-log-stores-work) +- [Log Store Types](#log-store-types) + - [File-based Logs](#file-based-logs) + - [Elasticsearch](#elasticsearch) + - [Loki](#loki) + - [Disabled](#disabled) +- [Configuration](#configuration) + - [Environment Variables](#environment-variables) + - [CLI Arguments](#cli-arguments) + - [Configuration Precedence](#configuration-precedence) +- [Querying Logs](#querying-logs) +- [Migrating from Deprecated Configuration](#migrating-from-deprecated-configuration) +- [Choosing the Right Backend](#choosing-the-right-backend) +- [Best Practices](#best-practices) +- [Troubleshooting](#troubleshooting) + +## Overview + +Graph Node supports multiple logs storage backends for subgraph indexing logs. Subgraph indexing logs include: +- **User-generated logs**: Explicit logging from subgraph mapping code (`log.info()`, `log.error()`, etc.) +- **Runtime logs**: Handler execution, event processing, data source activity +- **System logs**: Warnings, errors, and diagnostics from the indexing system + +**Available backends:** +- **File**: JSON Lines files on local filesystem (for local development) +- **Elasticsearch**: Enterprise-grade search and analytics (for production) +- **Loki**: Grafana's lightweight log aggregation system (for production) +- **Disabled**: No log storage (default) + +All backends share the same query interface through GraphQL, making it easy to switch between them. + +**Important Note:** When log storage is disabled (the default), subgraph logs still appear in stdout/stderr as they always have. The "disabled" setting simply means logs are not stored separately in a queryable format. You can still see logs in your terminal or container logs - they just won't be available via the `_logs` GraphQL query. + +## How Log Stores Work + +### Architecture + +``` +┌─────────────────┐ +│ Subgraph Code │ +│ (mappings) │ +└────────┬────────┘ + │ log.info(), log.error(), etc. + ▼ +┌─────────────────┐ +│ Graph Runtime │ +│ (WebAssembly) │ +└────────┬────────┘ + │ Log events + ▼ +┌─────────────────┐ +│ Log Drain │ ◄─── slog-based logging system +└────────┬────────┘ + │ Write + ▼ +┌─────────────────┐ +│ Log Store │ ◄─── Configurable backend +│ (ES/Loki/File) │ +└────────┬────────┘ + │ + ▼ +┌─────────────────┐ +│ GraphQL API │ ◄─── Unified query interface +│ (port 8000) │ +└─────────────────┘ +``` + +### Log Flow + +1. **Log sources** generate logs from: + - User mapping code (explicit `log.info()`, `log.error()`, etc. calls) + - Subgraph runtime (handler execution, event processing, data source triggers) + - System warnings and errors (indexing issues, constraint violations, etc.) +2. **Graph runtime** captures these logs with metadata (timestamp, level, source location) +3. **Log drain** formats logs and writes to configured backend +4. **Log store** persists logs and handles queries +5. **GraphQL API** exposes logs through the `_logs` query + +### Log Entry Structure + +Each log entry contains: +- **`id`**: Unique identifier +- **`subgraphId`**: Deployment hash (QmXxx...) +- **`timestamp`**: ISO 8601 timestamp (e.g., `2024-01-15T10:30:00.123456789Z`) +- **`level`**: CRITICAL, ERROR, WARNING, INFO, or DEBUG +- **`text`**: Log message +- **`arguments`**: Key-value pairs from structured logging +- **`meta`**: Source location (module, line, column) + +## Log Store Types + +### File-based Logs + +**Best for:** Local development, testing + +#### How It Works + +File-based logs store each subgraph's logs in a separate JSON Lines (`.jsonl`) file: + +``` +graph-logs/ +├── QmSubgraph1Hash.jsonl +├── QmSubgraph2Hash.jsonl +└── QmSubgraph3Hash.jsonl +``` + +Each line in the file is a complete JSON object representing one log entry. + +#### Storage Format + +```json +{"id":"QmTest-2024-01-15T10:30:00.123456789Z","subgraphId":"QmTest","timestamp":"2024-01-15T10:30:00.123456789Z","level":"error","text":"Handler execution failed, retries: 3","arguments":[{"key":"retries","value":"3"}],"meta":{"module":"mapping.ts","line":42,"column":10}} +``` + +#### Query Performance + +File-based logs stream through files line-by-line with bounded memory usage. + +**Performance characteristics:** +- Query time: O(n) where n = number of log entries +- Memory usage: O(skip + first) - only matching entries kept in memory +- Suitable for: Development and testing + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend file \ + --log-store-file-dir ./graph-logs +``` + +**Full configuration (environment variables):** +```bash +export GRAPH_LOG_STORE_BACKEND=file +export GRAPH_LOG_STORE_FILE_DIR=/var/log/graph-node +export GRAPH_LOG_STORE_FILE_MAX_SIZE=104857600 # 100MB +export GRAPH_LOG_STORE_FILE_RETENTION_DAYS=30 +``` + +#### Features + +**Advantages:** +- No external dependencies +- Simple setup (just specify a directory) +- Human-readable format (JSON Lines) +- Easy to inspect with standard tools (`jq`, `grep`, etc.) +- Good for debugging during development + +**Limitations:** +- Not suitable for production with high log volume +- No indexing (O(n) query time scales with file size) +- No automatic log rotation or retention management +- Single file per subgraph (no sharding) + +#### When to Use + +Use file-based logs when: +- Developing subgraphs locally +- Testing on a development machine +- Running low-traffic subgraphs (< 1000 total logs/day including system logs) +- You want simple log access without external services + +### Elasticsearch + +**Best for:** Production deployments, high log volume, advanced search + +#### How It Works + +Elasticsearch stores logs in indices with full-text search capabilities, making it ideal for production deployments with high log volume. + +**Architecture:** +``` +graph-node → Elasticsearch HTTP API → Elasticsearch cluster + → Index: subgraph-logs-* + → Query DSL for filtering +``` + +#### Features + +**Advantages:** +- **Indexed searching**: Fast queries even with millions of logs +- **Full-text search**: Powerful text search across log messages +- **Scalability**: Handles billions of log entries +- **High availability**: Supports clustering and replication +- **Kibana integration**: Rich visualization and dashboards for operators +- **Time-based indices**: Efficient retention management + +**Considerations:** +- Requires Elasticsearch cluster (infrastructure overhead) +- Resource-intensive (CPU, memory, disk) + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend elasticsearch \ + --log-store-elasticsearch-url http://localhost:9200 +``` + +**Full configuration with authentication:** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend elasticsearch \ + --log-store-elasticsearch-url https://es.example.com:9200 \ + --log-store-elasticsearch-user elastic \ + --log-store-elasticsearch-password secret \ + --log-store-elasticsearch-index subgraph-logs +``` + +**Environment variables:** +```bash +export GRAPH_LOG_STORE_BACKEND=elasticsearch +export GRAPH_LOG_STORE_ELASTICSEARCH_URL=http://localhost:9200 +export GRAPH_LOG_STORE_ELASTICSEARCH_USER=elastic +export GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD=secret +export GRAPH_LOG_STORE_ELASTICSEARCH_INDEX=subgraph-logs +``` + +#### Index Configuration + +Logs are stored in the configured index (default: `subgraph`). The index mapping is automatically created. + +**Recommended index settings for production:** +```json +{ + "settings": { + "number_of_shards": 3, + "number_of_replicas": 1, + "refresh_interval": "5s" + } +} +``` + +#### Query Performance + +**Performance characteristics:** +- Query time: O(log n) with indexing +- Memory usage: Minimal (server-side filtering) +- Suitable for: Millions to billions of log entries + +#### When to Use + +Use Elasticsearch when: +- Running production deployments +- High log volume +- Need advanced search and filtering +- Want to build dashboards with Kibana +- Need high availability and scalability +- Have DevOps resources to manage Elasticsearch or can set up a managed ElasticSearch deployment + +### Loki + +**Best for:** Production deployments, Grafana users, cost-effective at scale + +#### How It Works + +Loki is Grafana's log aggregation system, designed to be cost-effective and easy to operate. Unlike Elasticsearch, Loki only indexes metadata (not full-text), making it more efficient for time-series log data. + +**Architecture:** +``` +graph-node → Loki HTTP API → Loki + → Stores compressed chunks + → Indexes labels only +``` + +#### Features + +**Advantages:** +- **Cost-effective**: Lower storage costs than Elasticsearch +- **Grafana integration**: Native integration with Grafana +- **Horizontal scalability**: Designed for cloud-native deployments +- **Multi-tenancy**: Built-in tenant isolation +- **Efficient compression**: Optimized for log data +- **LogQL**: Powerful query language similar to PromQL +- **Lower resource usage**: Less CPU/memory than Elasticsearch + +**Considerations:** +- No full-text indexing (slower text searches) +- Best used with Grafana (less tooling than Elasticsearch) +- Younger ecosystem than Elasticsearch +- Query performance depends on label cardinality + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend loki \ + --log-store-loki-url http://localhost:3100 +``` + +**With multi-tenancy:** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend loki \ + --log-store-loki-url http://localhost:3100 \ + --log-store-loki-tenant-id my-graph-node +``` + +**Environment variables:** +```bash +export GRAPH_LOG_STORE_BACKEND=loki +export GRAPH_LOG_STORE_LOKI_URL=http://localhost:3100 +export GRAPH_LOG_STORE_LOKI_TENANT_ID=my-graph-node +``` + +#### Labels + +Loki uses labels for indexing. Graph Node automatically creates labels: +- `subgraph_id`: Deployment hash +- `level`: Log level +- `job`: "graph-node" + +#### Query Performance + +**Performance characteristics:** +- Query time: O(n) for text searches, O(log n) for label queries +- Memory usage: Minimal (server-side processing) +- Suitable for: Millions to billions of log entries +- Best performance with label-based filtering + +#### When to Use + +Use Loki when: +- Already using Grafana for monitoring +- Need cost-effective log storage at scale +- Want simpler operations than Elasticsearch +- Multi-tenancy is required +- Log volume is very high (> 1M logs/day) +- Full-text search is not critical + +### Disabled + +**Best for:** Minimalist deployments, reduced overhead + +#### How It Works + +When log storage is disabled (the default), subgraph logs are **still written to stdout/stderr** along with all other graph-node logs. They are just **not stored separately** in a queryable format. + +**Important:** "Disabled" does NOT mean logs are discarded. It means: +- Logs appear in stdout/stderr (traditional behavior) +- Logs are not stored in a separate queryable backend +- The `_logs` GraphQL query returns empty results + +This is the default behavior - logs continue to work exactly as they did before this feature was added. + +#### Configuration + +**Explicitly disable:** +```bash +export GRAPH_LOG_STORE_BACKEND=disabled +``` + +**Or simply don't configure a backend** (defaults to disabled): +```bash +# No log store configuration = disabled +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 +``` + +#### Features + +**Advantages:** +- Zero additional overhead +- No external dependencies +- Minimal configuration +- Logs still appear in stdout/stderr for debugging + +**Limitations:** +- Cannot query logs via GraphQL (`_logs` returns empty results) +- No separation of subgraph logs from other graph-node logs in stdout +- Logs mixed with system logs (harder to filter programmatically) +- No structured querying or filtering capabilities + +#### When to Use + +Use disabled log storage when: +- Running minimal test deployments with less dependencies +- Exposing logs to users is not required for your use case +- You'd like subgraph logs sent to external log collection (e.g., container logs) + +## Configuration + +### Environment Variables + +Environment variables are the recommended way to configure log stores, especially in containerized deployments. + +#### Backend Selection + +```bash +GRAPH_LOG_STORE_BACKEND= +``` +Valid values: `disabled`, `elasticsearch`, `loki`, `file` + +#### Elasticsearch + +```bash +GRAPH_LOG_STORE_ELASTICSEARCH_URL=http://localhost:9200 +GRAPH_LOG_STORE_ELASTICSEARCH_USER=elastic # Optional +GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD=secret # Optional +GRAPH_LOG_STORE_ELASTICSEARCH_INDEX=subgraph # Default: "subgraph" +``` + +#### Loki + +```bash +GRAPH_LOG_STORE_LOKI_URL=http://localhost:3100 +GRAPH_LOG_STORE_LOKI_TENANT_ID=my-tenant # Optional +``` + +#### File + +```bash +GRAPH_LOG_STORE_FILE_DIR=/var/log/graph-node +GRAPH_LOG_STORE_FILE_MAX_SIZE=104857600 # Default: 100MB +GRAPH_LOG_STORE_FILE_RETENTION_DAYS=30 # Default: 30 +``` + +### CLI Arguments + +CLI arguments provide the same functionality as environment variables and the two can be mixed together. + +#### Backend Selection + +```bash +--log-store-backend +``` + +#### Elasticsearch + +```bash +--log-store-elasticsearch-url +--log-store-elasticsearch-user +--log-store-elasticsearch-password +--log-store-elasticsearch-index +``` + +#### Loki + +```bash +--log-store-loki-url +--log-store-loki-tenant-id +``` + +#### File + +```bash +--log-store-file-dir +--log-store-file-max-size +--log-store-file-retention-days +``` + +### Configuration Precedence + +When multiple configuration methods are used: + +1. **CLI arguments** take highest precedence +2. **Environment variables** are used if no CLI args provided +3. **Defaults** are used if neither is set + +## Querying Logs + +All log backends share the same GraphQL query interface. Logs are queried through the subgraph-specific GraphQL endpoint: + +- **Subgraph by deployment**: `http://localhost:8000/subgraphs/id/` +- **Subgraph by name**: `http://localhost:8000/subgraphs/name/` + +The `_logs` query is automatically scoped to the subgraph in the URL, so you don't need to pass a `subgraphId` parameter. + +**Note**: Queries return all log types - both user-generated logs from mapping code and system-generated runtime logs (handler execution, events, warnings, etc.). Use the `search` filter to search for specific messages, or `level` to filter by severity. + +### Basic Query + +Query the `_logs` field at your subgraph's GraphQL endpoint: + +```graphql +query { + _logs( + first: 100 + ) { + id + timestamp + level + text + } +} +``` + +**Example endpoint**: `http://localhost:8000/subgraphs/id/QmYourDeploymentHash` + +### Query with Filters + +```graphql +query { + _logs( + level: ERROR + from: "2024-01-01T00:00:00Z" + to: "2024-01-31T23:59:59Z" + search: "timeout" + first: 50 + skip: 0 + ) { + id + timestamp + level + text + arguments { + key + value + } + meta { + module + line + column + } + } +} +``` + +### Available Filters + +| Filter | Type | Description | +|--------|------|-------------| +| `level` | LogLevel | Filter by level: CRITICAL, ERROR, WARNING, INFO, DEBUG | +| `from` | String | Start timestamp (ISO 8601) | +| `to` | String | End timestamp (ISO 8601) | +| `search` | String | Case-insensitive substring search in log messages | +| `first` | Int | Number of results to return (default: 100, max: 1000) | +| `skip` | Int | Number of results to skip for pagination (max: 10000) | + +### Response Fields + +| Field | Type | Description | +|-------|------|-------------| +| `id` | String | Unique log entry ID | +| `timestamp` | String | ISO 8601 timestamp with nanosecond precision | +| `level` | LogLevel | Log level (CRITICAL, ERROR, WARNING, INFO, DEBUG) | +| `text` | String | Complete log message with arguments | +| `arguments` | [(String, String)] | Structured key-value pairs | +| `meta.module` | String | Source file name | +| `meta.line` | Int | Line number | +| `meta.column` | Int | Column number | + +### Query Examples + +#### Recent Errors + +```graphql +query RecentErrors { + _logs( + level: ERROR + first: 20 + ) { + timestamp + text + meta { + module + line + } + } +} +``` + +#### Search for Specific Text + +```graphql +query SearchTimeout { + _logs( + search: "timeout" + first: 50 + ) { + timestamp + level + text + } +} +``` + +#### Handler Execution Logs + +```graphql +query HandlerLogs { + _logs( + search: "handler" + first: 50 + ) { + timestamp + level + text + } +} +``` + +#### Time Range Query + +```graphql +query LogsInRange { + _logs( + from: "2024-01-15T00:00:00Z" + to: "2024-01-15T23:59:59Z" + first: 1000 + ) { + timestamp + level + text + } +} +``` + +#### Pagination + +```graphql +# First page +query Page1 { + _logs( + first: 100 + skip: 0 + ) { + id + text + } +} + +# Second page +query Page2 { + _logs( + first: 100 + skip: 100 + ) { + id + text + } +} +``` + +### Querying the logs store using cURL + +```bash +curl -X POST http://localhost:8000/subgraphs/id/ \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ _logs(level: ERROR, first: 10) { timestamp level text } }" + }' +``` + +### Performance Considerations + +**File-based:** _for development only_ +- Streams through files line-by-line (bounded memory usage) +- Memory usage limited to O(skip + first) entries +- Query time is O(n) where n = total log entries in file + +**Elasticsearch:** +- Indexed queries are fast regardless of size +- Text searches are optimized with full-text indexing +- Can handle billions of log entries +- Best for production with high query volume + +**Loki:** +- Label-based queries are fast (indexed) +- Text searches scan compressed chunks (slower than Elasticsearch) +- Good performance with proper label filtering +- Best for production with Grafana integration + +## Choosing the Right Backend + +### Decision Matrix + +| Scenario | Recommended Backend | Reason | +|----------|-------------------|-----------------------------------------------------------------------------------| +| Local development | **File** | Simple, no dependencies, easy to inspect | +| Testing/staging | **File** or **Elasticsearch** | File for simplicity, ES if testing production config | +| Production | **Elasticsearch** or **Loki** | Both handle scale well | +| Using Grafana | **Loki** | Native integration | +| Cost-sensitive at scale | **Loki** | Lower storage costs | +| Want rich ecosystem | **Elasticsearch** | More tools and plugins | +| Minimal deployment | **Disabled** | No overhead | + +### Resource Requirements + +#### File-based +- **Disk**: Minimal (log files only) +- **Memory**: Depends on file size during queries +- **CPU**: Minimal +- **Network**: None +- **External services**: None + +#### Elasticsearch +- **Disk**: High (indices + replicas) +- **Memory**: 4-8GB minimum for small deployments +- **CPU**: Medium to high +- **Network**: HTTP API calls +- **External services**: Elasticsearch cluster + +#### Loki +- **Disk**: Medium (compressed chunks) +- **Memory**: 2-4GB minimum +- **CPU**: Low to medium +- **Network**: HTTP API calls +- **External services**: Loki server + +## Best Practices + +### General + +1. **Start with file-based for development** - Simplest setup, easy debugging +2. **Use Elasticsearch or Loki for production** - Better performance and features +3. **Monitor log volume** - Set up alerts if log volume grows unexpectedly (includes both user logs and system-generated runtime logs) +4. **Set retention policies** - Don't keep logs forever (disk space and cost) +5. **Use structured logging** - Pass key-value pairs to log functions for better filtering + +### File-based Logs + +1. **Monitor file size** - While queries use bounded memory, larger files take longer to scan (O(n) query time) +2. **Archive old logs** - Manually archive/delete old files or implement external rotation +3. **Monitor disk usage** - Files can grow quickly with verbose logging +4. **Use JSON tools** - `jq` is excellent for inspecting .jsonl files locally + +**Example local inspection:** +```bash +# Count logs by level +cat graph-logs/QmExample.jsonl | jq -r '.level' | sort | uniq -c + +# Find errors in last 1000 lines +tail -n 1000 graph-logs/QmExample.jsonl | jq 'select(.level == "error")' + +# Search for specific text +cat graph-logs/QmExample.jsonl | jq 'select(.text | contains("timeout"))' +``` + +### Elasticsearch + +1. **Use index patterns** - Time-based indices for easier management +2. **Configure retention** - Use Index Lifecycle Management (ILM) +3. **Monitor cluster health** - Set up Elasticsearch monitoring +4. **Tune for your workload** - Adjust shards/replicas based on log volume +5. **Use Kibana** - Visualize and explore logs effectively + +**Example Elasticsearch retention policy:** +```json +{ + "policy": "graph-logs-policy", + "phases": { + "hot": { "min_age": "0ms", "actions": {} }, + "warm": { "min_age": "7d", "actions": {} }, + "delete": { "min_age": "30d", "actions": { "delete": {} } } + } +} +``` + +### Loki + +1. **Use proper labels** - Don't over-index, keep label cardinality low +2. **Configure retention** - Set retention period in Loki config +3. **Use Grafana** - Native integration provides best experience +4. **Compress efficiently** - Loki's compression works best with batch writes +5. **Multi-tenancy** - Use tenant IDs if running multiple environments + +**Example Grafana query:** +```logql +{subgraph_id="QmExample", level="error"} |= "timeout" +``` + +## Troubleshooting + +### File-based Logs + +**Problem: Log file doesn't exist** +- Check `GRAPH_LOG_STORE_FILE_DIR` is set correctly +- Verify directory is writable by graph-node + +**Problem: Queries are slow** +- Subgraph logs file may be very large +- Consider archiving old logs or implementing retention +- For high-volume production use, switch to Elasticsearch or Loki + +**Problem: Disk filling up** +- Implement log rotation +- Reduce log verbosity in subgraph code +- Set up monitoring for disk usage + +### Elasticsearch + +**Problem: Cannot connect to Elasticsearch** +- Verify `GRAPH_LOG_STORE_ELASTICSEARCH_URL` is correct +- Check Elasticsearch is running: `curl http://localhost:9200` +- Verify authentication credentials if using security features +- Check network connectivity and firewall rules + +**Problem: No logs appearing in Elasticsearch** +- Check Elasticsearch cluster health +- Verify index exists: `curl http://localhost:9200/_cat/indices` +- Check graph-node logs for write errors +- Verify Elasticsearch has disk space + +**Problem: Queries are slow** +- Check Elasticsearch cluster health and resources +- Verify indices are not over-sharded +- Consider adding replicas for query performance +- Review query patterns and add appropriate indices + +### Loki + +**Problem: Cannot connect to Loki** +- Verify `GRAPH_LOG_STORE_LOKI_URL` is correct +- Check Loki is running: `curl http://localhost:3100/ready` +- Verify tenant ID if using multi-tenancy +- Check network connectivity + +**Problem: No logs appearing in Loki** +- Check Loki service health +- Verify Loki has disk space for chunks +- Check graph-node logs for write errors +- Verify Loki retention settings aren't deleting logs immediately + +**Problem: Queries return no results in Grafana** +- Check label selectors match what graph-node is sending +- Verify time range includes when logs were written +- Check Loki retention period +- Verify tenant ID matches if using multi-tenancy + +## Further Reading + +- [Environment Variables Reference](environment-variables.md) +- [Graph Node Configuration](config.md) +- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) +- [Grafana Loki Documentation](https://grafana.com/docs/loki/latest/) diff --git a/docs/plans/gnd-cli-expansion.md b/docs/plans/gnd-cli-expansion.md new file mode 100644 index 00000000000..b1bef8bc108 --- /dev/null +++ b/docs/plans/gnd-cli-expansion.md @@ -0,0 +1,418 @@ +# Plan: Extend gnd with graph-cli functionality + +## Overview + +Extend the existing `gnd` (Graph Node Dev) CLI to be a **drop-in replacement** for the TypeScript-based `graph-cli`. The goal is identical CLI behavior, flags, output format, and byte-for-byte identical AssemblyScript code generation. + +**Spec document**: `docs/specs/gnd-cli-expansion.md` + +**Scope**: + +- Ethereum protocol only (other protocols are future work) +- All graph-cli commands except `local` and `node` subcommand +- Match latest graph-cli version (0.98.x) + +**Current state of gnd** (`~/code/graph-node/gnd/`): + +- ~700 lines of Rust across 3 files +- Single command with flags (no subcommands) +- Already uses clap, imports graph crates +- Runs graph-node in dev mode with file watching + +## Status Summary (Updated 2026-01-19) + +**Overall Progress**: ~99% complete (implementation + docs + automated tests done, only publish/test commands need manual verification) + +| Phase | Status | Notes | +|-------|--------|-------| +| Phase 1: CLI Restructure | ✅ Complete | | +| Phase 2: Infrastructure | ✅ Complete | | +| Phase 3: Simple Commands | ✅ Complete | | +| Phase 4: Code Generation | ✅ Complete | 11 verification fixtures passing | +| Phase 5: Migrations | ✅ Complete | | +| Phase 6: Build Command | ✅ Complete | Tested via `just test-gnd-commands` | +| Phase 7: Deploy Command | ✅ Complete | Tested via `just test-gnd-cli` | +| Phase 8: Init Command | ✅ Complete | Tested via `just test-gnd-commands` | +| Phase 9: Add Command | ✅ Complete | | +| Phase 10: Publish Command | 🟡 Needs Testing | Manual end-to-end verification | +| Phase 11: Test Command | 🟡 Needs Testing | Manual Matchstick test | +| Phase 12: Testing & Polish | ✅ Complete | Documentation done, test porting analyzed | +| Phase 13: CLI Integration Tests | ✅ Complete | Test infrastructure created, ready for manual verification | + +**Total LOC**: ~15,000 lines of Rust (158 unit tests, 11 verification tests passing) + +**Documentation**: +- `gnd/README.md` - CLI command reference with examples +- `gnd/docs/MIGRATING_FROM_GRAPH_CLI.md` - Migration guide from graph-cli + +## Git Workflow + +**Branch**: All work should be committed to the `gnd-cli` branch. + +**Commit discipline**: + +- Commit work in small, reviewable chunks +- Each commit should be self-contained and pass all checks +- Prefer many small commits over few large ones +- Each commit message should clearly describe what it does + +**Before each commit**: + +```bash +just format +just lint +just test-unit # only run the tests in gnd/ +``` + +- MANDATORY: Work must be committed, a task is only done when work is committed +- MANDATORY: Make sure to follow the commit discipline above + +## Commands + +| Command | Description | Complexity | +| ------------- | ----------------------------------- | ----------------------- | +| `gnd dev` | Existing gnd functionality | Done (restructure only) | +| `gnd codegen` | Generate AssemblyScript types | High | +| `gnd build` | Compile to WASM | Medium | +| `gnd deploy` | Deploy to Graph Node | Medium | +| `gnd init` | Scaffold new subgraph | High | +| `gnd add` | Add datasource to existing subgraph | Medium | +| `gnd create` | Register subgraph name | Low | +| `gnd remove` | Unregister subgraph name | Low | +| `gnd auth` | Set deploy key | Low | +| `gnd publish` | Publish to decentralized network | Medium | +| `gnd test` | Run Matchstick tests | Low (shell out) | +| `gnd clean` | Remove build artifacts | Low | + +**Not implemented** (documented differences): + +- `gnd local` - Use existing test infrastructure +- `gnd node` - Use graphman for node management +- `--uncrashable` flag - Float Capital third-party feature + +## Reusable from graph-node + +| Feature | Location | Status | +| ------------------- | ------------------------------------- | ---------------------------- | +| Manifest parsing | `graph/src/data/subgraph/mod.rs` | Ready | +| Manifest validation | `graph/src/data/subgraph/mod.rs` | Ready (may need refactoring) | +| GraphQL schema | `graph/src/schema/input/` | Ready | +| ABI parsing | `ethabi` + chain/ethereum wrappers | Ready | +| IPFS client | `graph/src/ipfs/` | Ready | +| Link resolution | `graph/src/components/link_resolver/` | Ready | +| File watching | `gnd/src/watcher.rs` | Ready | +| CLI framework | `clap` (already in gnd) | Ready | + +## Module Structure + +``` +gnd/src/ +├── main.rs # Entry point, clap setup +├── lib.rs +├── commands/ +│ ├── mod.rs +│ ├── dev.rs # Existing gnd functionality +│ ├── codegen.rs +│ ├── build.rs +│ ├── deploy.rs +│ ├── init.rs +│ ├── add.rs +│ ├── create.rs +│ ├── remove.rs +│ ├── auth.rs +│ ├── publish.rs +│ ├── test.rs +│ └── clean.rs +├── codegen/ +│ ├── mod.rs +│ ├── schema.rs # Entity class generation +│ ├── abi.rs # ABI binding generation +│ └── template.rs # Template binding generation +├── scaffold/ +│ ├── mod.rs +│ ├── manifest.rs # Generate subgraph.yaml +│ ├── schema.rs # Generate schema.graphql +│ └── mapping.rs # Generate mapping.ts +├── migrations/ +│ ├── mod.rs +│ └── ... # One module per migration version +├── compiler/ +│ ├── mod.rs +│ └── asc.rs # Shell out to asc +├── services/ +│ ├── mod.rs +│ ├── etherscan.rs # Etherscan API client +│ ├── sourcify.rs # Sourcify API client +│ ├── registry.rs # Network registry client +│ └── graph_node.rs # Graph Node JSON-RPC client +├── config/ +│ ├── mod.rs +│ ├── auth.rs # ~/.graphprotocol/ management +│ └── networks.rs # networks.json handling +├── output/ +│ ├── mod.rs +│ └── spinner.rs # Progress/spinner output (match TS CLI exactly) +└── watcher.rs # Existing file watcher +``` + +## Dependencies to Add + +| Crate | Purpose | +| ----------- | ------------------------------------------- | +| `inquire` | Interactive prompts (init) | +| `minijinja` | Template rendering (scaffold) | +| `indicatif` | Progress bars and spinners | +| `reqwest` | HTTP client (Etherscan, Sourcify, registry) | + +## TODO List + +### Phase 1: CLI Restructure + +- [x] Create `gnd-cli` branch +- [x] Restructure main.rs with clap subcommands +- [x] Move existing gnd logic to `commands/dev.rs` +- [x] Add empty command stubs for all commands +- [x] Verify `gnd dev` works exactly as before +- [x] Add `--version` output showing both gnd and graph-cli versions + +### Phase 2: Infrastructure + +- [x] Implement `output/spinner.rs` matching TS CLI output format exactly +- [x] Implement `config/auth.rs` for `~/.graph-cli.json` management (in commands/auth.rs) +- [x] Implement `config/networks.rs` for networks.json parsing +- [x] Add reqwest dependency and basic HTTP client setup (in services/graph_node.rs) +- [x] Implement `services/registry.rs` for @pinax/graph-networks-registry (implemented as `NetworksRegistry` in services/contract.rs) + +### Phase 3: Simple Commands + +- [x] Implement `gnd clean` command +- [x] Implement `gnd auth` command +- [x] Implement `gnd create` command +- [x] Implement `gnd remove` command +- [x] Add tests for simple commands + +### Phase 4: Code Generation (High Priority, High Risk) + +- [x] Study TS CLI codegen output in detail +- [x] Implement `codegen/schema.rs` - entity classes from GraphQL +- [x] Implement `codegen/typescript.rs` - AST builders for TypeScript/AssemblyScript +- [x] Implement `codegen/types.rs` - type conversion utilities +- [x] Implement `codegen/abi.rs` - ABI bindings +- [x] Implement `codegen/template.rs` - template bindings +- [x] Integrate prettier formatting (shell out) +- [x] Implement `commands/codegen.rs` with all flags +- [x] Implement `--watch` mode using existing file watcher +- [x] Create snapshot tests comparing output to TS CLI (11 fixtures from graph-cli validation tests) +- [x] Verify byte-for-byte identical output - tests pass with documented known differences: + - Int8 import always included (gnd simplicity) + - Trailing commas in multi-line constructs (gnd style) + - 2D array accessors use correct `toStringMatrix()` (gnd fixes graph-cli bug) + +### Phase 5: Migrations + +- [x] Study TS CLI migrations in `/packages/cli/src/migrations/` +- [x] Implement migration framework in `migrations/mod.rs` +- [x] Implement each migration version (0.0.1 → current) +- [x] Add `--skip-migrations` flag support +- [x] Test migrations with old manifest versions + +### Phase 6: Build Command + +- [x] Implement `compiler/asc.rs` - shell out to asc +- [x] Implement `commands/build.rs` with all flags (~1084 lines, full build pipeline) +- [x] Handle network file resolution +- [x] Implement `--watch` mode +- [x] Implement optional IPFS upload with deduplication +- [x] Test build output matches TS CLI - Covered by `just test-gnd-commands` (test_build_after_codegen) + +### Phase 7: Deploy Command + +- [x] Implement `services/graph_node.rs` - JSON-RPC client (~258 lines, create/remove/deploy) +- [x] Implement `commands/deploy.rs` with all flags (~239 lines) +- [x] Support all deploy targets (local, studio, hosted service) - defaults to Subgraph Studio +- [x] Handle access token / deploy key authentication +- [x] Test deployment to local Graph Node - Covered by `just test-gnd-cli` (block-handlers, value-roundtrip, int8) + +### Phase 8: Init Command + +- [x] Add `inquire` dependency for interactive prompts +- [x] Implement `services/contract.rs` - ABI fetching (Etherscan, Blockscout, Sourcify) (~730 lines) +- [x] Implement networks registry loading +- [x] Implement `scaffold/manifest.rs` (~271 lines) +- [x] Implement `scaffold/schema.rs` (~206 lines) +- [x] Implement `scaffold/mapping.rs` (~205 lines) +- [x] Implement `commands/init.rs` with all flags (~1060 lines including --from-subgraph) +- [x] Implement --from-example mode +- [x] Implement --from-contract mode (uses ContractService for ABI fetching) +- [x] Add interactive prompts when required options are missing +- [x] Implement --from-subgraph mode (fetches manifest from IPFS, extracts immutable entities) +- [x] Test scaffold output matches TS CLI - Covered by `just test-gnd-commands` (test_init_from_example, test_init_from_contract_with_abi) + +### Phase 9: Add Command + +- [x] Implement `commands/add.rs` (~773 lines, fully functional) +- [x] Reuse scaffold components for actual implementation +- [x] ABI validation, entity generation, mapping creation, manifest updates +- [x] Test adding datasource to existing subgraph (unit tests for helper functions) +- [x] Comprehensive error handling + +### Phase 10: Publish Command + +- [x] Implement `commands/publish.rs` (~230 lines, fully functional) +- [x] Build subgraph and upload to IPFS (reuses build command) +- [x] Open browser to webapp URL with query params (same approach as graph-cli) +- [x] Support --ipfs-hash flag to skip build +- [x] Support --subgraph-id, --api-key for updating existing subgraphs +- [x] Support --protocol-network (arbitrum-one, arbitrum-sepolia) +- [x] Interactive prompt before opening browser +- [ ] Test publish workflow (NEEDS HUMAN: manual end-to-end verification) + +### Phase 11: Test Command + +- [x] Implement Matchstick binary download/detection +- [x] Implement `commands/test.rs` (~254 lines, shell out to Matchstick) +- [x] Handle Docker mode (Dockerfile generation on demand) +- [x] Recompilation flag support +- [ ] Test with actual Matchstick tests (NEEDS HUMAN: install matchstick + run end-to-end) + +### Phase 12: Testing & Polish + +- [x] Port all tests from TS CLI test suite not already covered by gnd tests + - Location: `/home/lutter/code/subgraphs/graph-cli/packages/cli/tests/` + - Review existing coverage and identify gaps + - Analysis: gnd has 11/12 success fixtures from graph-cli validation tests (missing `near-is-valid` which is out of scope for Ethereum-only support). Error/validation fixtures test manifest parsing which is handled by graph-node's validation layer. +- [x] Add snapshot tests for code generation scenarios (11 fixtures from graph-cli validation tests) +- [x] Add tests for overloaded events/functions in ABI codegen (disambiguation) +- [x] Add tests for simple array fields in schema codegen +- [x] Add tests for nested array types (`[[String]]` etc.) - schema codegen refactored to track list depth +- [x] Add tests for tuple/struct types in ABI codegen (functions, events, nested, arrays) +- [x] Add tests for array types in ABI codegen (array params in events, 2D matrices) +- [x] Codegen verification test framework (gnd/tests/codegen_verification.rs) +- [x] Add comprehensive tests for prompt module (network completer, source type) +- [x] Add comprehensive tests for init command (interactive mode detection) +- [ ] Ensure all edge cases are covered (ongoing) +- [x] Documentation: + - [x] CLI usage docs (README with command reference, examples, common workflows) + - [x] Migration guide from graph-cli to gnd (differences, compatibility notes) +- [x] Shell completions (bash, elvish, fish, powershell, zsh via clap_complete) + +### Phase 13: CLI Integration Tests + +Verify gnd works as a drop-in replacement for graph-cli by running integration tests with `GRAPH_CLI=../target/debug/gnd`. + +**Key Insight**: The existing integration test infrastructure uses `CONFIG.graph_cli` which can be set via the `GRAPH_CLI` environment variable. By setting this to the gnd binary, existing integration tests will use gnd for the deployment flow (codegen, create, deploy). + +#### Integration Deployment Tests (`tests/tests/gnd_cli_tests.rs`) + +- [x] Create test file that sets `GRAPH_CLI` env var to gnd binary +- [x] Verify gnd binary exists (fail fast with clear message) +- [x] Run subset of integration tests: `block-handlers`, `value-roundtrip`, `int8` +- [x] Reuse `Contract::deploy_all()`, `CONFIG.reset_database()`, `CONFIG.spawn_graph_node()` +- [x] Use `Subgraph::deploy()` which calls: `gnd codegen`, `gnd create`, `gnd deploy` +- [x] Verify subgraphs sync and queries return expected data + +**Commands Tested via Subgraph::deploy()**: +- `gnd codegen` - via `Subgraph::deploy()` +- `gnd create` - via `Subgraph::deploy()` +- `gnd deploy` - via `Subgraph::deploy()` + +#### Standalone Command Tests (`gnd/tests/cli_commands.rs`) + +Tests for commands that don't require running graph-node: + +**`gnd init` Tests**: +- [x] `test_init_from_example` - Run `gnd init --from-example ethereum-gravatar`, verify scaffold created +- [x] `test_init_from_contract_with_abi` - Run with `--from-contract` + `--abi`, verify manifest/schema/mapping created +- [ ] `test_init_from_subgraph` - Run with `--from-subgraph`, verify immutable entities extracted (requires IPFS) - **Skipped**: Requires IPFS with subgraph data + +**`gnd add` Tests**: +- [x] `test_add_datasource` - Create base subgraph with init, then add contract, verify manifest updated + +**`gnd build` Tests**: +- [x] `test_build_after_codegen` - Copy fixture, run codegen + build, verify `build/` directory created with WASM + +**`gnd codegen` Tests**: +- [x] `test_codegen_generates_types` - Run codegen on scaffolded subgraph, verify `generated/schema.ts` exists + +**`gnd clean` Tests**: +- [x] `test_clean_removes_artifacts` - Verify clean removes `generated/` and `build/` directories + +**ABIs Available**: `tests/contracts/abis/*.json` + +#### Justfile Targets + +- [x] Add `test-gnd-cli` - Run gnd CLI integration tests (deployment with gnd as CLI) +- [x] Add `test-gnd-commands` - Run gnd standalone command tests (init, add, build) + +#### Service Requirements + +Tests use the same services as `just test-integration`: + +| Service | Port | Start Command | +|---------|------|---------------| +| PostgreSQL | 3011 | `nix run .#integration` | +| IPFS | 3001 | (included above) | +| Anvil | 3021 | (included above) | + +#### Verification + +1. `just build` - Build gnd binary +2. `just test-gnd-cli` - Run integration deployment tests with gnd +3. `just test-gnd-commands` - Run standalone command tests + +Expected output: All tests pass, showing gnd works as drop-in replacement. + +## Key Decisions + +| Decision | Choice | Rationale | +| ---------------- | ---------------------------- | ------------------------------------------- | +| Binary name | `gnd` | Existing name, avoid confusion with `graph` | +| Protocol support | Ethereum only | Simplify initial scope | +| Compatibility | Drop-in replacement | Same flags, output, exit codes | +| Code generation | Byte-for-byte identical | Snapshot testable, no surprises | +| Formatting | Shell out to prettier | Guarantees identical output | +| Compilation | Shell out to asc | Same as TS CLI | +| Testing | Shell out to Matchstick | Same as TS CLI | +| Debug logging | RUST_LOG | Rust standard, not DEBUG env var | +| Validation | Use graph-node's | Refactor if needed, single source of truth | +| Network registry | Fetch at runtime | Registry contents change over time | +| Error messages | Same info, format may differ | Convey same information | +| TS CLI bugs | Fix them | Don't replicate bugs | + +## Risk Assessment + +| Risk | Severity | Mitigation | +| --------------------------- | -------- | ------------------------------------ | +| Code generation fidelity | High | Snapshot tests against TS CLI output | +| Migration correctness | High | Test with real old manifests | +| Breaking existing gnd users | Medium | Keep `gnd dev` behavior identical | +| `asc` version compat | Medium | Version detection, clear errors | +| Network API changes | Low | Error handling, clear messages | + +## TS CLI References + +Key files to study in `/home/lutter/code/subgraphs/graph-cli/packages/cli/src/`: + +| Feature | TS CLI Location | +| ---------------- | ----------------------------------- | +| Commands | `commands/*.ts` | +| Type generator | `type-generator.ts` | +| Schema codegen | `codegen/schema.ts` | +| ABI codegen | `protocols/ethereum/codegen/abi.ts` | +| Template codegen | `codegen/template.ts` | +| Compiler | `compiler/index.ts` | +| Migrations | `migrations/` | +| Scaffold | `scaffold/` | +| Spinner/output | `command-helpers/spinner.ts` | +| Auth | `command-helpers/auth.ts` | +| Network config | `command-helpers/network.ts` | +| Tests | `../tests/cli/` | + +## Effort Estimate + +**~3-4 person-months** for full implementation because: + +- Code generation must be byte-for-byte identical (requires careful study) +- All migrations must be implemented +- Full test coverage needed +- But we reuse: manifest parsing, validation, IPFS client, file watching, CLI framework diff --git a/docs/plans/runner-refactor.md b/docs/plans/runner-refactor.md new file mode 100644 index 00000000000..c5ce612f8db --- /dev/null +++ b/docs/plans/runner-refactor.md @@ -0,0 +1,281 @@ +# Runner Refactor Implementation Plan + +This document outlines the implementation plan for the runner refactor described in [the spec](../specs/runner-refactor.md). + +## Overview + +The refactor transforms `core/src/subgraph/runner.rs` from a complex nested-loop structure into a cleaner state machine with explicit pipeline stages. + +## Git Workflow + +**Branch**: All work should be committed to the `runner-refactor` branch. + +**Commit discipline**: + +- Commit work in small, reviewable chunks +- Each commit should be self-contained and pass all checks +- Prefer many small commits over few large ones +- Each commit message should clearly describe what it does +- Each step in the implementation phases should correspond to one or more commits + +**Before each commit**: + +```bash +just format +just lint +just test-unit +just test-runner +``` + +- MANDATORY: Work must be committed, a task is only done when work is committed +- MANDATORY: Make sure to follow the commit discipline above +- IMPORTANT: The runner tests produce output in `tests/runner-tests.log`. Use that to investigate failures. + +## Implementation Phases + +### Phase 1: Extract TriggerRunner Component + +**Goal:** Eliminate duplicated trigger processing code (lines 616-656 vs 754-790). + +**Files to modify:** + +- `core/src/subgraph/runner.rs` - Extract logic +- Create `core/src/subgraph/runner/trigger_runner.rs` + +**Steps:** + +1. Create `TriggerRunner` struct with execute method +2. Replace first trigger loop (lines 616-656) with `TriggerRunner::execute()` +3. Replace second trigger loop (lines 754-790) with same call +4. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- No behavioral changes + +### Phase 2: Define RunnerState Enum + +**Goal:** Introduce explicit state machine types without changing control flow yet. + +**Files to modify:** + +- Create `core/src/subgraph/runner/state.rs` +- `core/src/subgraph/runner.rs` - Add state field + +**Steps:** + +1. Define `RunnerState` enum with all variants +2. Define `RestartReason` and `StopReason` enums +3. Add `state: RunnerState` field to `SubgraphRunner` +4. Initialize state in constructor +5. Verify tests pass (no behavioral changes yet) + +**Verification:** + +- Code compiles +- Tests pass unchanged + +### Phase 3: Refactor run_inner to State Machine + +**Goal:** Replace nested loops with explicit state transitions. + +**Files to modify:** + +- `core/src/subgraph/runner.rs` - Rewrite `run_inner` + +**Steps:** + +1. Extract `initialize()` method for pre-loop setup +2. Extract `await_block()` method for stream event handling +3. Extract `restart()` method for restart logic +4. Extract `finalize()` method for cleanup +5. Rewrite `run_inner` as state machine loop +6. Remove nested loop structure +7. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Same behavior, cleaner structure + +### Phase 4: Define Pipeline Stages + +**Goal:** Break `process_block` into explicit stages. + +**Files to modify:** + +- Create `core/src/subgraph/runner/pipeline.rs` +- `core/src/subgraph/runner.rs` - Refactor `process_block` + +**Steps:** + +1. Extract `match_triggers()` stage method +2. Extract `execute_triggers()` stage method (uses `TriggerRunner`) +3. Extract `process_dynamic_data_sources()` stage method +4. Extract `process_offchain_triggers()` stage method +5. Extract `persist_block_state()` stage method +6. Rewrite `process_block` to call stages in sequence +7. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Same behavior, cleaner structure + +### Phase 5: Consolidate Error Handling + +**Goal:** Unify scattered error handling into explicit classification. + +**Files to modify:** + +- `graph/src/components/subgraph/error.rs` (or wherever `ProcessingError` lives) +- `core/src/subgraph/runner.rs` - Use new error methods + +**Steps:** + +1. Add `ProcessingErrorKind` enum with Deterministic/NonDeterministic/PossibleReorg variants +2. Add `kind()` method to `ProcessingError` +3. Add helper methods: `should_stop_processing()`, `should_restart()`, `is_retryable()` +4. Replace scattered error checks in `process_block` with unified logic +5. Replace scattered error checks in dynamic DS handling +6. Replace scattered error checks in `handle_offchain_triggers` +7. Document error handling invariants in code comments +8. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Error behavior unchanged (same semantics, cleaner code) + +### Phase 6: Add BlockState Checkpoints + +**Goal:** Enable rollback capability with minimal overhead. + +**Files to modify:** + +- `graph/src/prelude.rs` or wherever `BlockState` is defined +- `core/src/subgraph/runner.rs` - Use checkpoints + +**Steps:** + +1. Add `checkpoint()` method to `BlockState` +2. Add `BlockStateCheckpoint` struct +3. Add `restore()` method to `BlockState` +4. Use checkpoint before dynamic DS processing +5. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- No performance regression (checkpoints are lightweight) + +### Phase 7: Module Organization + +**Goal:** Organize code into proper module structure. + +**Files to create/modify:** + +- `core/src/subgraph/runner/mod.rs` +- Move/organize existing extracted modules + +**Steps:** + +1. Create `runner/` directory +2. Move `state.rs`, `pipeline.rs`, `trigger_runner.rs` into it +3. Update `runner.rs` to re-export from module +4. Update imports in dependent files +5. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- `just lint` shows no warnings + +## Completion Criteria + +Each phase is complete when: + +1. `just format` - Code is formatted +2. `just lint` - Zero warnings +3. `just check --release` - Builds in release mode +4. `just test-unit` - Unit tests pass +5. `just test-runner` - Runner tests pass + +## Progress Checklist + +### Phase 1: Extract TriggerRunner Component + +- [x] Create `TriggerRunner` struct with execute method +- [x] Replace first trigger loop (lines 616-656) +- [x] Replace second trigger loop (lines 754-790) +- [x] Verify tests pass + +### Phase 2: Define RunnerState Enum + +- [x] Define `RunnerState` enum with all variants +- [x] Define `RestartReason` and `StopReason` enums +- [x] Add `state: RunnerState` field to `SubgraphRunner` +- [x] Initialize state in constructor +- [x] Verify tests pass + +### Phase 3: Refactor run_inner to State Machine + +- [x] Extract `initialize()` method +- [x] Extract `await_block()` method +- [x] Extract `restart()` method +- [x] Extract `finalize()` method +- [x] Rewrite `run_inner` as state machine loop +- [x] Remove nested loop structure +- [x] Verify tests pass + +### Phase 4: Define Pipeline Stages + +- [x] Extract `match_triggers()` stage method +- [x] Extract `execute_triggers()` stage method +- [x] Extract `process_dynamic_data_sources()` stage method +- [x] Extract `process_offchain_triggers()` stage method +- [x] Extract `persist_block_state()` stage method +- [x] Rewrite `process_block` to call stages in sequence +- [x] Verify tests pass + +### Phase 5: Consolidate Error Handling + +- [x] Add `ProcessingErrorKind` enum +- [x] Add `kind()` method to `ProcessingError` +- [x] Add helper methods (`should_stop_processing()`, `should_restart()`, `is_retryable()`) +- [x] Replace scattered error checks in `process_block` +- [x] Replace scattered error checks in dynamic DS handling (preserved existing behavior per spec) +- [x] Replace scattered error checks in `handle_offchain_triggers` (preserved existing behavior per spec) +- [x] Document error handling invariants +- [x] Verify tests pass + +### Phase 6: Add BlockState Checkpoints + +- [x] Add `BlockStateCheckpoint` struct +- [x] Add `checkpoint()` method to `BlockState` +- [x] Add `restore()` method to `BlockState` +- [x] Use checkpoint before dynamic DS processing +- [x] Verify tests pass + +### Phase 7: Module Organization + +- [x] Create `runner_components/` directory (named to avoid conflict with `runner.rs`) +- [x] Move `state.rs`, `trigger_runner.rs` into it (pipeline.rs was not created as stages are methods) +- [x] Create `runner_components/mod.rs` with re-exports +- [x] Update imports in `runner.rs` to use the new module path +- [x] Verify tests pass +- [x] `just lint` shows zero warnings + +## Notes + +- Each phase should be a separate, reviewable PR +- Phases 1-4 can potentially be combined if changes are small +- Phase 3 (FSM refactor of run_inner) is the most invasive and should be reviewed carefully +- Phase 5 (error handling) can be done earlier if it helps simplify other phases +- Preserve all existing behavior - this is a refactor, not a feature change diff --git a/docs/specs/dump-restore.md b/docs/specs/dump-restore.md new file mode 100644 index 00000000000..6d7e117b78d --- /dev/null +++ b/docs/specs/dump-restore.md @@ -0,0 +1,269 @@ +# Spec: Parquet Dump/Restore for Subgraph Data + +## Problem + +Subgraph entity data lives exclusively in PostgreSQL. There's no way to export a subgraph's data for backup, migration between environments, or sharing. We need a file-based dump/restore mechanism. + +## Goals + +1. **Dump** a subgraph's entity data to parquet files +2. **Restore** a subgraph from parquet files +3. **Incremental append** -- add newly arrived data to an existing dump without rewriting it +4. Long-term: make dumping an **ongoing process** (not just a one-off CLI operation) + +## Non-goals (for now) + +- S3/GCS output +- Schema evolution / migration between different schema versions + +## Data Model Recap + +Each subgraph deployment has: +- A PostgreSQL schema (e.g., `sgd123`) +- One table per entity type, with columns: + - `vid` (bigserial) -- row version ID, primary key + - `block_range` (int4range) for mutable entities OR `block$` (int) for immutable + - `causality_region` (int, optional) -- for offchain data sources + - Data columns matching GraphQL fields (text, int, bigint, numeric, bytea, bool, timestamptz, arrays, enums) +- `data_sources$` table -- dynamic data sources created at runtime (defined in `dynds/private.rs`, separate from Layout): + - `vid` (int, identity PK), `block_range` (int4range), `causality_region` (int) + - `manifest_idx` (int), `parent` (int, self-ref FK), `id` (bytea) + - `param` (bytea, nullable), `context` (jsonb, nullable), `done_at` (int, nullable) + - Note: `parent` and `id` exist in the DDL but are currently unused by `insert()`, `load()`, and `copy_to()`. The dump should include them for completeness (they may contain data in older deployments). +- `poi2$` table -- Proof of Indexing data (a **mutable** entity table in the Layout, i.e. it uses `block_range` not `block$`; has `digest` (bytea), `id` (text), optionally `block_time$` (int8); `has_causality_region: false`). Conditionally created when `catalog.use_poi` is true. Excluded from some Layout operations like `find_changes()`. +- Metadata in `subgraphs.subgraph_manifest`, `subgraphs.deployment`, and `subgraphs.head` tables + +## Dump Format + +### Directory layout + +``` +/ + metadata.json -- deployment metadata + per-table state + schema.graphql -- raw GraphQL schema text + subgraph.yaml -- raw subgraph manifest YAML (optional) + / + chunk_000000.parquet -- rows ordered by vid + chunk_000001.parquet -- incremental append + ... + data_sources$/ + chunk_000000.parquet -- dynamic data sources +``` + +One parquet file per entity type (each has a different columnar schema). The `data_sources$` table is dumped alongside entity tables. The `poi2$` table, when present, is a regular entity table in the `Layout` (conditionally created when `catalog.use_poi` is true) and appears as any other entity type directory. Incremental dumps produce new chunk files rather than rewriting existing ones. + +The GraphQL schema and subgraph manifest YAML are stored as separate files rather than embedded in `metadata.json`. This matches the existing dump code in `dump.rs` and keeps the files human-readable and diffable. + +### Parquet schema per entity type + +System columns (always present): +- `vid` -> Int64 +- Immutable entities: `block$` -> Int32 +- Mutable entities: `block_range_start` -> Int32, `block_range_end` -> Int32 (nullable; null = unbounded/current) +- `causality_region` -> Int32 (only if table has it) + +Data columns mapped from `ColumnType` (all 10 variants defined in `relational.rs:1342`): + +| ColumnType | Arrow DataType | Notes | +|--------------------|---------------------------|-------------------------------| +| Boolean | Boolean | | +| Int | Int32 | | +| Int8 | Int64 | | +| Bytes | Binary | Raw bytes | +| BigInt | Utf8 | Arbitrary precision as string | +| BigDecimal | Utf8 | Arbitrary precision as string | +| Timestamp | TimestampMicrosecond(None) | Matches Value::Timestamp | +| String | Utf8 | | +| Enum(EnumType) | Utf8 | String value of enum variant | +| TSVector(FulltextConfig) | **Skip** | Generated; rebuild on restore | + +**List/array columns:** `List(T)` is not a `ColumnType` variant. Whether a column is an array is determined by `Column.is_list()` (delegates to the GraphQL `field_type`). A `[String]` field has `column_type: ColumnType::String` with a list-typed `field_type`. For Arrow mapping, check `column.is_list()` and wrap the base Arrow type in `List`. In `OidValue`, arrays have separate variants (`StringArray`, `BytesArray`, `BoolArray`, `Ints`, `Int8Array`, `BigDecimalArray`, `TimestampArray`). + +Nullability follows the GraphQL schema (non-null fields -> non-nullable Arrow columns). + +### metadata.json + +Contains everything needed to reconstruct the deployment's table structure, plus diagnostic information (health, indexes) captured at dump time. The GraphQL schema and manifest YAML are stored in separate files (`schema.graphql`, `subgraph.yaml`), not embedded here. + +The struct backing this file is `Metadata` (evolved from the existing `Control` struct in `dump.rs`). + +```json +{ + "version": 1, + "deployment": "Qm...", + "network": "mainnet", + + "manifest": { + "spec_version": "1.0.0", + "description": "Optional subgraph description", + "repository": "https://github.com/...", + "features": ["..."], + "entities_with_causality_region": ["EntityType1"], + "history_blocks": 2147483647 + }, + + "earliest_block_number": 12345, + "start_block": { "number": 12345, "hash": "0xabc..." }, + "head_block": { "number": 99999, "hash": "0xdef..." }, + "entity_count": 150000, + + "graft_base": null, + "graft_block": null, + "debug_fork": null, + + "health": { + "failed": false, + "health": "healthy", + "fatal_error": null, + "non_fatal_errors": [] + }, + + "indexes": { + "token": [ + "CREATE INDEX CONCURRENTLY IF NOT EXISTS attr_0_0_id ON sgd.token USING btree (id)" + ] + }, + + "tables": { + "Token": { + "immutable": true, + "has_causality_region": false, + "chunks": [ + { "file": "Token/chunk_000000.parquet", "min_vid": 0, "max_vid": 50000, "row_count": 50000 } + ], + "max_vid": 50000 + }, + "data_sources$": { + "immutable": false, + "has_causality_region": true, + "chunks": [ + { "file": "data_sources$/chunk_000000.parquet", "min_vid": 0, "max_vid": 100, "row_count": 100 } + ], + "max_vid": 100 + } + } +} +``` + +**Field sources:** + +| Field | Source | Code path | +|-------|--------|-----------| +| `manifest.*` | `subgraphs.subgraph_manifest` | `SubgraphManifestEntity` via `deployment_entity()` in `detail.rs` | +| `start_block` | `subgraphs.subgraph_manifest` | `start_block_number`, `start_block_hash` columns; available via `StoredSubgraphManifest` in `detail.rs:542-543`, assembled into `SubgraphDeploymentEntity.start_block` | +| `earliest_block_number` | `subgraphs.deployment` | `SubgraphDeploymentEntity.earliest_block_number` | +| `graft_base`, `graft_block` | `subgraphs.deployment` | `SubgraphDeploymentEntity.graft_base`, `.graft_block` | +| `debug_fork` | `subgraphs.deployment` | `SubgraphDeploymentEntity.debug_fork` | +| `head_block` | `subgraphs.head` | `SubgraphDeploymentEntity.latest_block` | +| `entity_count` | `subgraphs.head` | `DeploymentDetail.entity_count` (i64 in DB, usize in Rust) | +| `health.*` | `subgraphs.deployment` + `subgraph_error` | `SubgraphDeploymentEntity.{failed, health, fatal_error, non_fatal_errors}` | +| `indexes` | `pg_indexes` catalog | `IndexList::load()` → `CreateIndex::to_sql()` (existing code in `dump.rs:163-179`) | +| `network` | `deployment_schemas` | `Site.network` | +| `tables.*` | `Layout.tables` | `Table.{immutable, has_causality_region}` | + +**Notes:** +- `use_bytea_prefix` is not stored in the dump. It is hardcoded to `true` in `create_deployment` (deployment.rs:1302) and will always be set to `true` on restore. +- `health` and `indexes` are point-in-time diagnostic snapshots. They are not used during restore (a restored deployment starts healthy; indexes are auto-created by `Layout::create_relational_schema()`). They are included for inspection and debugging. +- `indexes` are serialized as SQL strings using `CreateIndex::with_nsp("sgd")` + `to_sql(true, true)`, producing `CREATE INDEX CONCURRENTLY IF NOT EXISTS` statements with a normalized `sgd` namespace. +- The `manifest` fields mirror the existing `Manifest` struct in `dump.rs` (derived from `SubgraphManifestEntity`). The `schema` and `raw_yaml` fields of `SubgraphManifestEntity` are written to separate files instead. +- The `poi2$` table, when present, is a regular mutable entity table in `Layout.tables` and appears in the `tables` map like any other entity. It does not need special handling. + +The raw GraphQL schema (in `schema.graphql`) is sufficient to reconstruct the full relational layout via `InputSchema::parse(spec_version, schema, deployment_hash)` → `Layout::new()`. The `InputSchema::parse()` call requires `manifest.spec_version` for version-specific parsing logic. + +## Dump Process + +**Existing code:** There is already a metadata-only dump in `store/postgres/src/relational/dump.rs` (`Layout::dump()`) that writes `control.json`, `schema.graphql`, and `subgraph.yaml`. It is called via `DeploymentStore::dump()` (deployment_store.rs:901) which loads `Layout` + `IndexList` and passes both to `Layout::dump()`. The connection is `AsyncPgConnection` via `pool.get_permitted()`. The new parquet dump extends this to include entity data. + +The existing `Control` struct is renamed to `Metadata` and extended with the fields described above. The existing `Manifest`, `BlockPtr`, `Health`, and `Error` structs in `dump.rs` are reused and extended. + +1. Resolve the deployment (by name, hash, or sgdN) +2. Read deployment metadata from `subgraph_manifest` + `deployment` + `head` tables (via `deployment_entity()` in `detail.rs`) +3. Write `schema.graphql` and `subgraph.yaml` (existing behavior) +4. For each entity type table in `Layout.tables` (sorted by name for determinism; includes `poi2$` when present): + a. Query rows in vid order, batched (adaptive sizing like `VidBatcher`) + b. Convert PG rows directly to Arrow `RecordBatch` (no JSON intermediate) + c. Write batches to parquet file + d. Record chunk info (file path, min_vid, max_vid, row_count) +5. Dump `data_sources$` table (fixed schema, same batch approach; include all DDL columns: `vid`, `block_range`, `causality_region`, `manifest_idx`, `parent`, `id`, `param`, `context`, `done_at`). Note: `parent` and `id` are in the DDL (`private.rs:68-69`) but not in the `DataSourcesTable` struct — dumping them requires raw SQL or extending the struct. +6. Write `metadata.json` atomically (write to tmp file, rename) + +### Incremental append + +- Read existing `metadata.json` to get `max_vid` per entity type +- Query rows with `vid > max_vid` +- Write as new chunk files (`chunk_000001.parquet`, etc.) +- Update metadata atomically + +## Restore Process + +1. Read `metadata.json` and `schema.graphql` +2. Parse schema via `InputSchema::parse(manifest.spec_version, schema_text, deployment_hash)` +3. Create a `Site` entry in `deployment_schemas` (needed for the deployment to be discoverable) +4. Create deployment via `create_deployment(conn, site, DeploymentCreate { .. })` -- this populates three tables: + - `subgraphs.head` (block pointers, entity count -- initially null/0) + - `subgraphs.deployment` (deployment hash, earliest_block, graft info, health) + - `subgraphs.subgraph_manifest` (schema from file, features/spec_version/etc. from metadata) +5. Create tables via `Layout::create_relational_schema()` -- this generates DDL from the parsed schema and creates all entity tables with default indexes +6. Restore `data_sources$` table via DDL from `DataSourcesTable::new().as_ddl()` + batch-insert +7. For each entity type (including `poi2$` if present), read all parquet chunks in order, batch-insert into PG +8. Reset vid sequences to `max_vid + 1` for all entity tables and data_sources$ +9. Update `subgraphs.head` with `head_block.number`, `head_block.hash`, and `entity_count` from dump metadata + +## PG Read Strategy: OidValue-based Dynamic Columns + +Use the existing `dsl::Table::select_cols()` + `DynamicRow` pattern (see `store/postgres/src/relational/dsl.rs` and `store/postgres/src/relational/value.rs`). This already solves dynamic-schema typed extraction through the connection pool: + +1. `select_cols()` builds a typed SELECT using `DynamicSelectClause` for any set of columns +2. Results are `DynamicRow` where `OidValue` dispatches on PG OID at runtime +3. `OidValue` captures all needed types: String, Bytes, Bool, Int, Int8, BigDecimal, Timestamp, plus array variants +4. Convert `OidValue` -> Arrow `ArrayBuilder` (analogous to existing `OidValue` -> `Entity` in `FromOidRow`) + +No JSON, no separate connection. The existing `DeploymentStore::dump()` already uses `AsyncPgConnection` via `pool.get_permitted()`. + +**Block range handling:** Add `OidValue::Int4Range(Bound, Bound)` variant (OID 3904). Diesel already has `FromSql, Pg>` for `(Bound, Bound)` which parses the binary format. ~15 lines of code in `value.rs` + fix the `BLOCK_RANGE_COL` placeholder in `dsl.rs:46-49` (currently `ColumnType::Bytes`, with comment "we can't deserialize in4range"). This resolves the existing TODO at dsl.rs line 294. + +**Key existing code:** +- `dsl::Table::select_cols()` (`store/postgres/src/relational/dsl.rs:305`) +- `OidValue` enum and `FromSql` impl (`store/postgres/src/relational/value.rs:33`) +- `FromOidRow` trait for result deserialization (`value.rs:206`) +- `selected_columns()` for building column list with system columns (`dsl.rs:246`) + +### Vid continuity on restore + +Preserve original vid values. Needed for incremental consistency and simpler to implement. Reset vid sequence to `max_vid + 1` after restore. + +## Where Code Lives + +- `store/postgres/src/relational/dump.rs` -- **Existing** metadata-only dump. Contains `Manifest`, `BlockPtr`, `Error`, `Health`, `Control` structs. `Control` will be renamed to `Metadata` and extended. The existing helper structs (`Manifest`, `BlockPtr`, `Health`, `Error`) are reused. Called via `DeploymentStore::dump()` → `Layout::dump()`. +- `store/postgres/src/parquet/` -- New module for parquet read/write/schema mapping +- `node/src/manager/commands/dump.rs` -- **Existing** CLI command skeleton (resolves deployment, calls `SubgraphStore::dump()`) +- `node/src/manager/commands/restore.rs` -- New CLI command for restore +- Expose via `command_support` in `store/postgres/src/lib.rs` + +## Dependencies to Add + +- `parquet = "=57.3.0"` (same version as existing `arrow`) to workspace and `store/postgres` +- `arrow` workspace dep to `store/postgres` + +## Existing Code to Reuse + +- `Layout`, `Table`, `Column`, `ColumnType` (`store/postgres/src/relational.rs`) -- schema introspection; `poi2$` is in Layout as a mutable entity table (conditionally, when `catalog.use_poi` is true) +- `Column.is_list()` (`relational.rs:1574`) -- determines if a column is an array type (delegates to GraphQL `field_type.is_list()`) +- `DataSourcesTable` (`store/postgres/src/dynds/private.rs`) -- `data_sources$` DDL via `as_ddl()` method; note that `parent` and `id` columns are in the DDL but not in the struct's typed fields +- `VidBatcher` (`store/postgres/src/vid_batcher.rs`) -- adaptive batch iteration using PG histogram statistics +- `copy.rs` pattern -- progress reporting, batch operation lifecycle; already handles both entity tables and `data_sources$` copying +- `InputSchema::parse(spec_version, raw, id)` (`graph/src/schema/input/mod.rs:965`) -- schema reconstruction from text +- `DeploymentSearch` (`node/src/manager/deployment.rs`) -- CLI deployment resolution (supports name, Qm hash, sgdN namespace) +- `create_deployment` (`store/postgres/src/deployment.rs:1224`) -- populates `head`, `deployment`, and `subgraph_manifest` tables +- `DeploymentCreate` + `SubgraphManifestEntity` (`graph/src/data/subgraph/schema.rs:103`) -- structs needed by `create_deployment` +- `deployment_entity()` (`store/postgres/src/detail.rs`) -- reads deployment metadata into `SubgraphDeploymentEntity`; note that `start_block_*` is in the DB table (`StoredSubgraphManifest`) but only partially exposed through `SubgraphDeploymentEntity` +- `IndexList::load()` + `CreateIndex::to_sql()` (`store/postgres/src/relational/index.rs`) -- loads and serializes indexes +- Existing `dump.rs` (`store/postgres/src/relational/dump.rs`) -- metadata serialization types: `Manifest`, `BlockPtr`, `Health`, `Error`, `Control` (to be renamed `Metadata`) + +## Implementation Order + +1. Schema mapping + metadata types (foundation, unit-testable) +2. Parquet writer (dump from PG) + graphman `dump` command +3. Incremental append support +4. Parquet reader (restore to PG) + graphman `restore` command +5. Ongoing dump integration (run as part of graph-node, not just CLI) diff --git a/docs/specs/gnd-cli-expansion.md b/docs/specs/gnd-cli-expansion.md new file mode 100644 index 00000000000..7d0cf0d37fe --- /dev/null +++ b/docs/specs/gnd-cli-expansion.md @@ -0,0 +1,891 @@ +# Spec: gnd CLI Expansion + +This spec describes the expansion of `gnd` (Graph Node Dev) to include functionality currently provided by the TypeScript-based `graph-cli`. The goal is a drop-in replacement for subgraph development workflows, implemented in Rust and integrated with graph-node's existing infrastructure. + +## Goals + +- **Drop-in replacement**: Same commands, flags, output format, and exit codes as `graph-cli` +- **Identical code generation**: AssemblyScript output must be byte-for-byte identical after formatting +- **Ethereum only**: Initial scope limited to Ethereum protocol +- **Reuse graph-node internals**: Leverage existing manifest parsing, validation, IPFS client, etc. + +## Non-Goals + +- Multi-protocol support (NEAR, Cosmos, Arweave, Substreams) - future work +- Rollout/migration plan from TS CLI +- Performance optimization beyond reasonable behavior + +## Commands + +### Command Matrix + +| Command | TS CLI | gnd | Notes | +| --------- | ------ | --- | ------------------------------------------ | +| `codegen` | Yes | Yes | Generate AssemblyScript types | +| `build` | Yes | Yes | Compile to WASM | +| `deploy` | Yes | Yes | Deploy to Graph Node | +| `init` | Yes | Yes | Scaffold new subgraph | +| `add` | Yes | Yes | Add datasource to existing subgraph | +| `remove` | Yes | Yes | Unregister subgraph name | +| `create` | Yes | Yes | Register subgraph name | +| `auth` | Yes | Yes | Set deploy key | +| `publish` | Yes | Yes | Publish to decentralized network | +| `test` | Yes | Yes | Run Matchstick tests | +| `clean` | Yes | Yes | Remove build artifacts | +| `dev` | No | Yes | Run graph-node in dev mode (existing gnd) | +| `local` | Yes | No | Skipped - use existing test infrastructure | +| `node` | Yes | No | Skipped - use graphman for node management | + +### Known Differences from TS CLI + +**Commands:** + +1. **`local` command**: Not implemented. Users should use existing integration test infrastructure. +2. **`node` subcommand**: Not implemented. Use `graphman` for node management operations. +3. **`--uncrashable` flag on codegen**: Not implemented. Float Capital's uncrashable helper generation is a niche third-party feature. +4. **Debug output**: Uses `RUST_LOG` environment variable instead of `DEBUG=graph-cli:*`. + +**Code generation** (intentional, documented in verification tests): 5. **Int8 import**: gnd always imports `Int8` for simplicity, even when not used. 6. **Trailing commas**: gnd uses trailing commas in multi-line constructs. 7. **2D array accessors**: gnd uses correct `toStringMatrix()` while graph-cli has a bug using `toStringArray()` for 2D GraphQL array types. 8. **Tuple/struct component names**: gnd correctly extracts component names from raw ABI JSON, while graph-cli's ethabi parser loses nested component names for some tuple types. 9. **Subgraph data source file naming**: gnd uses the data source name (`subgraph-SourceName.ts`) instead of IPFS hash (`subgraph-QmHash.ts`) for stable import paths that don't break when source subgraphs are redeployed. + +## CLI Interface + +### Binary and Invocation + +``` +gnd [options] [arguments] +``` + +The binary name is `gnd`. All graph-cli commands become gnd subcommands. + +### Version Output + +``` +$ gnd --version +gnd 0.1.0 (graph-cli compatible: 0.98.1) +``` + +Shows both gnd version and the graph-cli version it emulates. + +### Flag Compatibility + +All flags must match the TS CLI exactly: + +- Same long names (`--output-dir`) +- Same short names (`-o`) +- Same defaults +- Same validation behavior + +Reference: Each command section below lists flags with references to TS CLI source. + +### Output Format + +Output must match TS CLI format exactly, including: + +- Spinner/progress indicators +- Success checkmarks (`✔`) +- Step descriptions +- File paths displayed +- Error formatting (information must match; exact wording may differ) + +Reference: `/packages/cli/src/command-helpers/spinner.ts` + +### Exit Codes + +Exit codes must match TS CLI behavior: + +- `0`: Success +- `1`: Error (validation, compilation, deployment failure, etc.) + +### Configuration Files + +Use same paths and formats as TS CLI: + +| File | Path | Purpose | +| -------------- | ------------------------------ | ----------------------------- | +| Auth tokens | `~/.graphprotocol/` | Deploy keys and access tokens | +| Network config | `networks.json` (project root) | Network-specific addresses | + +Reference: `/packages/cli/src/command-helpers/auth.ts` + +## Command Specifications + +### `gnd codegen` + +Generates AssemblyScript types from subgraph manifest. + +**Usage:** + +``` +gnd codegen [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--output-dir` | `-o` | `generated/` | Output directory for generated types | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--watch` | `-w` | `false` | Regenerate on file changes | +| `--ipfs` | `-i` | `https://api.thegraph.com/ipfs/` | IPFS node URL | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Load and validate manifest +2. Apply migrations (unless `--skip-migrations`) +3. Assert minimum API version (0.0.5) and graph-ts version (0.25.0) +4. Generate entity classes from GraphQL schema +5. Generate ABI bindings for each contract +6. Generate template datasource bindings +7. Format output with prettier +8. Write to output directory + +**Output Structure:** + +``` +generated/ +├── schema.ts # Entity classes +├── / +│ └── .ts # ABI bindings +├── templates/ +│ └── / +│ └── .ts # Template ABI bindings +└── subgraph-.ts # Entity types for subgraph data sources +``` + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/codegen.ts` +- Type generator: `/packages/cli/src/type-generator.ts` +- Schema codegen: `/packages/cli/src/codegen/schema.ts` +- ABI codegen: `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +### `gnd build` + +Compiles subgraph to WASM. + +**Usage:** + +``` +gnd build [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--output-dir` | `-o` | `build/` | Output directory | +| `--output-format` | `-t` | `wasm` | Output format: `wasm` or `wast` | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--watch` | `-w` | `false` | Rebuild on file changes | +| `--ipfs` | `-i` | | IPFS node URL (uploads if provided) | +| `--network` | | | Network to use from networks.json | +| `--network-file` | | `networks.json` | Path to networks config | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Run codegen (unless types already exist) +2. Apply migrations (unless `--skip-migrations`) +3. Resolve network-specific values from networks.json +4. Shell out to `asc` (AssemblyScript compiler) for each mapping +5. Copy ABIs and schema to build directory +6. Copy template ABIs to build/templates// directories +7. Generate build manifest +8. Optionally upload to IPFS + +**Build Output Structure:** + +``` +build/ +├── schema.graphql +├── subgraph.yaml +├── / +│ ├── .wasm +│ └── .json # ABI +└── templates/ + └── / + ├── .wasm + └── .json # ABI +``` + +**Note:** When multiple data sources or templates share the same mapping.ts file, gnd compiles it once and copies the resulting WASM to all required output locations. + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/build.ts` +- Compiler: `/packages/cli/src/compiler/index.ts` + +### `gnd deploy` + +Deploys subgraph to a Graph Node. + +**Usage:** + +``` +gnd deploy [subgraph-name] [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-name`: Name to deploy as (e.g., `user/subgraph`) +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--product` | | | Product: `subgraph-studio` or `hosted-service` | +| `--studio` | | `false` | Shorthand for `--product subgraph-studio` | +| `--node` | `-g` | | Graph Node URL | +| `--ipfs` | `-i` | | IPFS node URL | +| `--access-token` | | | Access token for authentication | +| `--deploy-key` | | | Deploy key (alias for access-token) | +| `--version-label` | `-l` | | Version label | +| `--headers` | | | Additional HTTP headers (JSON) | +| `--debug-fork` | | | Fork subgraph for debugging | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--network` | | | Network from networks.json | +| `--network-file` | | `networks.json` | Path to networks config | +| `--output-dir` | `-o` | `build/` | Build output directory | +| `--help` | `-h` | | Show help | + +**Deploy Targets:** + +- Local Graph Node (via `--node` and `--ipfs`) +- Subgraph Studio (`--product subgraph-studio` or `--studio`) +- Hosted Service (`--product hosted-service`) +- Decentralized network (via `publish` command) + +**Behavior:** + +1. Build subgraph (runs build command) +2. Upload build artifacts to IPFS +3. Send deployment request to Graph Node via JSON-RPC + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/deploy.ts` + +### `gnd init` + +Scaffolds a new subgraph project. + +**Usage:** + +``` +gnd init [directory] +``` + +**Arguments:** + +- `directory`: Directory to create subgraph in + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--protocol` | | | Protocol: `ethereum` | +| `--product` | | | Product for deployment | +| `--studio` | | `false` | Initialize for Subgraph Studio | +| `--from-contract` | | | Contract address to generate from | +| `--from-example` | | | Example subgraph to clone | +| `--contract-name` | | | Name for the contract | +| `--index-events` | | `false` | Index all contract events | +| `--start-block` | | | Start block for indexing | +| `--network` | | `mainnet` | Network name | +| `--abi` | | | Path to ABI file | +| `--spkg` | | | Path to Substreams package | +| `--allow-simple-name` | | `false` | Allow simple subgraph names | +| `--help` | `-h` | | Show help | + +**Example Subgraphs:** + +The `--from-example ` flag clones from `https://github.com/graphprotocol/graph-tooling/tree/main/examples`. +The example name is **required** - no default is provided. Available examples include: +`ethereum-gravatar`, `aggregations`, `ethereum-basic-event-handlers`, `substreams-powered-subgraph`, etc. +Legacy name `ethereum/gravatar` is automatically converted to `ethereum-gravatar`. + +**Behavior:** + +1. Prompt for missing information (protocol, network, contract, etc.) +2. Fetch ABI from Etherscan/Sourcify if `--from-contract` and no `--abi` +3. Generate scaffold: + - `subgraph.yaml` manifest + - `schema.graphql` with entities for events + - `src/mapping.ts` with event handlers + - `package.json` with dependencies + - `tsconfig.json` + - ABIs directory +4. Optionally initialize git repository +5. Install dependencies + +**External APIs:** + +- Etherscan API: Fetch verified contract ABIs +- Sourcify API: Fetch verified contract ABIs (fallback) +- Network registry: `@pinax/graph-networks-registry` for chain configuration + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/init.ts` +- Scaffold: `/packages/cli/src/scaffold/index.ts` +- Schema generation: `/packages/cli/src/scaffold/schema.ts` +- Mapping generation: `/packages/cli/src/scaffold/mapping.ts` +- Etherscan client: `/packages/cli/src/command-helpers/contracts.ts` + +### `gnd add` + +Adds a new datasource to an existing subgraph. + +**Usage:** + +``` +gnd add
[subgraph-manifest] +``` + +**Arguments:** + +- `address`: Contract address +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--abi` | | | Path to ABI file | +| `--contract-name` | | | Name for the contract | +| `--merge-entities` | | `false` | Merge with existing entities | +| `--network-file` | | `networks.json` | Path to networks config | +| `--start-block` | | | Start block | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/add.ts` + +### `gnd create` + +Registers a subgraph name with a Graph Node. + +**Usage:** + +``` +gnd create +``` + +**Arguments:** + +- `subgraph-name`: Name to register + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--node` | `-g` | | Graph Node URL | +| `--access-token` | | | Access token | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/create.ts` + +### `gnd remove` + +Unregisters a subgraph name from a Graph Node. + +**Usage:** + +``` +gnd remove +``` + +**Arguments:** + +- `subgraph-name`: Name to unregister + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--node` | `-g` | | Graph Node URL | +| `--access-token` | | | Access token | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/remove.ts` + +### `gnd auth` + +Sets the deploy key for a Graph Node. + +**Usage:** + +``` +gnd auth +``` + +**Arguments:** + +- `deploy-key`: Deploy key to store + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--product` | | | Product: `subgraph-studio` or `hosted-service` | +| `--studio` | | `false` | Shorthand for subgraph-studio | +| `--help` | `-h` | | Show help | + +**Behavior:** +Stores deploy key in `~/.graphprotocol/` for later use by deploy/publish commands. + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/auth.ts` +- Auth helpers: `/packages/cli/src/command-helpers/auth.ts` + +### `gnd publish` + +Publishes a subgraph to The Graph's decentralized network. + +**Usage:** + +``` +gnd publish [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--subgraph-id` | | | Subgraph ID to publish to | +| `--ipfs` | `-i` | | IPFS node URL | +| `--protocol-network` | | | Protocol network (e.g., `arbitrum-one`) | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/publish.ts` + +### `gnd test` + +Runs Matchstick tests for the subgraph. + +**Usage:** + +``` +gnd test [datasource] +``` + +**Arguments:** + +- `datasource`: Specific datasource to test (optional) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--coverage` | `-c` | `false` | Run with coverage | +| `--docker` | `-d` | `false` | Run in Docker container | +| `--force` | `-f` | `false` | Force recompilation | +| `--logs` | `-l` | `false` | Show logs | +| `--recompile` | `-r` | `false` | Recompile before testing | +| `--version` | `-v` | | Matchstick version | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Download Matchstick binary (if not present) +2. Shell out to Matchstick with appropriate flags +3. Report test results + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/test.ts` + +### `gnd clean` + +Removes build artifacts and generated files. + +**Usage:** + +``` +gnd clean +``` + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--codegen-dir` | | `generated/` | Codegen output directory | +| `--build-dir` | | `build/` | Build output directory | +| `--help` | `-h` | | Show help | + +**Behavior:** +Removes `generated/` and `build/` directories (or custom paths if specified). + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/clean.ts` + +### `gnd dev` + +Runs graph-node in development mode with file watching. + +This is the existing `gnd` functionality, preserved as a subcommand. The implementation can be adjusted to fit the new subcommand structure. + +**Usage:** + +``` +gnd dev [options] +``` + +**Flags:** +Preserve existing gnd flags, adjusted as needed for subcommand structure. + +## Code Generation + +Code generation is the most complex component and must produce byte-for-byte identical output to the TS CLI (after prettier formatting). + +### Generated File Types + +#### 1. Entity Classes (`schema.ts`) + +Generated from GraphQL schema. Each entity type becomes an AssemblyScript class with: + +- Constructor +- Static `load(id)` method +- Static `loadInBlock(id)` method +- `save()` method +- Getters/setters for each field +- Proper type mappings (GraphQL → AssemblyScript) + +**TS CLI Reference:** `/packages/cli/src/codegen/schema.ts` + +#### 2. ABI Bindings (`.ts`) + +Generated from contract ABI. Includes: + +- Event classes with typed parameters +- Function call result classes +- Contract class with typed call methods +- Proper Ethereum type mappings + +**Tuple/Struct Handling:** +When an event or function has tuple parameters with named components (e.g., a struct), gnd generates proper struct classes with named getters: + +```typescript +class AssetTransfer__ParamsAssetStruct extends ethereum.Tuple { + get addr(): Address { + return this[0].toAddress(); + } + get amount(): BigInt { + return this[1].toBigInt(); + } + get active(): boolean { + return this[2].toBoolean(); + } +} +``` + +gnd parses the raw ABI JSON to extract component names, which ethabi loses during parsing. + +**TS CLI Reference:** `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +#### 3. Template Bindings + +Generated for template datasources with the same structure as ABI bindings. + +**TS CLI Reference:** `/packages/cli/src/codegen/template.ts` + +#### 4. Subgraph Data Source Bindings (`subgraph-.ts`) + +Generated for `kind: subgraph` data sources. When a manifest contains a subgraph data source: + +```yaml +dataSources: + - kind: subgraph + name: SourceSubgraph + source: + address: "QmRWTEejPDDwALaquFGm6X2GBbbh5osYDXwCRRkoZ6KQhb" +``` + +gnd will: + +1. Validate that all subgraph data source names are unique (error if duplicates found) +2. Fetch the referenced subgraph's manifest from IPFS +3. Extract and fetch the schema from the manifest +4. Generate entity types (without store methods) to `generated/subgraph-{DataSourceName}.ts` + +**Note:** gnd uses the data source name (e.g., `SourceSubgraph`) for the generated filename rather than the IPFS hash. This produces stable import paths like `import { Entity } from '../generated/subgraph-SourceSubgraph'` that don't break when the source subgraph is redeployed with a new hash. This differs from graph-cli which uses the IPFS hash. + +The generated types include entity classes with getters for all fields, but no `save()`, `load()`, or `loadInBlock()` methods since these are read-only types from the source subgraph. + +**TS CLI Reference:** `/packages/cli/src/type-generator.ts` lines 72-136 + +### Type Mappings + +#### GraphQL → AssemblyScript + +| GraphQL | AssemblyScript | +| ---------------- | -------------- | +| `ID` | `string` | +| `String` | `string` | +| `Int` | `i32` | +| `BigInt` | `BigInt` | +| `BigDecimal` | `BigDecimal` | +| `Bytes` | `Bytes` | +| `Boolean` | `boolean` | +| `[T]` | `Array` | +| Entity reference | `string` (ID) | + +**TS CLI Reference:** `/packages/cli/src/codegen/schema.ts` (look for type mapping functions) + +#### Ethereum ABI → AssemblyScript + +| Solidity | AssemblyScript | +| --------- | --------------- | +| `address` | `Address` | +| `bool` | `boolean` | +| `bytes` | `Bytes` | +| `bytesN` | `Bytes` | +| `intN` | `BigInt` | +| `uintN` | `BigInt` | +| `string` | `string` | +| `T[]` | `Array` | +| tuple | Generated class | + +**TS CLI Reference:** `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +### API Version Handling + +Different `apiVersion` values in the manifest affect code generation. gnd must support all versions that the TS CLI supports. + +**TS CLI Reference:** `/packages/cli/src/codegen/` (version-specific logic throughout) + +### Formatting + +All generated code must be formatted with prettier before writing: + +- Shell out to `prettier` with same configuration as TS CLI +- Parser: `typescript` + +## Migrations + +Migrations update older manifest formats to newer versions. gnd must implement all migrations that TS CLI supports. + +**Migration Chain:** + +``` +0.0.1 → 0.0.2 → 0.0.3 → 0.0.4 → 0.0.5 → ... → current +``` + +**TS CLI Reference:** `/packages/cli/src/migrations/` + +Each migration is a transformation function that: + +1. Checks manifest version +2. Applies necessary changes +3. Updates version number + +## External Dependencies + +### Runtime Dependencies (shell out) + +| Tool | Purpose | Required | +| ------------ | ----------------------- | ------------- | +| `asc` | AssemblyScript compiler | For `build` | +| `prettier` | Code formatting | For `codegen` | +| `matchstick` | Test runner | For `test` | + +### Network APIs + +| API | Purpose | +| -------------------------------- | --------------------------------------- | +| Etherscan | Fetch verified contract ABIs | +| Sourcify | Fetch verified contract ABIs (fallback) | +| `@pinax/graph-networks-registry` | Network configuration (chain IDs, etc.) | + +The network registry should be fetched at runtime to get current network configurations. + +### graph-node Reuse + +| Component | graph-node Location | Purpose | +| ------------------- | ------------------------------------- | --------------------------- | +| Manifest parsing | `graph/src/data/subgraph/` | Load subgraph.yaml | +| Manifest validation | `graph/src/data/subgraph/` | Validate manifest structure | +| GraphQL schema | `graph/src/schema/input/` | Parse schema.graphql | +| IPFS client | `graph/src/ipfs/` | Upload to IPFS | +| Link resolver | `graph/src/components/link_resolver/` | Resolve file references | +| File watcher | `gnd/src/watcher.rs` | Watch mode | + +Refactor graph-node components as needed to make them reusable. + +## Module Structure + +``` +gnd/src/ +├── main.rs # Entry point, clap setup +├── lib.rs +├── formatter.rs # Prettier integration for code formatting +├── prompt.rs # Interactive prompts (network selection, etc.) +├── watcher.rs # File watcher for --watch modes +├── commands/ +│ ├── mod.rs +│ ├── add.rs +│ ├── auth.rs +│ ├── build.rs +│ ├── clean.rs +│ ├── codegen.rs +│ ├── create.rs +│ ├── deploy.rs +│ ├── dev.rs # Existing gnd functionality +│ ├── init.rs +│ ├── publish.rs +│ ├── remove.rs +│ └── test.rs +├── codegen/ +│ ├── mod.rs +│ ├── abi.rs # ABI binding generation +│ ├── schema.rs # Entity class generation +│ ├── template.rs # Template binding generation +│ ├── types.rs # Type conversion utilities +│ └── typescript.rs # AST builders for TypeScript/AssemblyScript +├── scaffold/ +│ ├── mod.rs +│ ├── manifest.rs # Generate subgraph.yaml +│ ├── mapping.rs # Generate mapping.ts +│ └── schema.rs # Generate schema.graphql +├── migrations/ +│ ├── mod.rs +│ ├── api_version.rs # API version migrations +│ ├── spec_version.rs # Spec version migrations +│ └── versions.rs # Version definitions +├── compiler/ +│ ├── mod.rs +│ └── asc.rs # Shell out to asc +├── services/ +│ ├── mod.rs +│ ├── contract.rs # ABI fetching (Etherscan, Blockscout, Sourcify, registry) +│ ├── graph_node.rs # Graph Node JSON-RPC client +│ └── ipfs.rs # IPFS client for uploads +├── config/ +│ ├── mod.rs +│ └── networks.rs # networks.json handling +└── output/ + ├── mod.rs + └── spinner.rs # Progress/spinner output +``` + +## Testing + +### Current Status + +- **166 unit tests** passing (`cargo test -p gnd --lib`) +- **12 codegen verification tests** passing (`cargo test -p gnd --test codegen_verification`) +- **7 CLI command tests** passing (`cargo test -p gnd --test cli_commands`) +- Test fixtures from graph-cli validation tests in `gnd/tests/fixtures/codegen_verification/` + +### Test Strategy + +1. **Unit tests**: Test individual functions (type mappings, migrations, etc.) +2. **Snapshot tests**: Compare generated output against TS CLI output +3. **Integration tests**: End-to-end command execution + +### Test Corpus + +Use the same test corpus as graph-cli. Tests must cover at least everything the TS CLI tests cover. + +**TS CLI Test Location:** `/packages/cli/tests/` + +### Codegen Verification Tests + +Located in `gnd/tests/codegen_verification.rs`, these tests verify compatibility with graph-cli: + +1. Copy fixture to temp directory (excluding `generated/`) +2. Run `gnd codegen` on the fixture +3. Compare output against expected `generated/` directory +4. Allow known differences (Int8 import, trailing commas, 2D array fix) + +Fixtures can be regenerated with `./tests/fixtures/regenerate.sh`. + +### Edge Cases + +When edge case bugs are discovered in the TS CLI, gnd should fix them rather than replicate them. Document any behavioral differences that result from bug fixes. + +### CLI Integration Tests + +Located in `tests/tests/gnd_cli_tests.rs`, these tests verify gnd works as a drop-in replacement for graph-cli by running the integration test suite with `GRAPH_CLI` environment variable pointing to the gnd binary. + +**How it works:** + +- The integration test infrastructure uses `CONFIG.graph_cli` for deployment commands +- Setting `GRAPH_CLI=../target/debug/gnd` makes tests use gnd instead of graph-cli +- `Subgraph::deploy()` calls `gnd codegen`, `gnd create`, `gnd deploy` + +**Commands tested:** + +- `gnd codegen` - Generate AssemblyScript types +- `gnd create` - Register subgraph name with Graph Node +- `gnd deploy` - Deploy subgraph to Graph Node + +**Running:** + +```bash +just test-gnd-cli +``` + +### Standalone Command Tests + +Located in `gnd/tests/cli_commands.rs`, these tests verify commands that don't require a running Graph Node: + +**Commands tested:** + +- `gnd init` - Scaffold generation (--from-example, --from-contract, --from-subgraph) +- `gnd add` - Add datasource to existing subgraph +- `gnd build` - WASM compilation + +**Running:** + +```bash +just test-gnd-commands +``` + +## Dependencies to Add + +### Cargo.toml additions + +```toml +[dependencies] +# CLI framework (already present) +clap = { version = "...", features = ["derive"] } + +# Interactive prompts (for init) +inquire = "..." + +# HTTP client (for Etherscan, Sourcify, registry) +reqwest = { version = "...", features = ["json"] } + +# Progress/spinner output +indicatif = "..." + +# Template rendering (for scaffold) +minijinja = "..." + +# JSON handling +serde_json = "..." +``` + +## Open Questions + +None at this time. All major decisions have been made. + +## References + +- TS CLI repository: https://github.com/graphprotocol/graph-tooling +- TS CLI source: `/packages/cli/src/` +- Local checkout: `/home/lutter/code/subgraphs/graph-cli` +- Original gnd expansion plan: `/LOCAL/plans/gnd-cli-expansion.md` diff --git a/docs/specs/runner-refactor.md b/docs/specs/runner-refactor.md new file mode 100644 index 00000000000..ee36ebe3342 --- /dev/null +++ b/docs/specs/runner-refactor.md @@ -0,0 +1,380 @@ +# Subgraph Runner Simplification Spec + +## Problem Statement + +`core/src/subgraph/runner.rs` is complex and hard to modify. Key issues: + +1. **Duplicated trigger processing** (lines 616-656 vs 754-790): Nearly identical loops +2. **Control flow confusion**: Nested loops in `run_inner` with 6 exit paths +3. **State management**: Mixed patterns (mutable fields, `std::mem::take`, drains) +4. **`process_block` monolith**: ~260 lines handling triggers, dynamic DS, offchain, persistence + +## Design Decisions + +| Aspect | Decision | +|--------|----------| +| Control flow | Enum-based FSM for full runner lifecycle | +| Trigger processing | New `TriggerRunner` component | +| Block processing | Pipeline with explicitly defined stages | +| State management | Mutable accumulator with checkpoints | +| Breaking changes | Moderate (internal APIs can change) | + +## Target Architecture + +### 1. Runner State Machine + +Replace nested loops in `run_inner` with an explicit enum FSM covering the full lifecycle: + +```rust +enum RunnerState { + /// Initial state, ready to start block stream + Initializing, + + /// Block stream active, waiting for next event + AwaitingBlock { + block_stream: Cancelable>>, + }, + + /// Processing a block through the pipeline + ProcessingBlock { + block: BlockWithTriggers, + cursor: FirehoseCursor, + }, + + /// Handling a revert event + Reverting { + to_ptr: BlockPtr, + cursor: FirehoseCursor, + }, + + /// Restarting block stream (new filters, store restart, etc.) + Restarting { + reason: RestartReason, + }, + + /// Terminal state + Stopped { + reason: StopReason, + }, +} + +enum RestartReason { + DynamicDataSourceCreated, + DataSourceExpired, + StoreError, + PossibleReorg, +} + +enum StopReason { + MaxEndBlockReached, + Canceled, + Unassigned, +} +``` + +The main loop becomes: + +```rust +async fn run(mut self) -> Result<(), SubgraphRunnerError> { + loop { + self.state = match self.state { + RunnerState::Initializing => self.initialize().await?, + RunnerState::AwaitingBlock { stream } => self.await_block(stream).await?, + RunnerState::ProcessingBlock { block, cursor } => { + self.process_block(block, cursor).await? + } + RunnerState::Reverting { to_ptr, cursor } => { + self.handle_revert(to_ptr, cursor).await? + } + RunnerState::Restarting { reason } => self.restart(reason).await?, + RunnerState::Stopped { reason } => return self.finalize(reason).await, + }; + } +} +``` + +### 2. Block Processing Pipeline + +Replace the `process_block` monolith with explicit stages: + +```rust +/// Pipeline stages for block processing +mod pipeline { + pub struct TriggerMatchStage; + pub struct TriggerExecuteStage; + pub struct DynamicDataSourceStage; + pub struct OffchainTriggerStage; + pub struct PersistStage; +} + +/// Result of block processing pipeline +pub struct BlockProcessingResult { + pub action: Action, + pub block_state: BlockState, +} + +impl SubgraphRunner { + async fn process_block( + &mut self, + block: BlockWithTriggers, + cursor: FirehoseCursor, + ) -> Result { + let block = Arc::new(block.block); + let triggers = block.trigger_data; + + // Stage 1: Match triggers to hosts and decode + let runnables = self.match_triggers(&block, triggers).await?; + + // Stage 2: Execute triggers (unified for initial + dynamic DS) + let mut block_state = self.execute_triggers(&block, runnables).await?; + + // Checkpoint before dynamic DS processing + let checkpoint = block_state.checkpoint(); + + // Stage 3: Process dynamic data sources (loop until none created) + block_state = self.process_dynamic_data_sources(&block, &cursor, block_state).await?; + + // Stage 4: Handle offchain triggers + let offchain_result = self.process_offchain_triggers(&block, &mut block_state).await?; + + // Stage 5: Persist to store + self.persist_block_state(block_state, offchain_result).await?; + + // Determine next state + Ok(self.determine_next_state()) + } +} +``` + +### 3. Error Handling Strategy + +Consolidate scattered error handling into explicit classification: + +```rust +/// Unified error classification for trigger processing +pub enum ProcessingErrorKind { + /// Stop processing, persist PoI only + Deterministic(anyhow::Error), + /// Retry with backoff, attempt to unfail + NonDeterministic(anyhow::Error), + /// Restart block stream cleanly (don't persist) + PossibleReorg(anyhow::Error), +} + +impl ProcessingError { + /// Classify error once, use classification throughout + pub fn kind(&self) -> ProcessingErrorKind { ... } + + /// Whether this error should stop processing the current block + pub fn should_stop_processing(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Deterministic(_)) + } + + /// Whether this error requires a clean restart + pub fn should_restart(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::PossibleReorg(_)) + } + + /// Whether this error is retryable with backoff + pub fn is_retryable(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::NonDeterministic(_)) + } +} +``` + +**Key Invariant (must be preserved):** +``` +Deterministic → Stop processing block, persist PoI only +NonDeterministic → Retry with backoff +PossibleReorg → Restart cleanly (don't persist) +``` + +Currently this logic is scattered across: +- `process_block` early return for PossibleReorg (line 664-677) +- Dynamic data sources error mapping (line 792-802) +- `transact_block_state` (line 405-430) +- `handle_offchain_triggers` (line 1180-1190) + +Consolidating into helper methods eliminates these scattered special cases. + +### 4. TriggerRunner Component + +Extract trigger execution into a dedicated component: + +```rust +/// Handles matching, decoding, and executing triggers +pub struct TriggerRunner<'a, C: Blockchain, T: RuntimeHostBuilder> { + decoder: &'a Decoder, + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a SubgraphMetrics, + debug_fork: &'a Option>, + instrument: bool, +} + +impl<'a, C, T> TriggerRunner<'a, C, T> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// Execute triggers against hosts, accumulating state + pub async fn execute( + &self, + block: &Arc, + runnables: Vec>, + mut block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &PoICausalityRegion, + ) -> Result { + for runnable in runnables { + block_state = self.processor + .process_trigger( + self.logger, + runnable.hosted_triggers, + block, + block_state, + proof_of_indexing, + causality_region, + self.debug_fork, + self.metrics, + self.instrument, + ) + .await + .map_err(|e| e.add_trigger_context(&runnable.trigger))?; + } + Ok(block_state) + } +} +``` + +This eliminates the duplicated loops (lines 616-656 and 754-790). + +### 5. State Management with Checkpoints + +**Explicit Input/Output Types for Pipeline Stages:** + +```rust +/// Input to trigger processing - makes dependencies explicit +struct TriggerProcessingContext<'a> { + block: &'a Arc, + proof_of_indexing: &'a SharedProofOfIndexing, + causality_region: &'a PoICausalityRegion, +} + +/// Output from trigger processing - makes results explicit +struct TriggerProcessingResult { + block_state: BlockState, + restart_needed: bool, +} +``` + +**Add checkpoint capability to `BlockState` for rollback scenarios:** + +```rust +impl BlockState { + /// Create a lightweight checkpoint for rollback + pub fn checkpoint(&self) -> BlockStateCheckpoint { + BlockStateCheckpoint { + created_data_sources_count: self.created_data_sources.len(), + persisted_data_sources_count: self.persisted_data_sources.len(), + // Note: entity_cache changes cannot be easily checkpointed + // Rollback clears the cache (acceptable per current behavior) + } + } + + /// Restore state to checkpoint (partial rollback) + pub fn restore(&mut self, checkpoint: BlockStateCheckpoint) { + self.created_data_sources.truncate(checkpoint.created_data_sources_count); + self.persisted_data_sources.truncate(checkpoint.persisted_data_sources_count); + // Entity cache is cleared on rollback (matches current behavior) + } +} +``` + +### 6. File Structure + +``` +core/src/subgraph/ +├── runner.rs # Main SubgraphRunner with FSM +├── runner/ +│ ├── mod.rs +│ ├── state.rs # RunnerState enum and transitions +│ ├── pipeline.rs # Pipeline stage definitions +│ └── trigger_runner.rs # TriggerRunner component +├── context.rs # IndexingContext (unchanged) +├── inputs.rs # IndexingInputs (unchanged) +└── state.rs # IndexingState (unchanged) +``` + +## Deferred Concerns + +### Fishy Block Refetch (Preserve Behavior) + +The TODO at lines 721-729 notes unclear behavior around block refetching in the dynamic DS loop. The restructure preserves this behavior without attempting to fix it. Investigate separately. + +## Key Interfaces + +### RunnerState Transitions + +``` +Initializing ──────────────────────────────────┐ + │ │ + v │ +AwaitingBlock ◄─────────────────────────────────┤ + │ │ + ├── ProcessBlock event ──► ProcessingBlock │ + │ │ │ + │ ├── success ┼──► AwaitingBlock + │ │ │ + │ └── restart ┼──► Restarting + │ │ + ├── Revert event ──────────► Reverting ────┤ + │ │ + ├── Error ─────────────────► Restarting ───┤ + │ │ + └── Cancel/MaxBlock ───────► Stopped │ + │ +Restarting ─────────────────────────────────────┘ +``` + +### Pipeline Data Flow + +``` +BlockWithTriggers + │ + v +┌──────────────────┐ +│ TriggerMatchStage│ ─► Vec +└──────────────────┘ + │ + v +┌────────────────────┐ +│ TriggerExecuteStage│ ─► BlockState (mutated) +└────────────────────┘ + │ + v (loop while has_created_data_sources) +┌─────────────────────────┐ +│ DynamicDataSourceStage │ ─► BlockState (mutated), new hosts added +└─────────────────────────┘ + │ + v +┌─────────────────────┐ +│ OffchainTriggerStage│ ─► offchain_mods, processed_ds +└─────────────────────┘ + │ + v +┌─────────────┐ +│ PersistStage│ ─► Store transaction +└─────────────┘ +``` + +## Verification + +After implementation, verify: + +1. **Unit tests pass**: `just test-unit` +2. **Runner tests pass**: `just test-runner` +3. **Lint clean**: `just lint` (zero warnings) +4. **Build succeeds**: `just check --release` + +For behavioral verification, the existing runner tests should catch regressions. No new integration tests required for a refactor that preserves behavior. diff --git a/docs/subgraph-manifest.md b/docs/subgraph-manifest.md index e595690e9f3..caad7943e84 100644 --- a/docs/subgraph-manifest.md +++ b/docs/subgraph-manifest.md @@ -74,6 +74,7 @@ The `mapping` field may be one of the following supported mapping manifests: | **event** | *String* | An identifier for an event that will be handled in the mapping script. For Ethereum contracts, this must be the full event signature to distinguish from events that may share the same name. No alias types can be used. For example, uint will not work, uint256 must be used.| | **handler** | *String* | The name of an exported function in the mapping script that should handle the specified event. | | **topic0** | optional *String* | A `0x` prefixed hex string. If provided, events whose topic0 is equal to this value will be processed by the given handler. When topic0 is provided, _only_ the topic0 value will be matched, and not the hash of the event signature. This is useful for processing anonymous events in Solidity, which can have their topic0 set to anything. By default, topic0 is equal to the hash of the event signature. | +| **calls** | optional [*CallDecl*](#153-declaring-calls) | A list of predeclared `eth_calls` that will be made before running the handler | #### 1.5.2.3 CallHandler @@ -95,6 +96,40 @@ The `mapping` field may be one of the following supported mapping manifests: | --- | --- | --- | | **kind** | *String* | The selected block handler filter. Only option for now: `call`: This will only run the handler if the block contains at least one call to the data source contract. | +### 1.5.3 Declaring calls + +_Available from spec version 1.2.0. Struct field access available from spec version 1.4.0_ + +Declared calls are performed in parallel before the handler is run and can +greatly speed up syncing. Mappings access the call results simply by using +`ethereum.call` from the mappings. The **calls** are a map of key value pairs: + +| Field | Type | Description | +| --- | --- | --- | +| **label** | *String* | A label for the call for error messages etc. | +| **call** | *String* | See below | + +Each call is of the form `[
].()`: + +| Field | Type | Description | +| --- | --- | --- | +| **ABI** | *String* | The name of an ABI from the `abis` section | +| **address** | *Expr* | The address of a contract that follows the `ABI` | +| **function** | *String* | The name of a view function in the contract | +| **args** | *[Expr]* | The arguments to pass to the function | + +#### Expression Types + +The `Expr` can be one of the following: + +| Expression | Description | +| --- | --- | +| **event.address** | The address of the contract that emitted the event | +| **event.params.<name>** | A simple parameter from the event | +| **event.params.<name>.<index>** | A field from a struct parameter by numeric index | +| **event.params.<name>.<fieldName>** | A field from a struct parameter by field name (spec version 1.4.0+) | + + ## 1.6 Path A path has one field `path`, which either refers to a path of a file on the local dev machine or an [IPLD link](https://github.com/ipld/specs/). diff --git a/entitlements.plist b/entitlements.plist new file mode 100644 index 00000000000..d9ce520f2e1 --- /dev/null +++ b/entitlements.plist @@ -0,0 +1,12 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-executable-page-protection + + + \ No newline at end of file diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..7d69cf5ac8c --- /dev/null +++ b/flake.lock @@ -0,0 +1,182 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1755585599, + "narHash": "sha256-tl/0cnsqB/Yt7DbaGMel2RLa7QG5elA8lkaOXli6VdY=", + "owner": "nix-community", + "repo": "fenix", + "rev": "6ed03ef4c8ec36d193c18e06b9ecddde78fb7e42", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1754487366, + "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "foundry": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1760001025, + "narHash": "sha256-j/jPNlW2W4zwV2rSqgw5ts1VoTXVe5n3CNXFvtQhK3Q=", + "owner": "shazow", + "repo": "foundry.nix", + "rev": "b7adb89167832516589c899addcd25ca2a78dcfe", + "type": "github" + }, + "original": { + "owner": "shazow", + "repo": "foundry.nix", + "rev": "b7adb89167832516589c899addcd25ca2a78dcfe", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1666753130, + "narHash": "sha256-Wff1dGPFSneXJLI2c0kkdWTgxnQ416KE6X4KnFkgPYQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f540aeda6f677354f1e7144ab04352f61aaa0118", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1753579242, + "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1756128520, + "narHash": "sha256-R94HxJBi+RK1iCm8Y4Q9pdrHZl0GZoDPIaYwjxRNPh4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "c53baa6685261e5253a1c355a1b322f82674a824", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "process-compose-flake": { + "locked": { + "lastModified": 1749418557, + "narHash": "sha256-wJHHckWz4Gvj8HXtM5WVJzSKXAEPvskQANVoRiu2w1w=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "91dcc48a6298e47e2441ec76df711f4e38eab94e", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "flake-parts": "flake-parts", + "foundry": "foundry", + "nixpkgs": "nixpkgs_2", + "process-compose-flake": "process-compose-flake", + "services-flake": "services-flake" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1755504847, + "narHash": "sha256-VX0B9hwhJypCGqncVVLC+SmeMVd/GAYbJZ0MiiUn2Pk=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "a905e3b21b144d77e1b304e49f3264f6f8d4db75", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "services-flake": { + "locked": { + "lastModified": 1755996515, + "narHash": "sha256-1RQQIDhshp1g4PP5teqibcFLfk/ckTDOJRckecAHiU0=", + "owner": "juspay", + "repo": "services-flake", + "rev": "e316d6b994fd153f0c35d54bd07d60e53f0ad9a9", + "type": "github" + }, + "original": { + "owner": "juspay", + "repo": "services-flake", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..933a35ef04c --- /dev/null +++ b/flake.nix @@ -0,0 +1,209 @@ +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + foundry.url = "github:shazow/foundry.nix/b7adb89167832516589c899addcd25ca2a78dcfe"; + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; + services-flake.url = "github:juspay/services-flake"; + flake-parts.url = "github:hercules-ci/flake-parts"; + }; + + outputs = inputs @ { + flake-parts, + process-compose-flake, + services-flake, + nixpkgs, + fenix, + foundry, + ... + }: + flake-parts.lib.mkFlake {inherit inputs;} { + imports = [process-compose-flake.flakeModule]; + systems = [ + "x86_64-linux" # 64-bit Intel/AMD Linux + "aarch64-linux" # 64-bit ARM Linux + "x86_64-darwin" # 64-bit Intel macOS + "aarch64-darwin" # 64-bit ARM macOS + ]; + + perSystem = { + config, + self', + inputs', + pkgs, + system, + ... + }: let + overlays = [ + fenix.overlays.default + foundry.overlay + ]; + + pkgs = import nixpkgs { + inherit overlays system; + }; + + toolchain = with fenix.packages.${system}; + combine [ + (fromToolchainFile { + file = ./rust-toolchain.toml; + sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; + }) + stable.rust-src # This is needed for rust-analyzer to find stdlib symbols. Should use the same channel as the toolchain. + ]; + in { + formatter = pkgs.alejandra; + devShells.default = pkgs.mkShell { + packages = with pkgs; [ + toolchain + foundry-bin + solc + protobuf + uv + cmake + corepack + nodejs + postgresql + just + cargo-nextest + ]; + }; + + process-compose = let + inherit (services-flake.lib) multiService; + ipfs = multiService ./nix/ipfs.nix; + anvil = multiService ./nix/anvil.nix; + + # Helper function to create postgres configuration with graph-specific defaults + mkPostgresConfig = { + name, + port, + user, + password, + database, + dataDir, + }: { + enable = true; + inherit port dataDir; + initialScript = { + before = '' + CREATE ROLE postgres WITH LOGIN; + CREATE USER \"${user}\" WITH PASSWORD '${password}' SUPERUSER; + ''; + }; + initialDatabases = [ + { + inherit name; + schemas = [ + (pkgs.writeText "init-${name}.sql" '' + CREATE EXTENSION IF NOT EXISTS pg_trgm; + CREATE EXTENSION IF NOT EXISTS btree_gist; + CREATE EXTENSION IF NOT EXISTS postgres_fdw; + CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO "${user}"; + ALTER DATABASE "${database}" OWNER TO "${user}"; + '') + ]; + } + ]; + settings = { + shared_preload_libraries = "pg_stat_statements"; + log_statement = "all"; + default_text_search_config = "pg_catalog.english"; + max_connections = 500; + }; + }; + in { + # Unit tests configuration + unit = { + imports = [ + services-flake.processComposeModules.default + ipfs + anvil + ]; + + cli = { + environment.PC_DISABLE_TUI = true; + options = { + port = 8881; + }; + }; + + services.postgres."postgres-unit" = mkPostgresConfig { + name = "graph-test"; + port = 5432; + dataDir = "./.data/unit/postgres"; + user = "graph"; + password = "graph"; + database = "graph-test"; + }; + + # Set PGUSER so psql connects as the OS user (matching initdb's superuser) + settings.processes."postgres-unit-init".environment = { + PGUSER = builtins.getEnv "USER"; + }; + + services.ipfs."ipfs-unit" = { + enable = true; + dataDir = "./.data/unit/ipfs"; + port = 5001; + gateway = 8080; + }; + }; + + # Integration tests configuration + integration = { + imports = [ + services-flake.processComposeModules.default + ipfs + anvil + ]; + + cli = { + environment.PC_DISABLE_TUI = true; + options = { + port = 8882; + }; + }; + + services.postgres."postgres-integration" = mkPostgresConfig { + name = "graph-node"; + port = 3011; + dataDir = "./.data/integration/postgres"; + user = "graph-node"; + password = "let-me-in"; + database = "graph-node"; + }; + + # Set PGUSER so psql connects as the OS user (matching initdb's superuser) + settings.processes."postgres-integration-init".environment = { + PGUSER = builtins.getEnv "USER"; + }; + + services.ipfs."ipfs-integration" = { + enable = true; + dataDir = "./.data/integration/ipfs"; + port = 3001; + gateway = 3002; + }; + + services.anvil."anvil-integration" = { + enable = true; + package = pkgs.foundry-bin; + port = 3021; + timestamp = 1743944919; + gasLimit = 100000000000; + baseFee = 1; + blockTime = null; + state = "./.data/integration/anvil/state.json"; + stateInterval = 30; + preserveHistoricalStates = true; + }; + }; + }; + }; + }; +} diff --git a/gnd/Cargo.toml b/gnd/Cargo.toml new file mode 100644 index 00000000000..754949a4722 --- /dev/null +++ b/gnd/Cargo.toml @@ -0,0 +1,92 @@ +[package] +name = "gnd" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "gnd" +path = "src/main.rs" + +[[test]] +name = "cli_commands" +path = "tests/cli_commands.rs" + +[[test]] +name = "codegen_verification" +path = "tests/codegen_verification.rs" + +[[test]] +name = "gnd_test" +path = "tests/gnd_test.rs" + +[dependencies] +# Core graph dependencies +graph = { path = "../graph" } +graph-chain-ethereum = { path = "../chain/ethereum" } +graph-core = { path = "../core" } +graph-node = { path = "../node" } +graph-graphql = { path = "../graphql" } +graph-store-postgres = { path = "../store/postgres" } + +# Test command dependencies +hex = "0.4" +async-trait = { workspace = true } +tower = { workspace = true } + +# Direct dependencies from current dev.rs +anyhow = { workspace = true } +clap = { workspace = true } +clap_complete = { workspace = true } +env_logger = { workspace = true } +git-testament = { workspace = true } +lazy_static = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } + +# File watching +notify = "8.2.0" +globset = "0.4.18" + +# Config and auth +url = { workspace = true } +serde_json = { workspace = true } +pq-sys = { version = "0.7.5", features = ["bundled"] } + +# HTTP client for Graph Node API +reqwest = { workspace = true } +thiserror = { workspace = true } + +# Console output +similar = "3" +indicatif = { workspace = true } +console = { workspace = true } + +# Code generation +graphql-tools = { workspace = true } +regex = { workspace = true } +serde_yaml = { workspace = true } +Inflector = { workspace = true } + +# Migrations +semver = { workspace = true } + +# Build command +sha1 = "0.11" +wasmparser = { workspace = true } + +# Interactive prompts +inquire = "0.9" + +# Temp directories for init +tempfile = "3" + +# Browser opening for publish command +open = "5" + +[target.'cfg(unix)'.dependencies] +pgtemp = { git = "https://github.com/graphprotocol/pgtemp", branch = "initdb-args" } + +[dev-dependencies] +tempfile = "3" +walkdir = "2" diff --git a/gnd/README.md b/gnd/README.md new file mode 100644 index 00000000000..375e1ef0e01 --- /dev/null +++ b/gnd/README.md @@ -0,0 +1,524 @@ +# gnd - Graph Node Dev CLI + +A drop-in replacement for `graph-cli` written in Rust. `gnd` provides all the subgraph development commands with identical output, flags, and behavior. + +## Installation + +Build from source: + +```bash +cargo build -p gnd --release +``` + +The binary will be at `target/release/gnd`. + +## Quick Start + +```bash +# Create a new subgraph from a contract +gnd init --from-contract 0x1234... --network mainnet my-subgraph + +# Generate AssemblyScript types +gnd codegen + +# Build the subgraph +gnd build + +# Deploy to a local Graph Node +gnd create --node http://localhost:8020 my-name/my-subgraph +gnd deploy --node http://localhost:8020 --ipfs http://localhost:5001 -l v0.0.1 my-name/my-subgraph + +# Or deploy to Subgraph Studio +gnd auth YOUR_DEPLOY_KEY +gnd deploy -l v0.0.1 my-name/my-subgraph +``` + +## Commands + +### `gnd init` + +Create a new subgraph with basic scaffolding. + +```bash +gnd init [SUBGRAPH_NAME] [DIRECTORY] +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--protocol` | | Protocol: `ethereum`, `near`, `cosmos`, `arweave`, `substreams` | +| `--from-contract` | | Create from an existing contract address | +| `--from-example` | | Create from an example subgraph template | +| `--from-subgraph` | | Create from an existing deployed subgraph | +| `--contract-name` | | Name for the contract (with `--from-contract`) | +| `--index-events` | | Index all contract events as entities | +| `--network` | | Network the contract is deployed to | +| `--start-block` | | Block number to start indexing from | +| `--abi` | | Path to the contract ABI file | +| `--node` | `-g` | Graph Node URL | +| `--ipfs` | `-i` | IPFS node URL | +| `--skip-install` | | Skip installing npm dependencies | +| `--skip-git` | | Skip initializing a Git repository | + +**Examples:** + +```bash +# Interactive mode (prompts for all options) +gnd init + +# From contract with ABI fetched from Etherscan +gnd init --from-contract 0x1234... --network mainnet my-subgraph + +# From contract with local ABI +gnd init --from-contract 0x1234... --abi ./MyContract.json my-subgraph + +# From an existing deployed subgraph +gnd init --from-subgraph QmHash... my-subgraph +``` + +### `gnd codegen` + +Generate AssemblyScript types from the subgraph manifest. + +```bash +gnd codegen [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | `-o` | Output directory (default: `generated/`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--watch` | `-w` | Regenerate on file changes | +| `--ipfs` | `-i` | IPFS node URL | + +**Examples:** + +```bash +# Generate types +gnd codegen + +# Generate to custom directory +gnd codegen -o src/generated/ + +# Watch mode +gnd codegen --watch +``` + +### `gnd build` + +Compile the subgraph to WASM. + +```bash +gnd build [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | `-o` | Output directory (default: `build/`) | +| `--output-format` | `-t` | Output format: `wasm` or `wast` (default: `wasm`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--watch` | `-w` | Rebuild on file changes | +| `--ipfs` | `-i` | IPFS node URL (uploads if provided) | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config (default: `networks.json`) | + +**Examples:** + +```bash +# Build subgraph +gnd build + +# Build and upload to IPFS +gnd build --ipfs http://localhost:5001 + +# Build for specific network +gnd build --network mainnet +``` + +### `gnd deploy` + +Deploy a subgraph to a Graph Node. + +```bash +gnd deploy [MANIFEST] +``` + +**Arguments:** +- `SUBGRAPH_NAME`: Name to deploy as (e.g., `user/subgraph`) +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (defaults to Subgraph Studio) | +| `--ipfs` | `-i` | IPFS node URL | +| `--deploy-key` | | Deploy key for authentication | +| `--version-label` | `-l` | Version label for the deployment (required in non-interactive mode) | +| `--ipfs-hash` | | IPFS hash of already-uploaded manifest | +| `--output-dir` | `-o` | Build output directory (default: `build/`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config | +| `--debug-fork` | | Fork subgraph ID for debugging | + +If `--version-label` is omitted in an interactive terminal, `gnd deploy` prompts for it. +In non-interactive environments (CI/scripts), you must pass `--version-label`. + +**Examples:** + +```bash +# Deploy to Subgraph Studio (uses saved auth key) +gnd deploy -l v1.0.0 my-name/my-subgraph + +# Deploy to local Graph Node +gnd deploy --node http://localhost:8020 --ipfs http://localhost:5001 -l v1.0.0 my-name/my-subgraph + +# Deploy without --version-label in an interactive terminal (prompts) +gnd deploy my-name/my-subgraph +``` + +### `gnd publish` + +Publish a subgraph to The Graph's decentralized network. + +```bash +gnd publish [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--ipfs` | `-i` | IPFS node URL | +| `--ipfs-hash` | | Skip build, use existing IPFS hash | +| `--subgraph-id` | | Subgraph ID for updating existing subgraphs | +| `--protocol-network` | | Network: `arbitrum-one` or `arbitrum-sepolia` | +| `--api-key` | | API key (required when updating existing) | +| `--output-dir` | `-o` | Build output directory | +| `--skip-migrations` | | Skip manifest migrations | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config | + +**Examples:** + +```bash +# Publish new subgraph +gnd publish + +# Update existing subgraph +gnd publish --subgraph-id Qm... --api-key YOUR_KEY +``` + +### `gnd add` + +Add a new data source to an existing subgraph. + +```bash +gnd add
[MANIFEST] +``` + +**Arguments:** +- `ADDRESS`: Contract address to add +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--abi` | | Path to the contract ABI | +| `--contract-name` | | Name for the new data source | +| `--merge-entities` | | Merge with existing entities of same name | +| `--network` | | Network the contract is deployed to | +| `--start-block` | | Block number to start indexing from | + +**Examples:** + +```bash +# Add contract with ABI from Etherscan +gnd add 0x1234... --network mainnet + +# Add contract with local ABI +gnd add 0x1234... --abi ./NewContract.json --contract-name MyContract +``` + +### `gnd create` + +Register a subgraph name with a Graph Node. + +```bash +gnd create --node +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (required) | +| `--access-token` | | Access token for authentication | + +### `gnd remove` + +Unregister a subgraph name from a Graph Node. + +```bash +gnd remove --node +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (required) | +| `--access-token` | | Access token for authentication | + +### `gnd auth` + +Store a deploy key for authentication. + +```bash +gnd auth +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (default: Subgraph Studio) | + +Keys are stored in `~/.graph-cli.json`. + +### `gnd test` + +Run subgraph tests. + +```bash +gnd test [TEST_FILES...] +``` + +**Arguments:** +- `PATHS`: Test JSON files or directories to scan. Defaults to `tests/` when nothing is specified. + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--manifest` | `-m` | Path to subgraph manifest (default: `subgraph.yaml`) | +| `--skip-build` | `-s` | Skip building the subgraph before testing | +| `--postgres-url` | | PostgreSQL connection URL (env: `POSTGRES_URL`) | +| `--matchstick` | | Use legacy Matchstick runner (**deprecated** — migrate to JSON-based tests) | +| `--docker` | `-d` | Run Matchstick in Docker (requires `--matchstick`) | +| `--coverage` | `-c` | Run with coverage reporting (requires `--matchstick`) | +| `--recompile` | `-r` | Force recompilation (requires `--matchstick`) | +| `--force` | `-f` | Force redownload of Matchstick binary (requires `--matchstick`) | + +**Examples:** + +```bash +# Run all tests in tests/ directory (default) +gnd test + +# Run specific test files +gnd test transfer.json approval.json +gnd test tests/transfer.json + +# Scan a custom directory +gnd test my-tests/ + +# Use a different manifest +gnd test -m subgraph.staging.yaml tests/transfer.json + +# Skip automatic build +gnd test -s +``` + +### `gnd clean` + +Remove build artifacts and generated files. + +```bash +gnd clean +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--codegen-dir` | | Codegen directory (default: `generated/`) | +| `--build-dir` | | Build directory (default: `build/`) | + +### `gnd dev` + +Run graph-node in development mode with file watching. + +```bash +gnd dev [OPTIONS] +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--watch` | | Watch build directory for changes | +| `--manifests` | | Subgraph manifest locations | +| `--sources` | | Source manifest locations for aliases | +| `--database-dir` | | Database directory (default: `./build`) | +| `--postgres-url` | | PostgreSQL connection URL | +| `--ethereum-rpc` | | Ethereum RPC URL | +| `--ipfs` | | IPFS node URL | + +### `gnd indexer` + +Manage indexer operations via [`indexer-cli`](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-cli). + +Requires `graph-indexer` to be installed and on `$PATH`: + +```bash +npm install -g @graphprotocol/indexer-cli +``` + +```bash +gnd indexer [args...] +``` + +**Help:** + +There are two ways to get help: + +| Command | What it shows | +|---------|---------------| +| `gnd indexer --help` | gnd's own help for the indexer subcommand (works without `graph-indexer` installed) | +| `gnd indexer help` | Full `graph-indexer` help with all available commands (requires `graph-indexer`) | + +**Examples:** + +```bash +# Check indexer status +gnd indexer status --network arbitrum-one + +# Manage indexing rules +gnd indexer rules get all --network mainnet +gnd indexer rules set decisionBasis always --network mainnet + +# Manage allocations +gnd indexer allocations get --network arbitrum-one + +# Manage cost models +gnd indexer cost get + +# View available indexer commands +gnd indexer help + +# Check graph-indexer version +gnd indexer version +``` + +### `gnd completions` + +Generate shell completions. + +```bash +gnd completions +``` + +**Arguments:** +- `SHELL`: One of `bash`, `elvish`, `fish`, `powershell`, `zsh` + +**Examples:** + +```bash +# Bash +gnd completions bash > ~/.bash_completion.d/gnd + +# Zsh +gnd completions zsh > ~/.zfunc/_gnd + +# Fish +gnd completions fish > ~/.config/fish/completions/gnd.fish +``` + +## Configuration Files + +### `~/.graph-cli.json` + +Stores deploy keys for different Graph Node URLs. Created by `gnd auth`. + +### `networks.json` + +Network-specific configuration for contract addresses and start blocks: + +```json +{ + "mainnet": { + "MyContract": { + "address": "0x1234...", + "startBlock": 12345678 + } + }, + "sepolia": { + "MyContract": { + "address": "0x5678...", + "startBlock": 1000000 + } + } +} +``` + +Use with `--network mainnet` on build/deploy commands. + +## Differences from graph-cli + +`gnd` is designed as a drop-in replacement for `graph-cli`. Some intentional differences: + +### Commands Not Implemented + +- **`local`**: Use graph-node's integration test infrastructure instead +- **`node`**: Use `graphman` for node management operations + +### Code Generation Differences + +These are documented and tested: + +1. **Int8 import**: Always imported for simplicity, even when not used +2. **Trailing commas**: Used in multi-line constructs +3. **2D array accessors**: Uses correct `toStringMatrix()` (fixes a bug in graph-cli) + +### Other Differences + +- **Debug logging**: Uses `RUST_LOG` environment variable instead of `DEBUG=graph-cli:*` +- **`--uncrashable` flag**: Not implemented (Float Capital third-party feature) + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `RUST_LOG` | Debug logging level (e.g., `RUST_LOG=gnd=debug`) | +| `POSTGRES_URL` | PostgreSQL URL for `gnd dev` | +| `ETHEREUM_RPC` | Ethereum RPC URL for `gnd dev` | + +## Version + +```bash +gnd --version +# gnd 0.1.0 (graph-cli compatible: 0.98.1) +``` + +Shows both the gnd version and the graph-cli version it emulates. + +## License + +Apache-2.0 OR MIT diff --git a/gnd/docs/gnd-test.md b/gnd/docs/gnd-test.md new file mode 100644 index 00000000000..6fcc118fc29 --- /dev/null +++ b/gnd/docs/gnd-test.md @@ -0,0 +1,939 @@ +# gnd test + +Mock-based subgraph test runner that feeds JSON-defined blocks through real graph-node infrastructure (store, WASM runtime, trigger processing) with only the blockchain layer mocked. + +## Quick Start + +```bash +# Run all tests in tests/ directory +gnd test + +# Run a specific test file +gnd test tests/transfer.json + +# Skip automatic build (if subgraph already built) +gnd test --skip-build + +# Use legacy Matchstick runner +gnd test --matchstick +``` + +## Test File Format + +Tests are JSON files that define: +- Mock blockchain blocks with events +- Mock Ethereum RPC responses (`eth_call`, `eth_getBalance`, `eth_getCode`) +- GraphQL assertions to validate entity state + +Place test files in a `tests/` directory with `.json` or `.test.json` extension. + +### Basic Example + +```json +{ + "name": "Transfer creates entity", + "blocks": [ + { + "number": 1, + "timestamp": 1672531200, + "events": [ + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { + "from": "0xaaaa...", + "to": "0xbbbb...", + "value": "1000" + } + } + ], + "ethCalls": [ + { + "address": "0x1234...", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa..."], + "returns": ["1000000000000000000"] + } + ] + } + ], + "assertions": [ + { + "query": "{ transfer(id: \"1\") { from to value } }", + "expected": { + "transfer": { + "from": "0xaaaa...", + "to": "0xbbbb...", + "value": "1000" + } + } + } + ] +} +``` + +## Block Fields + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `number` | No | Auto-increments from lowest defined `startBlock` in the manifest file, or from `0` if no `startBlock` are defined | Block number | +| `hash` | No | `keccak256(block_number)` | Block hash | +| `timestamp` | No | `block_number` | Unix timestamp | +| `baseFeePerGas` | No | None (pre-EIP-1559) | Base fee in wei | +| `events` | No | Empty array | Log events in this block | +| `ethCalls` | No | Empty array | Mock `eth_call` responses | +| `getBalanceCalls` | No | Empty array | Mock `eth_getBalance` responses for `ethereum.getBalance()` | +| `hasCodeCalls` | No | Empty array | Mock `eth_getCode` responses for `ethereum.hasCode()` | + +### Empty Blocks + +Empty blocks (no events) still trigger block handlers: + +```json +{ + "name": "Test block handlers", + "blocks": [ + { + "number": 1, + "events": [...] + }, + {} // Block 2 with no events - block handlers still fire + ] +} +``` + +## Event Fields + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `address` | Yes | — | Contract address (lowercase hex with 0x prefix) | +| `event` | Yes | — | Full event signature with `indexed` keywords | +| `params` | No | Empty object | Event parameter values | +| `txHash` | No | `keccak256(block_number \|\| log_index)` | Transaction hash | + +### Event Signature Format + +**Important:** Include `indexed` keywords in the signature: + +```json +{ + "event": "Transfer(address indexed from, address indexed to, uint256 value)" +} +``` + +Not: +```json +{ + "event": "Transfer(address,address,uint256)" // ❌ Missing indexed keywords +} +``` + +### Parameter Types + +Event parameters are automatically ABI-encoded based on the signature. Supported formats: + +```json +{ + "params": { + "from": "0xaaaa...", // address + "to": "0xbbbb...", // address + "value": "1000", // uint256 (string or number) + "amount": 1000, // uint256 (number) + "enabled": true, // bool + "data": "0x1234...", // bytes + "name": "Token" // string + } +} +``` + +## Transaction Receipts + +Mock receipts are constructed for every log trigger and attached only to handlers that declare `receipt: true` in the manifest, mirroring production behaviour. Handlers without `receipt: true` receive a null receipt — the same as on a real node. + +**Limitation:** Only `receipt.logs` reflects your test data. All other receipt fields (`from`, `to`, `gas_used`, `status`, etc.) are hardcoded stubs and do not correspond to real transaction data. If your handler reads those fields, the values will be fixed defaults regardless of what you put in the test JSON. + +### How receipts are built + +Every event gets a mock receipt attached automatically. The key rule is **`txHash` grouping**: + +- Events sharing the same `txHash` share **one receipt** — `event.receipt!.logs` contains all of their logs in declaration order. +- Events without an explicit `txHash` each get a unique auto-generated hash (`keccak256(block_number || log_index)`), so each gets its own single-log receipt. + +### Example: Two events sharing a receipt + +```json +{ + "events": [ + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { "from": "0xaaaa...", "to": "0xbbbb...", "value": "100" }, + "txHash": "0xdeadbeef0000000000000000000000000000000000000000000000000000000" + }, + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { "from": "0xbbbb...", "to": "0xcccc...", "value": "50" }, + "txHash": "0xdeadbeef0000000000000000000000000000000000000000000000000000000" + } + ] +} +``` + +Both handlers receive a receipt where `receipt.logs` has two entries, in declaration order. + +### Mock receipt defaults + +| Field | Value | +|-------|-------| +| `status` | success | +| `cumulative_gas_used` | `21000` | +| `gas_used` | `21000` | +| transaction type | `2` (EIP-1559) | +| `from` | `0x000...000` | +| `to` | `null` | +| `effective_gas_price` | `0` | + +Handlers without `receipt: true` in the manifest are unaffected — they never access `event.receipt`. + +## Block Handlers + +Block handlers are **automatically triggered** for every block. You don't need to specify block triggers in the JSON. + +### How Block Handlers Work + +The test runner auto-injects both `Start` and `End` block triggers for each block, ensuring all block handler filters work correctly: + +- **`once` filter** → Fires once at `startBlock` (via `Start` trigger) +- **No filter** → Fires on every block (via `End` trigger) +- **`polling` filter** → Fires every N blocks based on formula: `(block_number - startBlock) % every == 0` + +### Example: Basic Block Handlers + +```json +{ + "name": "Block handlers test", + "blocks": [ + {}, // Block 0 - both 'once' and regular block handlers fire + {} // Block 1 - only regular block handlers fire + ], + "assertions": [ + { + "query": "{ blocks { number } }", + "expected": { + "blocks": [ + {"number": "0"}, + {"number": "1"} + ] + } + }, + { + "query": "{ blockOnces { msg } }", + "expected": { + "blockOnces": [ + {"msg": "This fires only once at block 0"} + ] + } + } + ] +} +``` + +### Polling Block Handlers + +Polling handlers fire at regular intervals specified by the `every` parameter. The handler fires when: + +``` +(block_number - startBlock) % every == 0 +``` + +**Manifest example:** +```yaml +blockHandlers: + - handler: handleEveryThreeBlocks + filter: + kind: polling + every: 3 +``` + +**Test example (startBlock: 0):** +```json +{ + "name": "Polling handler test", + "blocks": [ + {}, // Block 0 - handler fires (0 % 3 == 0) + {}, // Block 1 - handler doesn't fire + {}, // Block 2 - handler doesn't fire + {}, // Block 3 - handler fires (3 % 3 == 0) + {}, // Block 4 - handler doesn't fire + {}, // Block 5 - handler doesn't fire + {} // Block 6 - handler fires (6 % 3 == 0) + ], + "assertions": [ + { + "query": "{ pollingBlocks(orderBy: number) { number } }", + "expected": { + "pollingBlocks": [ + {"number": "0"}, + {"number": "3"}, + {"number": "6"} + ] + } + } + ] +} +``` + +**With non-zero startBlock:** + +When your data source has `startBlock > 0`, the polling interval is calculated from that starting point. + +**Manifest:** +```yaml +dataSources: + - name: Token + source: + startBlock: 100 + mapping: + blockHandlers: + - handler: handlePolling + filter: + kind: polling + every: 5 +``` + +**Test:** +```json +{ + "name": "Polling from block 100", + "blocks": [ + {"number": 100}, // Fires: (100-100) % 5 == 0 + {"number": 101}, // Doesn't fire + {"number": 102}, // Doesn't fire + {"number": 103}, // Doesn't fire + {"number": 104}, // Doesn't fire + {"number": 105}, // Fires: (105-100) % 5 == 0 + {"number": 106}, // Doesn't fire + {"number": 107}, // Doesn't fire + {"number": 108}, // Doesn't fire + {"number": 109}, // Doesn't fire + {"number": 110} // Fires: (110-100) % 5 == 0 + ], + "assertions": [ + { + "query": "{ pollingBlocks(orderBy: number) { number } }", + "expected": { + "pollingBlocks": [ + {"number": "100"}, + {"number": "105"}, + {"number": "110"} + ] + } + } + ] +} +``` + +**Note:** The test runner automatically handles `startBlock > 0`, so blocks default to numbering from the manifest's `startBlock`. + +## eth_call Mocking + +Mock contract calls made from mapping handlers using `contract.call()`: + +```json +{ + "ethCalls": [ + { + "address": "0x1234...", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa..."], + "returns": ["1000000000000000000"] + } + ] +} +``` + +### ethCall Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Contract address | +| `function` | Yes | Full signature: `"functionName(inputTypes)(returnTypes)"` | +| `params` | Yes | Array of input parameters (as strings) | +| `returns` | Yes | Array of return values (as strings, ignored if `reverts: true`) | +| `reverts` | No | Default `false`. If `true`, the mock transport returns an RPC error | + +### Function Signature Format + +Use full signatures with input and return types: + +```json +{ + "function": "symbol()(string)", // No inputs, returns string + "function": "balanceOf(address)(uint256)", // One input, returns uint256 + "function": "decimals()(uint8)" // No inputs, returns uint8 +} +``` + +### Mocking Reverts + +```json +{ + "address": "0x1234...", + "function": "transfer(address,uint256)(bool)", + "params": ["0xaaaa...", "1000"], + "returns": [], + "reverts": true +} +``` + +### Real-World Example + +From the ERC20 test: + +```json +{ + "ethCalls": [ + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "symbol()(string)", + "params": [], + "returns": ["GRT"] + }, + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "name()(string)", + "params": [], + "returns": ["TheGraph"] + }, + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa000000000000000000000000000000000000"], + "returns": ["3000000000000000000"] + } + ] +} +``` + +## ethereum.getBalance() Mocking + +Mock balance lookups made from mapping handlers using `ethereum.getBalance()`: + +```json +{ + "getBalanceCalls": [ + { + "address": "0xaaaa000000000000000000000000000000000000", + "value": "1000000000000000000" + } + ] +} +``` + +### getBalanceCalls Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Account address (checksummed or lowercase hex) | +| `value` | Yes | Balance in Wei as a decimal string | + +## ethereum.hasCode() Mocking + +Mock code existence checks made from mapping handlers using `ethereum.hasCode()`: + +```json +{ + "hasCodeCalls": [ + { + "address": "0x1234000000000000000000000000000000000000", + "hasCode": true + } + ] +} +``` + +### hasCodeCalls Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Contract address (checksummed or lowercase hex) | +| `hasCode` | Yes | Whether the address has deployed bytecode | + +## File Data Sources + +Mock IPFS and Arweave file contents for file data source handlers. Files are defined at the top level of the test JSON (not inside blocks). + +### IPFS Files + +```json +{ + "name": "File data source test", + "files": [ + { + "cid": "QmExample...", + "content": "{\"name\": \"Token\", \"description\": \"A token\"}" + }, + { + "cid": "QmAnother...", + "file": "fixtures/metadata.json" + } + ], + "blocks": [...], + "assertions": [...] +} +``` + +#### files Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `cid` | Yes | IPFS CID (`Qm...` or `bafy...`). The mock ignores hash/content relationship | +| `content` | One of `content`/`file` | Inline UTF-8 content | +| `file` | One of `content`/`file` | File path, resolved relative to the test JSON | + +### Arweave Files + +```json +{ + "name": "Arweave data source test", + "arweaveFiles": [ + { + "txId": "abc123", + "content": "{\"name\": \"Token\"}" + }, + { + "txId": "def456/metadata.json", + "file": "fixtures/arweave-data.json" + } + ], + "blocks": [...], + "assertions": [...] +} +``` + +#### arweaveFiles Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `txId` | Yes | Arweave transaction ID or bundle path (e.g. `"txid/filename.json"`) | +| `content` | One of `content`/`file` | Inline UTF-8 content | +| `file` | One of `content`/`file` | File path, resolved relative to the test JSON | + +## Assertions + +GraphQL queries to validate the indexed entity state after processing all blocks. + +### Assertion Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `query` | Yes | GraphQL query string | +| `expected` | Yes | Expected JSON response | + +### Comparison Behavior + +| Aspect | Behavior | +|--------|----------| +| Objects | Key-compared, order-insensitive | +| Arrays | **Order-insensitive** (set comparison) | +| String vs Number | Coerced — `"123"` matches `123` | +| Nulls/Booleans | Strict equality | +| Hex strings | **Must be lowercase** — graph-node returns all hex values in lowercase | + +**Important:** Arrays are compared as sets (order doesn't matter). + +**Important:** graph-node returns all hex-encoded values (addresses, transaction hashes, byte arrays) in **lowercase**. Expected values in assertions must match exactly — mixed-case hex will not match: + +```json +{ + "expected": { + "transfer": { + "from": "0xaaaa000000000000000000000000000000000000", // ✅ lowercase + "to": "0xBBBB000000000000000000000000000000000000" // ❌ will not match + } + } +} +``` + +Note: hex values in event inputs (`params`, `address`) are normalized automatically and can be mixed case. If you need ordered results, use `orderBy` in your GraphQL query: + +```json +{ + "query": "{ transfers(orderBy: timestamp, orderDirection: asc) { id from to value } }", + "expected": { ... } +} +``` + +### Multiple Assertions + +You can have multiple assertions per test. They run sequentially after all blocks are processed: + +```json +{ + "assertions": [ + { + "query": "{ tokens { id name symbol } }", + "expected": { ... } + }, + { + "query": "{ accounts { id balance } }", + "expected": { ... } + } + ] +} +``` + +### Nested Entity Queries + +Test relationships and nested entities: + +```json +{ + "query": "{ accounts { id balances { token { symbol } amount } } }", + "expected": { + "accounts": [ + { + "id": "0xbbbb...", + "balances": [ + { + "token": { "symbol": "GRT" }, + "amount": "5000000000000000000" + } + ] + } + ] + } +} +``` + +## startBlock Handling + +The test runner automatically reads `startBlock` from your subgraph manifest and handles it correctly — **no real blockchain connection needed**. + +### How It Works + +1. Extracts the **minimum `startBlock`** across all data sources in your manifest +2. If min > 0, creates a `start_block_override` to bypass graph-node's on-chain block validation +3. Test blocks without explicit `"number"` auto-increment starting from that minimum `startBlock` + +### Default Block Numbering + +The starting block number depends on your manifest: + +| Manifest Configuration | Test Block Numbers | +|----------------------|-------------------| +| `startBlock: 0` (or unset) | 0, 1, 2, ... | +| `startBlock: 100` | 100, 101, 102, ... | +| Multiple data sources: `startBlock: 50` and `startBlock: 200` | 50, 51, 52, ... (uses minimum) | + +### Example: Single Data Source + +**Manifest:** +```yaml +dataSources: + - name: Token + source: + startBlock: 1000 +``` + +**Test:** +```json +{ + "blocks": [ + {}, // Block 1000 (auto-numbered) + {} // Block 1001 (auto-numbered) + ] +} +``` + +### Example: Explicit Block Numbers + +Override auto-numbering by specifying `"number"`: + +```json +{ + "blocks": [ + { + "number": 5000, + "events": [...] + }, + { + "number": 5001, + "events": [...] + } + ] +} +``` + +### Multi-Data Source Testing + +When your subgraph has multiple data sources with different `startBlock` values, you may need to use explicit block numbers. + +**Scenario:** DataSource A at `startBlock: 50` (Transfer events), DataSource B at `startBlock: 200` (Approval events). You want to test only DataSource B. + +**Manifest:** +```yaml +dataSources: + - name: TokenTransfers + source: + startBlock: 50 + mapping: + eventHandlers: + - event: Transfer(...) + handler: handleTransfer + - name: TokenApprovals + source: + startBlock: 200 + mapping: + eventHandlers: + - event: Approval(...) + handler: handleApproval +``` + +**Test:** +```json +{ + "name": "Test Approval handler", + "blocks": [ + { + "number": 200, // Explicit number >= DataSource B's startBlock + "events": [ + { + "address": "0x5678...", + "event": "Approval(address indexed owner, address indexed spender, uint256 value)", + "params": { + "owner": "0xaaaa...", + "spender": "0xbbbb...", + "value": "500" + } + } + ] + }, + { + "number": 201, + "events": [...] + } + ] +} +``` + +**Why explicit numbers are needed:** +- Default numbering starts at the **minimum** `startBlock` across all data sources (50 in this case) +- Blocks 50-199 are below DataSource B's `startBlock: 200`, so its handlers won't fire +- Use explicit `"number": 200` to ensure the block is in DataSource B's active range + +**Note:** DataSource A is still "active" from block 50 onward, but it simply sees no matching Transfer events in blocks 200-201, so no handlers fire for it. This is normal behavior — graph-node doesn't error on inactive handlers. + +## Test Organization + +### Directory Structure + +``` +my-subgraph/ +├── subgraph.yaml +├── schema.graphql +├── src/ +│ └── mapping.ts +└── tests/ + ├── transfer.json + ├── approval.json + └── edge-cases.test.json +``` + +### Naming Conventions + +- Use `.json` or `.test.json` extension +- Descriptive names: `transfer.json`, `mint-burn.json`, `edge-cases.json` +- The test runner discovers all `*.json` and `*.test.json` files in the test directory + +## Known Limitations + +| Feature | Status | +|---------|--------| +| Log events | ✅ Supported | +| Block handlers (all filters) | ✅ Supported | +| `eth_call` mocking | ✅ Supported | +| `ethereum.getBalance()` mocking | ✅ Supported | +| `ethereum.hasCode()` mocking | ✅ Supported | +| Dynamic/template data sources | ✅ Supported | +| Transaction receipts (`receipt: true`) | ⚠️ Partial — `receipt.logs` is populated and grouped by `txHash`; other fields (gas, from, to, etc.) are hardcoded stubs (see [Transaction Receipts](#transaction-receipts)) | +| File data sources (IPFS + Arweave) | ✅ Supported | +| Call triggers (traces) | ❌ Not implemented | +| `--json` CI output | ❌ Not implemented | +| Parallel test execution | ❌ Not implemented | +| Test name filtering (`--filter`) | ❌ Not implemented | + +## Tips & Best Practices + +### Use Lowercase Hex in Assertions + +graph-node returns **all** hex-encoded values in lowercase — addresses, transaction hashes, and any `Bytes`/`ID` fields. Expected values in assertions must use lowercase hex: + +```json +{ + "expected": { + "transfer": { + "from": "0xaaaa000000000000000000000000000000000000" // ✅ lowercase + } + } +} +``` + +Not: +```json +{ + "expected": { + "transfer": { + "from": "0xAAAA000000000000000000000000000000000000" // ❌ will not match + } + } +} +``` + +Event inputs (`address`, `params`) are normalized automatically and can be mixed case. + +### Test One Thing at a Time + +Write focused tests that validate a single behavior: + +```json +// ✅ Good - tests one scenario +{ + "name": "Transfer event creates TransferEvent entity", + "blocks": [...], + "assertions": [...] +} +``` + +```json +// ❌ Avoid - tests too many things +{ + "name": "Test everything", + "blocks": [/* 50 blocks */], + "assertions": [/* 20 assertions */] +} +``` + +### Order GraphQL Results + +If your assertion needs specific ordering, use `orderBy`: + +```json +{ + "query": "{ transfers(first: 10, orderBy: timestamp, orderDirection: asc) { id } }", + "expected": { ... } +} +``` + +### Test Block Handlers with Empty Blocks + +Use empty blocks to test that block handlers fire even without events: + +```json +{ + "blocks": [ + {}, // Empty block - block handlers still fire + {} + ] +} +``` + +### Split Complex Tests + +Instead of one large test with many blocks, split into multiple focused test files: + +``` +tests/ +├── transfer-basic.json # Basic transfer functionality +├── transfer-zero-value.json # Edge case: zero value +└── transfer-same-account.json # Edge case: self-transfer +``` + +## Architecture + +The test runner reuses real graph-node infrastructure: + +``` +test.json + ↓ +Parse & ABI encode events + ↓ +Mock block stream (StaticStreamBuilder) + ↓ +Real graph-node indexer + ├── WASM runtime + ├── Trigger processing + └── Entity storage (pgtemp database) + ↓ +GraphQL queries → Assertions +``` + +**Key design principles:** + +- **Isolated database per test:** Each test gets a pgtemp database dropped on completion (default), or a shared persistent database with post-test cleanup (`--postgres-url`) +- **Mock transport layer:** A mock Alloy transport serves `eth_call`, `eth_getBalance`, and `eth_getCode` from test JSON data. All three flow through the real production code path — only the transport returns mock responses. Unmocked RPC calls fail immediately with a descriptive error. +- **No IPFS for manifest:** Uses `FileLinkResolver` to load manifest/WASM from build directory + +## Troubleshooting + +### Test Fails: "Entity not found" + +**Cause:** Handler didn't create the expected entity. + +**Fix:** +1. Check event signature matches ABI (include `indexed` keywords) +2. Verify contract address matches manifest +3. Check block number is >= data source's `startBlock` +4. Add debug logging to your mapping handler + +### Test Timeout + +**Cause:** Indexer took longer than 60 seconds (default timeout). + +**Fix:** +1. Reduce number of blocks in test +2. Simplify mapping logic +3. Check for infinite loops in handler code + +### Unmocked RPC Call + +**Cause:** A mapping handler calls `ethereum.call`, `ethereum.getBalance`, or `ethereum.hasCode` for a call that has no matching mock entry. + +**Symptom:** Test fails immediately with a descriptive error like: +``` +gnd test: unmocked eth_call to 0x1234... at block hash 0xabcd... +Add a matching 'ethCalls' entry to this block in your test JSON. +``` + +**Fix:** +1. Add the missing mock to the appropriate field in your test block (`ethCalls`, `getBalanceCalls`, or `hasCodeCalls`) +2. If the call is not supposed to happen, check the mapping logic — a code path may be executing unexpectedly + +### Block Handler Not Firing + +**Cause:** Block handlers auto-fire, but might be outside data source's active range. + +**Fix:** +1. Check data source's `startBlock` in manifest +2. Use explicit `"number"` in test blocks to ensure they're >= `startBlock` +3. Verify handler is defined in manifest's `blockHandlers` section + +## Legacy Matchstick Mode + +Fall back to the external Matchstick test runner for backward compatibility: + +```bash +gnd test --matchstick +``` + +This is useful if: +- You have existing Matchstick tests +- You need features not yet supported by the mock-based runner +- You're migrating gradually from Matchstick to the new test format + +## See Also + +- [Subgraph Manifest Documentation](https://thegraph.com/docs/en/developing/creating-a-subgraph/) +- [AssemblyScript Mapping API](https://thegraph.com/docs/en/developing/assemblyscript-api/) +- [GraphQL Schema](https://thegraph.com/docs/en/developing/creating-a-subgraph/#the-graph-ql-schema) diff --git a/gnd/docs/migrating-from-graph-cli.md b/gnd/docs/migrating-from-graph-cli.md new file mode 100644 index 00000000000..6203a6f2f21 --- /dev/null +++ b/gnd/docs/migrating-from-graph-cli.md @@ -0,0 +1,303 @@ +# Migrating from graph-cli to gnd + +`gnd` is designed as a drop-in replacement for `graph-cli`. This guide covers the differences and how to migrate your workflow. + +## TL;DR + +For most users, migration is straightforward: + +```bash +# Instead of: +graph codegen +graph build +graph deploy --studio my-subgraph + +# Use: +gnd codegen +gnd build +gnd deploy my-subgraph # defaults to Studio; prompts for version label in interactive terminals +``` + +## Command Mapping + +| graph-cli | gnd | Notes | +|-----------|-----|-------| +| `graph init` | `gnd init` | Identical flags | +| `graph codegen` | `gnd codegen` | Identical flags | +| `graph build` | `gnd build` | Identical flags | +| `graph deploy` | `gnd deploy` | Defaults to Studio if `--node` not provided; pass `--version-label` in non-interactive mode | +| `graph create` | `gnd create` | Identical flags | +| `graph remove` | `gnd remove` | Identical flags | +| `graph auth` | `gnd auth` | Identical flags | +| `graph add` | `gnd add` | Identical flags | +| `graph test` | `gnd test` | Identical flags | +| `graph clean` | `gnd clean` | Identical flags (new in graph-cli 0.80+) | +| `graph publish` | `gnd publish` | Identical flags | +| `graph indexer` | `gnd indexer` | Delegates to `graph-indexer` — requires `indexer-cli` installed | +| `graph local` | N/A | Not implemented - use graph-node's test infrastructure | +| `graph node` | N/A | Not implemented - use `graphman` | + +## What's the Same + +### Flag Compatibility + +All flags use the same names, short forms, and defaults: + +```bash +# Both work identically +graph codegen -o generated/ --skip-migrations +gnd codegen -o generated/ --skip-migrations + +graph build -t wasm --network mainnet +gnd build -t wasm --network mainnet + +graph deploy -l v1.0.0 --ipfs http://localhost:5001 my-subgraph +gnd deploy -l v1.0.0 --ipfs http://localhost:5001 my-subgraph +``` + +### Output Format + +Same success checkmarks, step descriptions, and progress indicators: + +``` +✔ Generate types for data source: Gravity (...) +✔ Write types to generated/Gravity/Gravity.ts (...) +``` + +### Exit Codes + +- `0` for success +- `1` for any error +- `gnd indexer` passes through the exit code from `graph-indexer` + +### Configuration Files + +`gnd` uses the same configuration files as `graph-cli`: + +- `~/.graph-cli.json` - Deploy keys (from `gnd auth`) +- `networks.json` - Network-specific addresses + +Your existing auth keys work with both tools. + +### Generated Code + +Code generation produces identical AssemblyScript output (after formatting). Your existing subgraph code will work without changes. + +## What's Different + +### Indexer Commands + +`gnd indexer` provides access to indexer management commands via [`indexer-cli`](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-cli). Requires `graph-indexer` on `$PATH`: + +```bash +npm install -g @graphprotocol/indexer-cli + +gnd indexer status --network arbitrum-one +gnd indexer rules get all --network mainnet +gnd indexer allocations get --network arbitrum-one +``` + +Note: `gnd indexer --help` shows gnd's own help and works without `graph-indexer` installed. Use `gnd indexer help` to see the full list of `graph-indexer` commands. + +### Commands Not Available + +#### `graph local` + +Not implemented in `gnd`. Use graph-node's built-in integration test infrastructure or Docker compose setups instead. + +#### `graph node` + +Not implemented. Use `graphman` for node management operations: + +```bash +# Instead of graph node commands, use graphman: +graphman info subgraph-name +graphman reassign subgraph-name shard +``` + +### Debug Logging + +```bash +# graph-cli uses: +DEBUG=graph-cli:* graph codegen + +# gnd uses: +RUST_LOG=gnd=debug gnd codegen +``` + +### The `--uncrashable` Flag + +The `--uncrashable` flag on `codegen` (Float Capital's uncrashable helper generation) is not implemented. This is a third-party feature that most subgraphs don't use. + +### Minor Code Generation Differences + +These differences are intentional and documented: + +1. **Int8 import**: `gnd` always imports `Int8` in generated code for simplicity, even when not used. This doesn't affect functionality. + +2. **Trailing commas**: `gnd` uses trailing commas in multi-line constructs: + ```typescript + // gnd output + new Foo( + param1, + param2, // trailing comma + ) + + // graph-cli output + new Foo( + param1, + param2 + ) + ``` + +3. **2D array accessors**: `gnd` uses the correct `toStringMatrix()` method for 2D GraphQL arrays, fixing a bug in graph-cli that used `toStringArray()`. + +## Migration Steps + +### Step 1: Install gnd + +Build from source in the graph-node repository: + +```bash +cargo build -p gnd --release +# Binary at target/release/gnd +``` + +### Step 2: Verify Your Auth + +Your existing `~/.graph-cli.json` works with both tools. Verify with: + +```bash +# Check stored keys +cat ~/.graph-cli.json + +# Or just run a deploy dry-run to see if auth is working +gnd deploy --help +``` + +### Step 3: Test Locally + +Run your normal workflow with gnd: + +```bash +gnd codegen +gnd build +``` + +Compare the output to graph-cli if you want to verify: + +```bash +# graph-cli +graph codegen -o generated-graph/ +graph build -o build-graph/ + +# gnd +gnd codegen -o generated-gnd/ +gnd build -o build-gnd/ + +# Compare (should be identical after formatting) +diff -r generated-graph generated-gnd +diff -r build-graph build-gnd +``` + +### Step 4: Update CI/CD + +Replace `graph` with `gnd` in your CI configuration: + +```yaml +# Before +- run: graph codegen +- run: graph build +- run: graph deploy --studio -l ${{ github.sha }} ${{ secrets.SUBGRAPH_NAME }} + +# After +- run: gnd codegen +- run: gnd build +- run: gnd deploy -l ${{ github.sha }} ${{ secrets.SUBGRAPH_NAME }} +``` + +### Step 5: Update package.json Scripts (Optional) + +If you have npm scripts calling graph-cli: + +```json +{ + "scripts": { + "codegen": "gnd codegen", + "build": "gnd build", + "deploy": "gnd deploy -l $VERSION_LABEL" + } +} +``` + +Set `VERSION_LABEL` in your environment (for example, from a git tag or commit SHA). + +## Troubleshooting + +### "Command not found: gnd" + +Make sure the gnd binary is in your PATH: + +```bash +export PATH="$PATH:/path/to/graph-node/target/release" +``` + +### "prettier: command not found" + +`gnd codegen` shells out to `prettier` for formatting. Install it: + +```bash +npm install -g prettier +# or +pnpm add -g prettier +``` + +### "asc: command not found" + +`gnd build` shells out to the AssemblyScript compiler. Install it: + +```bash +npm install -g assemblyscript +# or have it in your subgraph's node_modules +npm install --save-dev assemblyscript +``` + +### Different Output After codegen + +If you notice differences in generated code: + +1. Check if it's one of the documented differences (Int8 import, trailing commas) +2. Run prettier on both outputs to normalize formatting +3. Report any unexpected differences as a bug + +### Authentication Issues + +Keys stored by graph-cli work with gnd and vice versa. If you have issues: + +```bash +# Re-authenticate +gnd auth YOUR_DEPLOY_KEY + +# Verify the key was saved +cat ~/.graph-cli.json +``` + +## Why Migrate? + +### Benefits of gnd + +- **Native performance**: Rust implementation is faster for large subgraphs +- **Integrated with graph-node**: Same codebase, consistent behavior +- **Better error messages**: Leverages graph-node's validation +- **`gnd dev` command**: Run graph-node in development mode + +### When to Keep Using graph-cli + +- If you need multi-protocol support (NEAR, Cosmos, Arweave, Substreams) - gnd currently focuses on Ethereum +- If you use the `--uncrashable` flag +- If you need the `graph local` or `graph node` commands + +## Getting Help + +- File issues at: https://github.com/graphprotocol/graph-node/issues +- Check gnd version: `gnd --version` diff --git a/gnd/npm/README.md b/gnd/npm/README.md new file mode 100644 index 00000000000..acca621b4dc --- /dev/null +++ b/gnd/npm/README.md @@ -0,0 +1,35 @@ +# The Graph CLI + +This package installs the CLI for developing subgraphs for [The Graph +Network](https://thegraph.com/subgraphs/) It supports all aspects of +[developing, testing, and deploying subgraphs](https://thegraph.com/docs/en/subgraphs/quick-start/) locally and +on the network. + +This package is a complete replacement for the older +[graph-cli](https://www.npmjs.com/package/@graphprotocol/graph-cli) which +will over time be migrated to be a simple wrapper for `gnd`. Older +documentation will reference `graph-cli` in its instructions; running +`alias graph=gnd` in the shell make it possible to follow these +instructions verbatim with `gnd`. + +Besides the tools to develop subgraphs, `gnd` also contains a version of +[graph-node](https://github.com/graphprotocol/graph-node) tailored to +running subgraphs locally via `gnd dev`. + + +## Getting started + +Run `npm install -g @graphprotocol/gnd` to install `gnd`. + +After installation, you can create and run an example subgraph locally with +```bash +gnd init --from-example ethereum-gravatar 'My new subgraph' new-subgraph +cd new-subgraph +gnd codegen +gnd build +gnd dev --ethereum-rpc 'mainnet:' +``` + +If you have an existing contract for which you want to write a subgraph, +have a look at `gnd help init` and the `--from-contract` option. You can +also simply run `gnd init` and follow the prompts to set up your subgraph. diff --git a/gnd/npm/bin/gnd.js b/gnd/npm/bin/gnd.js new file mode 100644 index 00000000000..82a7ae5bf09 --- /dev/null +++ b/gnd/npm/bin/gnd.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node + +const { execFileSync } = require("child_process"); +const path = require("path"); + +const PLATFORMS = { + "darwin-arm64": "@graphprotocol/gnd-darwin-arm64", + "darwin-x64": "@graphprotocol/gnd-darwin-x64", + "linux-arm64": "@graphprotocol/gnd-linux-arm64", + "linux-x64": "@graphprotocol/gnd-linux-x64", + "win32-x64": "@graphprotocol/gnd-win32-x64", +}; + +const key = `${process.platform}-${process.arch}`; +const pkg = PLATFORMS[key]; +if (!pkg) { + console.error(`Unsupported platform: ${key}`); + process.exit(1); +} + +const bin = process.platform === "win32" ? "gnd.exe" : "gnd"; +const binPath = path.join( + require.resolve(`${pkg}/package.json`), + "..", + "bin", + bin +); + +try { + execFileSync(binPath, process.argv.slice(2), { stdio: "inherit" }); +} catch (e) { + process.exit(e.status ?? 1); +} diff --git a/gnd/src/abi.rs b/gnd/src/abi.rs new file mode 100644 index 00000000000..0a5fa695cf4 --- /dev/null +++ b/gnd/src/abi.rs @@ -0,0 +1,101 @@ +//! ABI normalization utilities. +//! +//! Handles extraction of bare ABI arrays from various artifact formats +//! (raw arrays, Hardhat/Foundry, Truffle). + +use anyhow::{Context, Result, anyhow}; + +/// Normalize ABI JSON to extract the actual ABI array from various artifact formats. +/// +/// Supports: +/// - Raw ABI array: `[{...}]` +/// - Foundry/Hardhat format: `{"abi": [...], ...}` +/// - Truffle format: `{"compilerOutput": {"abi": [...], ...}, ...}` +pub fn normalize_abi_json(abi_str: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(abi_str).context("Failed to parse ABI JSON")?; + + // Case 1: Already an array - return as-is + if value.is_array() { + return Ok(value); + } + + // Case 2: Object with "abi" field (Foundry/Hardhat format) + if let Some(abi) = value.get("abi") + && abi.is_array() + { + return Ok(abi.clone()); + } + + // Case 3: Object with "compilerOutput.abi" field (Truffle format) + if let Some(compiler_output) = value.get("compilerOutput") + && let Some(abi) = compiler_output.get("abi") + && abi.is_array() + { + return Ok(abi.clone()); + } + + Err(anyhow!( + "Invalid ABI format: expected an array or an object with 'abi' field" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_abi_json_raw_array() { + let raw_abi = r#"[{"type": "event", "name": "Transfer"}]"#; + let result = normalize_abi_json(raw_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + } + + #[test] + fn test_normalize_abi_json_hardhat_format() { + let hardhat_abi = r#"{ + "_format": "hh-sol-artifact-1", + "contractName": "MyContract", + "abi": [{"type": "event", "name": "Transfer"}], + "bytecode": "0x..." + }"#; + let result = normalize_abi_json(hardhat_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!( + result.as_array().unwrap()[0].get("name").unwrap(), + "Transfer" + ); + } + + #[test] + fn test_normalize_abi_json_truffle_format() { + let truffle_abi = r#"{ + "contractName": "MyContract", + "compilerOutput": { + "abi": [{"type": "event", "name": "Transfer"}] + } + }"#; + let result = normalize_abi_json(truffle_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!( + result.as_array().unwrap()[0].get("name").unwrap(), + "Transfer" + ); + } + + #[test] + fn test_normalize_abi_json_invalid_format() { + let invalid_abi = r#"{"contractName": "MyContract"}"#; + let result = normalize_abi_json(invalid_abi); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Invalid ABI format") + ); + } +} diff --git a/gnd/src/codegen/abi.rs b/gnd/src/codegen/abi.rs new file mode 100644 index 00000000000..9b881fdcbe9 --- /dev/null +++ b/gnd/src/codegen/abi.rs @@ -0,0 +1,1955 @@ +//! ABI code generation for Ethereum contracts. +//! +//! Generates AssemblyScript bindings from contract ABIs: +//! - Event classes with typed parameters +//! - Call classes for function calls with inputs/outputs +//! - Contract class with typed call methods + +use std::collections::HashMap; + +use graph::abi::{ + DynSolType, Event, EventParam, Function, FunctionExt, JsonAbi, Param, StateMutability, +}; +use regex::Regex; + +use super::typescript::{self as ts, Class, ClassMember, Method, ModuleImports, Param as TsParam}; +use crate::shared::{capitalize, handle_reserved_word}; + +/// Resolve a `Param`'s type to `DynSolType`. +fn resolve_param_type(param: &Param) -> DynSolType { + param + .selector_type() + .parse::() + .expect("valid ABI type") +} + +/// Resolve an `EventParam`'s type to `DynSolType`. +fn resolve_event_param_type(param: &EventParam) -> DynSolType { + param + .selector_type() + .parse::() + .expect("valid ABI type") +} + +const GRAPH_TS_MODULE: &str = "@graphprotocol/graph-ts"; + +/// ABI code generator. +pub struct AbiCodeGenerator { + contract: JsonAbi, + name: String, +} + +impl AbiCodeGenerator { + /// Create a new ABI code generator. + pub fn new(contract: JsonAbi, name: impl Into) -> Self { + let mut name = name.into(); + // Sanitize name to be a valid class name + let re = Regex::new(r#"[!@#$%^&*()+\-=\[\]{};':\"|,.<>/?]+"#).unwrap(); + name = re.replace_all(&name, "_").to_string(); + Self { contract, name } + } + + /// Generate module imports for the ABI file. + pub fn generate_module_imports(&self) -> Vec { + vec![ModuleImports::new( + vec![ + "ethereum".to_string(), + "JSONValue".to_string(), + "TypedMap".to_string(), + "Entity".to_string(), + "Bytes".to_string(), + "Address".to_string(), + "BigInt".to_string(), + ], + GRAPH_TS_MODULE, + )] + } + + /// Generate all types from the ABI. + pub fn generate_types(&self) -> Vec { + let mut classes = Vec::new(); + classes.extend(self.generate_event_types()); + classes.extend(self.generate_smart_contract_class()); + classes.extend(self.generate_call_types()); + classes + } + + /// Generate event type classes. + fn generate_event_types(&self) -> Vec { + let mut classes = Vec::new(); + let events = self.disambiguate_events(); + + for (event, alias) in events { + let event_class_name = alias.clone(); + let mut tuple_classes = Vec::new(); + + // Generate params class + let params_class_name = [&event_class_name, "__Params"].concat(); + let mut params_class = ts::klass(¶ms_class_name).exported(); + params_class.add_member(ClassMember::new("_event", &event_class_name)); + params_class.add_method(Method::new( + "constructor", + vec![TsParam::new("event", ts::NamedType::new(&event_class_name))], + None, + "this._event = event", + )); + + // Generate getters for event params + let inputs = self.disambiguate_event_params(&event.inputs, "param"); + for (index, (param, param_name)) in inputs.iter().enumerate() { + let param_object = self.generate_event_param( + param, + param_name, + index, + &event_class_name, + &mut tuple_classes, + ); + params_class.add_method(param_object); + } + + // Generate event class + let mut event_class = ts::klass(&event_class_name) + .exported() + .extends("ethereum.Event"); + event_class.add_method(Method::new( + "get params", + vec![], + Some(ts::NamedType::new(¶ms_class_name).into()), + format!("return new {}(this)", params_class_name), + )); + + classes.push(event_class); + classes.push(params_class); + classes.extend(tuple_classes); + } + + classes + } + + /// Generate the smart contract class with call methods. + fn generate_smart_contract_class(&self) -> Vec { + let mut classes = Vec::new(); + + let mut contract_class = ts::klass(&self.name) + .exported() + .extends("ethereum.SmartContract"); + + // Add static bind method + let contract_name = &self.name; + contract_class.add_static_method(ts::StaticMethod::new( + "bind", + vec![TsParam::new("address", ts::NamedType::new("Address"))], + ts::NamedType::new(&self.name), + format!("return new {}('{}', address)", contract_name, contract_name), + )); + + // Get callable functions and sort alphabetically for deterministic output + let mut functions = self.get_callable_functions(); + functions.sort_by(|a, b| a.name.cmp(&b.name)); + let disambiguated = self.disambiguate_functions(&functions); + + for (func, alias) in disambiguated { + let (method, try_method, result_classes) = self.generate_function_methods(func, &alias); + contract_class.add_method(method); + contract_class.add_method(try_method); + classes.extend(result_classes); + } + + classes.push(contract_class); + classes + } + + /// Generate call type classes. + fn generate_call_types(&self) -> Vec { + let mut classes = Vec::new(); + let mut functions = self.get_call_functions(); + functions.sort_by(|a, b| a.name.cmp(&b.name)); + let disambiguated = self.disambiguate_call_functions(&functions); + + for (func, alias) in disambiguated { + let cap_alias = capitalize(&alias); + let call_class_name = format!("{}Call", cap_alias); + let mut tuple_classes = Vec::new(); + + // Generate inputs class + let inputs_class_name = [&call_class_name, "__Inputs"].concat(); + let mut inputs_class = ts::klass(&inputs_class_name).exported(); + inputs_class.add_member(ClassMember::new("_call", &call_class_name)); + inputs_class.add_method(Method::new( + "constructor", + vec![TsParam::new("call", ts::NamedType::new(&call_class_name))], + None, + "this._call = call", + )); + + let inputs = self.disambiguate_params(&func.inputs, "value"); + for (index, (param, param_name)) in inputs.iter().enumerate() { + let getter = self.generate_input_output_getter( + param, + param_name, + index, + &call_class_name, + "call", + "inputValues", + &mut tuple_classes, + ); + inputs_class.add_method(getter); + } + + // Generate outputs class + let outputs_class_name = [&call_class_name, "__Outputs"].concat(); + let mut outputs_class = ts::klass(&outputs_class_name).exported(); + outputs_class.add_member(ClassMember::new("_call", &call_class_name)); + outputs_class.add_method(Method::new( + "constructor", + vec![TsParam::new("call", ts::NamedType::new(&call_class_name))], + None, + "this._call = call", + )); + + let outputs = self.disambiguate_params(&func.outputs, "value"); + for (index, (param, param_name)) in outputs.iter().enumerate() { + let getter = self.generate_input_output_getter( + param, + param_name, + index, + &call_class_name, + "call", + "outputValues", + &mut tuple_classes, + ); + outputs_class.add_method(getter); + } + + // Generate call class + let mut call_class = ts::klass(&call_class_name) + .exported() + .extends("ethereum.Call"); + call_class.add_method(Method::new( + "get inputs", + vec![], + Some(ts::NamedType::new(&inputs_class_name).into()), + format!("return new {}(this)", inputs_class_name), + )); + call_class.add_method(Method::new( + "get outputs", + vec![], + Some(ts::NamedType::new(&outputs_class_name).into()), + format!("return new {}(this)", outputs_class_name), + )); + + classes.push(call_class); + classes.push(inputs_class); + classes.push(outputs_class); + classes.extend(tuple_classes); + } + + classes + } + + /// Generate a getter method for an event parameter. + fn generate_event_param( + &self, + param: &EventParam, + name: &str, + index: usize, + event_class_name: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_event_param_type(param); + + // Handle indexed params - strings, bytes and arrays are hashed to bytes32 + let value_type = if param.indexed { + indexed_input_type(¶m_type) + } else { + param_type.clone() + }; + + if contains_tuple_type(&value_type) { + self.generate_tuple_getter_for_event_param( + param, + ¶m_type, + name, + index, + event_class_name, + "event", + "parameters", + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(&value_type); + let access = format!("this._event.parameters[{}].value", index); + let conversion = ethereum_to_asc(&access, &value_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate a getter for call inputs/outputs. + #[allow(clippy::too_many_arguments)] + fn generate_input_output_getter( + &self, + param: &Param, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_getter( + param, + ¶m_type, + name, + index, + parent_class, + parent_type, + parent_field, + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(¶m_type); + let access = format!("this._{}.{}[{}].value", parent_type, parent_field, index); + let conversion = ethereum_to_asc(&access, ¶m_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate a tuple getter and its associated classes (for `Param`). + #[allow(clippy::too_many_arguments)] + fn generate_tuple_getter( + &self, + param: &Param, + param_type: &DynSolType, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let cap_name = capitalize(name); + let tuple_identifier = format!("{}{}", parent_class, cap_name); + let tuple_class_name = if parent_field == "outputValues" { + format!("{}OutputStruct", tuple_identifier) + } else { + format!("{}Struct", tuple_identifier) + }; + + let is_tuple = matches!(param_type, DynSolType::Tuple(_)); + let access_code = if parent_type == "tuple" { + format!("this[{}]", index) + } else { + format!("this._{}.{}[{}].value", parent_type, parent_field, index) + }; + + let return_value = ethereum_to_asc(&access_code, param_type, Some(&tuple_class_name)); + + let return_type = if is_tuple_matrix_type(param_type) { + format!("Array>", tuple_class_name) + } else if is_tuple_array_type(param_type) { + format!("Array<{}>", tuple_class_name) + } else { + tuple_class_name.clone() + }; + + let body = if is_tuple { + format!("return changetype<{}>({})", tuple_class_name, return_value) + } else { + format!("return {}", return_value) + }; + + // Generate tuple class from Param's components + let components = get_tuple_param_components(param); + if !components.is_empty() { + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_getter = self.generate_tuple_component_getter( + component, + component_name, + idx, + &tuple_identifier, + tuple_classes, + ); + tuple_class.add_method(component_getter); + } + + tuple_classes.push(tuple_class); + } + + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } + + /// Generate a tuple getter and its associated classes (for `EventParam`). + #[allow(clippy::too_many_arguments)] + fn generate_tuple_getter_for_event_param( + &self, + param: &EventParam, + param_type: &DynSolType, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let cap_name = capitalize(name); + let tuple_identifier = format!("{}{}", parent_class, cap_name); + let tuple_class_name = if parent_field == "outputValues" { + format!("{}OutputStruct", tuple_identifier) + } else { + format!("{}Struct", tuple_identifier) + }; + + let is_tuple = matches!(param_type, DynSolType::Tuple(_)); + let access_code = if parent_type == "tuple" { + format!("this[{}]", index) + } else { + format!("this._{}.{}[{}].value", parent_type, parent_field, index) + }; + + let return_value = ethereum_to_asc(&access_code, param_type, Some(&tuple_class_name)); + + let return_type = if is_tuple_matrix_type(param_type) { + format!("Array>", tuple_class_name) + } else if is_tuple_array_type(param_type) { + format!("Array<{}>", tuple_class_name) + } else { + tuple_class_name.clone() + }; + + let body = if is_tuple { + format!("return changetype<{}>({})", tuple_class_name, return_value) + } else { + format!("return {}", return_value) + }; + + // Generate tuple class from EventParam's components + // EventParam.components is Vec, same as Param.components + let components = ¶m.components; + if !components.is_empty() { + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_getter = self.generate_tuple_component_getter( + component, + component_name, + idx, + &tuple_identifier, + tuple_classes, + ); + tuple_class.add_method(component_getter); + } + + tuple_classes.push(tuple_class); + } + + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } + + /// Generate a getter for a tuple component. + fn generate_tuple_component_getter( + &self, + param: &Param, + name: &str, + index: usize, + parent_class: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_getter( + param, + ¶m_type, + name, + index, + parent_class, + "tuple", + "", + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(¶m_type); + let access = format!("this[{}]", index); + let conversion = ethereum_to_asc(&access, ¶m_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate methods for a callable function. + fn generate_function_methods( + &self, + func: &Function, + alias: &str, + ) -> (Method, Method, Vec) { + let mut result_classes = Vec::new(); + let fn_signature = func.signature_compat(); + let contract_name = &self.name; + let tuple_result_parent_type = [contract_name, "__", alias, "Result"].concat(); + let tuple_input_parent_type = [contract_name, "__", alias, "Input"].concat(); + + // Disambiguate outputs + let outputs = self.disambiguate_params(&func.outputs, "value"); + + // Determine return type + let (return_type, simple_return_type) = if outputs.len() > 1 { + // Multiple outputs - create a result struct + let result_class = self.generate_result_class( + &outputs, + &tuple_result_parent_type, + &mut result_classes, + ); + result_classes.push(result_class.clone()); + (result_class.name.clone(), false) + } else if !outputs.is_empty() { + let (param, _) = &outputs[0]; + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + let tuple_name = self.generate_tuple_return_type( + param, + ¶m_type, + 0, + &tuple_result_parent_type, + &mut result_classes, + ); + (tuple_name, true) + } else { + (asc_type_for_ethereum(¶m_type), true) + } + } else { + ("void".to_string(), true) + }; + + // Disambiguate inputs + let inputs = self.disambiguate_params(&func.inputs, "param"); + + // Generate tuple types for inputs + for (index, (param, _)) in inputs.iter().enumerate() { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_class_for_input( + param, + index, + &tuple_input_parent_type, + &mut result_classes, + ); + } + } + + // Build params + let params: Vec = inputs + .iter() + .enumerate() + .map(|(index, (param, name))| { + let p_type = resolve_param_type(param); + let param_type_str = + get_param_type_for_input(&p_type, index, &tuple_input_parent_type); + TsParam::new(name.clone(), ts::TypeExpr::Raw(param_type_str)) + }) + .collect(); + + // Build call arguments + let call_args: Vec = inputs + .iter() + .map(|(param, name)| { + let p_type = resolve_param_type(param); + ethereum_from_asc(name, &p_type) + }) + .collect(); + + let func_name = &func.name; + let call_args_str = call_args.join(", "); + let super_inputs = format!("'{}', '{}', [{}]", func_name, fn_signature, call_args_str); + + // Generate method body + let method_body = self.generate_call_body( + &outputs, + &return_type, + simple_return_type, + &super_inputs, + &tuple_result_parent_type, + false, + ); + + let try_method_body = self.generate_call_body( + &outputs, + &return_type, + simple_return_type, + &super_inputs, + &tuple_result_parent_type, + true, + ); + + let method = Method::new( + alias.to_string(), + params.clone(), + Some(ts::TypeExpr::Raw(return_type.clone())), + method_body, + ); + + let try_method = Method::new( + format!("try_{}", alias), + params, + Some(ts::TypeExpr::Raw(format!( + "ethereum.CallResult<{}>", + return_type + ))), + try_method_body, + ); + + (method, try_method, result_classes) + } + + /// Generate call method body. + fn generate_call_body( + &self, + outputs: &[(&Param, String)], + return_type: &str, + simple_return_type: bool, + super_inputs: &str, + tuple_result_parent_type: &str, + is_try: bool, + ) -> String { + let nl = "\n"; + let (call_stmt, result_var) = if is_try { + let mut lines = Vec::new(); + lines.push(format!("let result = super.tryCall({})", super_inputs)); + lines.push(" if (result.reverted) {".to_string()); + lines.push(" return new ethereum.CallResult()".to_string()); + lines.push(" }".to_string()); + lines.push(" let value = result.value".to_string()); + (lines.join(nl), "value") + } else { + ( + format!("let result = super.call({})", super_inputs), + "result", + ) + }; + + let return_val = if simple_return_type { + if outputs.is_empty() { + String::new() + } else { + let (param, _) = &outputs[0]; + let p_type = resolve_param_type(param); + let tuple_name = if is_tuple_array_type(&p_type) { + Some(tuple_type_name(0, tuple_result_parent_type)) + } else { + None + }; + let val = ethereum_to_asc( + &format!("{}[0]", result_var), + &p_type, + tuple_name.as_deref(), + ); + if matches!(p_type, DynSolType::Tuple(_)) { + format!("changetype<{}>({})", return_type, val) + } else { + val + } + } + } else { + let conversions: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let tuple_name = if is_tuple_array_type(&p_type) { + Some(tuple_type_name(index, tuple_result_parent_type)) + } else { + None + }; + let val = ethereum_to_asc( + &format!("{}[{}]", result_var, index), + &p_type, + tuple_name.as_deref(), + ); + if matches!(p_type, DynSolType::Tuple(_)) { + let tn = tuple_type_name(index, tuple_result_parent_type); + format!("changetype<{}>({})", tn, val) + } else { + val + } + }) + .collect(); + let conv_str = conversions.join(", "); + format!("new {}({})", return_type, conv_str) + }; + + if is_try { + [ + &call_stmt, + nl, + " return ethereum.CallResult.fromValue(", + &return_val, + ")", + ] + .concat() + } else if outputs.is_empty() { + call_stmt + } else { + [&call_stmt, nl, nl, " return (", &return_val, ")"].concat() + } + } + + /// Generate a result class for multiple outputs. + fn generate_result_class( + &self, + outputs: &[(&Param, String)], + tuple_result_parent_type: &str, + result_classes: &mut Vec, + ) -> Class { + let class_name = tuple_result_parent_type.to_string(); + let mut klass = ts::klass(&class_name).exported(); + + // Add constructor + let constructor_params: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let param_type_str = + get_param_type_for_input(&p_type, index, tuple_result_parent_type); + TsParam::new(format!("value{}", index), ts::TypeExpr::Raw(param_type_str)) + }) + .collect(); + + let nl = "\n"; + let constructor_body: Vec = outputs + .iter() + .enumerate() + .map(|(index, _)| format!("this.value{} = value{}", index, index)) + .collect(); + + klass.add_method(Method::new( + "constructor", + constructor_params, + None, + constructor_body.join(&format!("{} ", nl)), + )); + + // Add toMap method + let map_entries: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let this_val = format!("this.value{}", index); + let from_asc = ethereum_from_asc(&this_val, &p_type); + format!("map.set('value{}', {})", index, from_asc) + }) + .collect(); + + let map_body = [ + "let map = new TypedMap()", + nl, + " ", + &map_entries.join(&format!("{} ", nl)), + nl, + " return map", + ] + .concat(); + + klass.add_method(Method::new( + "toMap", + vec![], + Some(ts::TypeExpr::Raw( + "TypedMap".to_string(), + )), + map_body, + )); + + // Add members + for (index, (param, _)) in outputs.iter().enumerate() { + let p_type = resolve_param_type(param); + let param_type_str = get_param_type_for_input(&p_type, index, tuple_result_parent_type); + klass.add_member(ClassMember::new(format!("value{}", index), param_type_str)); + } + + // Add getters for outputs + for (index, (param, _)) in outputs.iter().enumerate() { + let getter_name = if param.name.trim().is_empty() { + format!("getValue{}", index) + } else { + let cap = capitalize(¶m.name); + format!("get{}", cap) + }; + let p_type = resolve_param_type(param); + let param_type_str = get_param_type_for_input(&p_type, index, tuple_result_parent_type); + klass.add_method(Method::new( + getter_name, + vec![], + Some(ts::TypeExpr::Raw(param_type_str)), + format!("return this.value{}", index), + )); + } + + // Generate tuple classes for outputs + for (index, (param, _)) in outputs.iter().enumerate() { + let p_type = resolve_param_type(param); + if contains_tuple_type(&p_type) { + self.generate_tuple_class_for_input( + param, + index, + tuple_result_parent_type, + result_classes, + ); + } + } + + klass + } + + /// Generate tuple return type name and classes. + fn generate_tuple_return_type( + &self, + param: &Param, + param_type: &DynSolType, + index: usize, + parent_type: &str, + result_classes: &mut Vec, + ) -> String { + self.generate_tuple_class_for_input(param, index, parent_type, result_classes); + let tn = tuple_type_name(index, parent_type); + if is_tuple_array_type(param_type) { + format!("Array<{}>", tn) + } else if is_tuple_matrix_type(param_type) { + format!("Array>", tn) + } else { + tn + } + } + + /// Generate tuple class for an input/output. + fn generate_tuple_class_for_input( + &self, + param: &Param, + index: usize, + parent_type: &str, + result_classes: &mut Vec, + ) { + let tuple_class_name = tuple_type_name(index, parent_type); + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let components = get_tuple_param_components(param); + if !components.is_empty() { + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_type = resolve_param_type(component); + let getter = if contains_tuple_type(&component_type) { + // Recursively generate tuple classes + let cap = capitalize(&format!("{}", index)); + let nested_parent = format!("{}Value{}", parent_type, cap); + self.generate_tuple_class_for_input( + component, + idx, + &nested_parent, + result_classes, + ); + let nested_tuple_name = tuple_type_name(idx, &nested_parent); + let access = format!("this[{}]", idx); + let conversion = + ethereum_to_asc(&access, &component_type, Some(&nested_tuple_name)); + let return_type = if is_tuple_array_type(&component_type) { + format!("Array<{}>", nested_tuple_name) + } else { + nested_tuple_name.clone() + }; + let body = if matches!(component_type, DynSolType::Tuple(_)) { + format!("return changetype<{}>({})", nested_tuple_name, conversion) + } else { + format!("return {}", conversion) + }; + Method::new( + format!("get {}", component_name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } else { + let asc_type = asc_type_for_ethereum(&component_type); + let access = format!("this[{}]", idx); + let conversion = ethereum_to_asc(&access, &component_type, None); + Method::new( + format!("get {}", component_name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + }; + tuple_class.add_method(getter); + } + } + + result_classes.push(tuple_class); + } + + /// Get callable functions (view, pure, nonpayable, constant with outputs). + fn get_callable_functions(&self) -> Vec<&Function> { + self.contract + .functions() + .filter(|f| { + !f.outputs.is_empty() + && matches!( + f.state_mutability, + StateMutability::View | StateMutability::Pure | StateMutability::NonPayable + ) + }) + .collect() + } + + /// Get functions that can be used as calls (non-view, non-pure functions). + fn get_call_functions(&self) -> Vec<&Function> { + self.contract + .functions() + .filter(|f| { + matches!( + f.state_mutability, + StateMutability::NonPayable | StateMutability::Payable + ) + }) + .collect() + } + + /// Disambiguate events with duplicate names. + fn disambiguate_events(&self) -> Vec<(&Event, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for event in self.contract.events() { + let name = handle_reserved_word(&event.name); + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((event, alias)); + } + + result + } + + /// Disambiguate functions. + fn disambiguate_functions<'a>( + &self, + functions: &[&'a Function], + ) -> Vec<(&'a Function, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for func in functions { + let name = handle_reserved_word(&func.name); + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((*func, alias)); + } + + result + } + + /// Disambiguate call functions. + fn disambiguate_call_functions<'a>( + &self, + functions: &[&'a Function], + ) -> Vec<(&'a Function, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for func in functions { + let name = if func.name.is_empty() { + "default".to_string() + } else { + handle_reserved_word(&func.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((*func, alias)); + } + + result + } + + /// Disambiguate event params. + fn disambiguate_event_params<'a>( + &self, + params: &'a [EventParam], + default_prefix: &str, + ) -> Vec<(&'a EventParam, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for (index, param) in params.iter().enumerate() { + let name = if param.name.is_empty() { + format!("{}{}", default_prefix, index) + } else { + handle_reserved_word(¶m.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let disambiguated = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((param, disambiguated)); + } + + result + } + + /// Disambiguate function params. + fn disambiguate_params<'a>( + &self, + params: &'a [Param], + default_prefix: &str, + ) -> Vec<(&'a Param, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for (index, param) in params.iter().enumerate() { + let name = if param.name.is_empty() { + format!("{}{}", default_prefix, index) + } else { + handle_reserved_word(¶m.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let disambiguated = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((param, disambiguated)); + } + + result + } +} + +/// Get tuple type name for a param. +fn tuple_type_name(index: usize, parent_type: &str) -> String { + format!("{}Value{}Struct", parent_type, index) +} + +/// Get the param type string for an input, handling tuples. +fn get_param_type_for_input(param_type: &DynSolType, index: usize, parent_type: &str) -> String { + if matches!(param_type, DynSolType::Tuple(_)) { + tuple_type_name(index, parent_type) + } else if is_tuple_matrix_type(param_type) { + let tn = tuple_type_name(index, parent_type); + format!("Array>", tn) + } else if is_tuple_array_type(param_type) { + let tn = tuple_type_name(index, parent_type); + format!("Array<{}>", tn) + } else { + asc_type_for_ethereum(param_type) + } +} + +/// Get the tuple components from a `Param`, following through arrays. +/// Returns the `components` from the innermost tuple `Param`. +fn get_tuple_param_components(param: &Param) -> &[Param] { + // If this param has components directly (tuple type), return them + if !param.components.is_empty() { + return ¶m.components; + } + // For array-of-tuple types, the components are on the param itself + // (alloy stores them on the outer param for tuple[] types) + &[] +} + +/// Disambiguate tuple components, reading names directly from `Param.name`. +fn disambiguate_tuple_components(components: &[Param]) -> Vec<(&Param, String)> { + components + .iter() + .enumerate() + .map(|(index, component)| { + let name = if component.name.is_empty() { + format!("value{}", index) + } else { + component.name.clone() + }; + (component, name) + }) + .collect() +} + +/// Get AssemblyScript type for an Ethereum type. +fn asc_type_for_ethereum(param_type: &DynSolType) -> String { + match param_type { + DynSolType::Address => "Address".to_string(), + DynSolType::Bool => "boolean".to_string(), + DynSolType::Bytes => "Bytes".to_string(), + DynSolType::FixedBytes(_) => "Bytes".to_string(), + DynSolType::Int(bits) => { + if *bits <= 32 { + "i32".to_string() + } else { + "BigInt".to_string() + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + "i32".to_string() + } else { + "BigInt".to_string() + } + } + DynSolType::String => "string".to_string(), + DynSolType::Array(inner) => { + let inner_type = asc_type_for_ethereum(inner); + format!("Array<{}>", inner_type) + } + DynSolType::FixedArray(inner, _) => { + let inner_type = asc_type_for_ethereum(inner); + format!("Array<{}>", inner_type) + } + DynSolType::Tuple(_) => "ethereum.Tuple".to_string(), + _ => "ethereum.Tuple".to_string(), // Function and other future variants + } +} + +/// Convert ethereum value to AssemblyScript. +fn ethereum_to_asc(code: &str, param_type: &DynSolType, tuple_type: Option<&str>) -> String { + match param_type { + DynSolType::Address => format!("{}.toAddress()", code), + DynSolType::Bool => format!("{}.toBoolean()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => format!("{}.toBytes()", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32()", code) + } else { + format!("{}.toBigInt()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32()", code) + } else { + format!("{}.toBigInt()", code) + } + } + DynSolType::String => format!("{}.toString()", code), + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => match inner.as_ref() { + DynSolType::Address => format!("{}.toAddressArray()", code), + DynSolType::Bool => format!("{}.toBooleanArray()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => { + format!("{}.toBytesArray()", code) + } + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32Array()", code) + } else { + format!("{}.toBigIntArray()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32Array()", code) + } else { + format!("{}.toBigIntArray()", code) + } + } + DynSolType::String => format!("{}.toStringArray()", code), + DynSolType::Tuple(_) => { + if let Some(tuple_name) = tuple_type { + format!("{}.toTupleArray<{}>()", code, tuple_name) + } else { + format!("{}.toTupleArray()", code) + } + } + DynSolType::Array(inner2) | DynSolType::FixedArray(inner2, _) => { + ethereum_to_asc_matrix(code, inner2.as_ref(), tuple_type) + } + _ => format!("{}.toString()", code), // fallback for Function etc. + }, + DynSolType::Tuple(_) => format!("{}.toTuple()", code), + _ => format!("{}.toTuple()", code), // fallback for Function etc. + } +} + +/// Convert matrix type to AssemblyScript. +fn ethereum_to_asc_matrix(code: &str, inner_type: &DynSolType, tuple_type: Option<&str>) -> String { + match inner_type { + DynSolType::Address => format!("{}.toAddressMatrix()", code), + DynSolType::Bool => format!("{}.toBooleanMatrix()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => format!("{}.toBytesMatrix()", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32Matrix()", code) + } else { + format!("{}.toBigIntMatrix()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32Matrix()", code) + } else { + format!("{}.toBigIntMatrix()", code) + } + } + DynSolType::String => format!("{}.toStringMatrix()", code), + DynSolType::Tuple(_) => { + if let Some(tuple_name) = tuple_type { + format!("{}.toTupleMatrix<{}>()", code, tuple_name) + } else { + format!("{}.toTupleMatrix()", code) + } + } + _ => format!("{}.toStringMatrix()", code), // fallback + } +} + +/// Convert AssemblyScript value to ethereum value. +fn ethereum_from_asc(code: &str, param_type: &DynSolType) -> String { + match param_type { + DynSolType::Address => format!("ethereum.Value.fromAddress({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBoolean({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytes({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytes({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32({})", code) + } else { + format!("ethereum.Value.fromSignedBigInt({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!( + "ethereum.Value.fromUnsignedBigInt(BigInt.fromI32({}))", + code + ) + } else { + format!("ethereum.Value.fromUnsignedBigInt({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromString({})", code), + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => { + ethereum_from_asc_array(code, inner.as_ref()) + } + DynSolType::Tuple(_) => format!("ethereum.Value.fromTuple({})", code), + _ => format!("ethereum.Value.fromTuple({})", code), // fallback + } +} + +/// Convert array to ethereum value. +fn ethereum_from_asc_array(code: &str, inner_type: &DynSolType) -> String { + match inner_type { + DynSolType::Address => format!("ethereum.Value.fromAddressArray({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBooleanArray({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytesArray({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytesArray({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32Array({})", code) + } else { + format!("ethereum.Value.fromSignedBigIntArray({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("ethereum.Value.fromI32Array({})", code) + } else { + format!("ethereum.Value.fromUnsignedBigIntArray({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromStringArray({})", code), + DynSolType::Tuple(_) => format!("ethereum.Value.fromTupleArray({})", code), + DynSolType::Array(inner2) | DynSolType::FixedArray(inner2, _) => { + ethereum_from_asc_matrix(code, inner2.as_ref()) + } + _ => format!("ethereum.Value.fromStringArray({})", code), // fallback + } +} + +/// Convert matrix to ethereum value. +fn ethereum_from_asc_matrix(code: &str, inner_type: &DynSolType) -> String { + match inner_type { + DynSolType::Address => format!("ethereum.Value.fromAddressMatrix({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBooleanMatrix({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytesMatrix({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytesMatrix({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32Matrix({})", code) + } else { + format!("ethereum.Value.fromSignedBigIntMatrix({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("ethereum.Value.fromI32Matrix({})", code) + } else { + format!("ethereum.Value.fromUnsignedBigIntMatrix({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromStringMatrix({})", code), + DynSolType::Tuple(_) => format!("ethereum.Value.fromTupleMatrix({})", code), + _ => format!("ethereum.Value.fromStringMatrix({})", code), // fallback + } +} + +/// Check if param type contains a tuple. +fn contains_tuple_type(param_type: &DynSolType) -> bool { + match param_type { + DynSolType::Tuple(_) => true, + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => contains_tuple_type(inner), + _ => false, + } +} + +/// Check if param type is a tuple array. +fn is_tuple_array_type(param_type: &DynSolType) -> bool { + matches!( + param_type, + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) + if matches!(inner.as_ref(), DynSolType::Tuple(_)) + ) +} + +/// Check if param type is a tuple matrix (2D array). +fn is_tuple_matrix_type(param_type: &DynSolType) -> bool { + match param_type { + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => is_tuple_array_type(inner), + _ => false, + } +} + +/// Handle indexed input type conversion. +fn indexed_input_type(param_type: &DynSolType) -> DynSolType { + // Strings, bytes, and arrays are encoded and hashed to bytes32 + match param_type { + DynSolType::String | DynSolType::Bytes | DynSolType::Tuple(_) => DynSolType::FixedBytes(32), + DynSolType::Array(_) | DynSolType::FixedArray(_, _) => DynSolType::FixedBytes(32), + _ => param_type.clone(), + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn parse_abi(json: &str) -> JsonAbi { + serde_json::from_str(json).unwrap() + } + + #[test] + fn test_simple_event() { + let abi_json = r#"[ + { + "type": "event", + "name": "Transfer", + "inputs": [ + {"name": "from", "type": "address", "indexed": true}, + {"name": "to", "type": "address", "indexed": true}, + {"name": "value", "type": "uint256", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + assert!(types.iter().any(|c| c.name == "Transfer")); + assert!(types.iter().any(|c| c.name == "Transfer__Params")); + } + + #[test] + fn test_function_with_outputs() { + let abi_json = r#"[ + { + "type": "function", + "name": "balanceOf", + "inputs": [{"name": "owner", "type": "address"}], + "outputs": [{"name": "", "type": "uint256"}], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + assert!(types.iter().any(|c| c.name == "Token")); + let token_class = types.iter().find(|c| c.name == "Token").unwrap(); + + assert!(token_class.methods.iter().any(|m| m.name == "balanceOf")); + assert!( + token_class + .methods + .iter() + .any(|m| m.name == "try_balanceOf") + ); + } + + #[test] + fn test_asc_type_for_ethereum() { + assert_eq!(asc_type_for_ethereum(&DynSolType::Address), "Address"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Bool), "boolean"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Uint(256)), "BigInt"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Uint(8)), "i32"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Int(32)), "i32"); + assert_eq!(asc_type_for_ethereum(&DynSolType::String), "string"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Bytes), "Bytes"); + } + + #[test] + fn test_name_sanitization() { + let generator = AbiCodeGenerator::new(JsonAbi::default(), "Test!Contract@Name"); + assert_eq!(generator.name, "Test_Contract_Name"); + } + + #[test] + fn test_indexed_input_type() { + assert_eq!( + indexed_input_type(&DynSolType::String), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Bytes), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Array(Box::new(DynSolType::Uint(256)))), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Address), + DynSolType::Address + ); + assert_eq!( + indexed_input_type(&DynSolType::Uint(256)), + DynSolType::Uint(256) + ); + } + + /// Test that overloaded events (same name, different inputs) are disambiguated. + /// The TS CLI generates unique names like Transfer, Transfer1, Transfer2. + #[test] + fn test_overloaded_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "Transfer", + "inputs": [], + "anonymous": false + }, + { + "type": "event", + "name": "Transfer", + "inputs": [{"name": "to", "type": "address", "indexed": false}], + "anonymous": false + }, + { + "type": "event", + "name": "Transfer", + "inputs": [ + {"name": "from", "type": "address", "indexed": false}, + {"name": "to", "type": "address", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Get all event class names + let event_names: Vec<&str> = types + .iter() + .filter(|c| c.extends == Some("ethereum.Event".to_string())) + .map(|c| c.name.as_str()) + .collect(); + + // Verify we have 3 distinct Transfer events with disambiguation + assert_eq!( + event_names.len(), + 3, + "Should have 3 Transfer event variants" + ); + + // Check that Transfer (with no suffix) exists + assert!( + event_names.contains(&"Transfer"), + "Should have base Transfer event" + ); + + // Check that numbered variants exist (Transfer1, Transfer2) + assert!( + event_names.contains(&"Transfer1"), + "Should have Transfer1 event" + ); + assert!( + event_names.contains(&"Transfer2"), + "Should have Transfer2 event" + ); + } + + /// Test that overloaded functions are disambiguated. + #[test] + fn test_overloaded_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "getSomething", + "inputs": [], + "outputs": [{"name": "result", "type": "bytes32"}], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getSomething", + "inputs": [{"name": "owner", "type": "address"}], + "outputs": [{"name": "result", "type": "bytes32"}], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Find the Token contract class + let token_class = types.iter().find(|c| c.name == "Token").unwrap(); + + // Get all method names (excluding try_ variants and bind) + let method_names: Vec<&str> = token_class + .methods + .iter() + .map(|m| m.name.as_str()) + .filter(|n| !n.starts_with("try_") && *n != "bind") + .collect(); + + // Verify we have disambiguated getSomething variants + assert!( + method_names.contains(&"getSomething"), + "Should have base getSomething method" + ); + assert!( + method_names.contains(&"getSomething1"), + "Should have getSomething1 method" + ); + } + + /// Test that tuple/struct types in function inputs and outputs are handled correctly. + #[test] + fn test_tuple_types_in_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "doSomething", + "inputs": [ + { + "name": "data", + "type": "tuple", + "components": [ + {"name": "owner", "type": "address"}, + {"name": "value", "type": "uint256"} + ] + } + ], + "outputs": [ + { + "name": "result", + "type": "tuple", + "components": [ + {"name": "success", "type": "bool"}, + {"name": "newValue", "type": "uint256"} + ] + } + ], + "stateMutability": "nonpayable" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have main contract class + assert!( + class_names.contains(&"TestContract"), + "Should have TestContract class" + ); + + // Should have struct classes for input and output tuples + // Input struct: TestContract__doSomethingInputValue0Struct + // Output struct: TestContract__doSomethingResultValue0Struct + let has_input_struct = class_names + .iter() + .any(|n| n.contains("Input") && n.contains("Struct")); + let has_output_struct = class_names + .iter() + .any(|n| n.contains("Result") && n.contains("Struct")); + + assert!( + has_input_struct, + "Should have input struct class, found: {:?}", + class_names + ); + assert!( + has_output_struct, + "Should have output/result struct class, found: {:?}", + class_names + ); + + // Verify the output struct extends ethereum.Tuple + let output_struct = types + .iter() + .find(|c| c.name.contains("Result") && c.name.contains("Struct")) + .expect("Should find output struct"); + assert_eq!( + output_struct.extends, + Some("ethereum.Tuple".to_string()), + "Output struct should extend ethereum.Tuple" + ); + + // Verify the output struct has getters for its components + // With alloy, component names are preserved from the ABI + let method_names: Vec<&str> = output_struct + .methods + .iter() + .map(|m| m.name.as_str()) + .collect(); + assert!( + method_names.iter().any(|n| n.contains("success")), + "Should have success getter for first component, found methods: {:?}", + method_names + ); + assert!( + method_names.iter().any(|n| n.contains("newValue")), + "Should have newValue getter for second component, found methods: {:?}", + method_names + ); + } + + /// Test that tuple types in events are handled correctly. + #[test] + fn test_tuple_types_in_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "DataUpdated", + "inputs": [ + {"name": "id", "type": "uint256", "indexed": true}, + { + "name": "data", + "type": "tuple", + "indexed": false, + "components": [ + {"name": "timestamp", "type": "uint256"}, + {"name": "value", "type": "bytes32"} + ] + } + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have event class + assert!( + class_names.contains(&"DataUpdated"), + "Should have DataUpdated event class" + ); + + // Should have params class + assert!( + class_names.contains(&"DataUpdated__Params"), + "Should have DataUpdated__Params class" + ); + + // Should have struct class for the tuple parameter + let has_struct = class_names + .iter() + .any(|n| n.contains("Struct") && n.contains("DataUpdated")); + assert!( + has_struct, + "Should have struct class for tuple parameter, found: {:?}", + class_names + ); + } + + /// Test that nested tuple types (struct with struct field) are handled. + #[test] + fn test_nested_tuple_types() { + let abi_json = r#"[ + { + "type": "function", + "name": "getNestedData", + "inputs": [], + "outputs": [ + { + "name": "result", + "type": "tuple", + "components": [ + {"name": "id", "type": "uint256"}, + { + "name": "inner", + "type": "tuple", + "components": [ + {"name": "x", "type": "uint256"}, + {"name": "y", "type": "uint256"} + ] + } + ] + } + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have main contract class + assert!( + class_names.contains(&"TestContract"), + "Should have TestContract class" + ); + + // Should have at least two struct classes (outer and inner) + let struct_count = class_names.iter().filter(|n| n.contains("Struct")).count(); + assert!( + struct_count >= 2, + "Should have at least 2 struct classes for nested tuple, found {} in {:?}", + struct_count, + class_names + ); + } + + /// Test that tuple arrays are handled correctly. + #[test] + fn test_tuple_array_types() { + let abi_json = r#"[ + { + "type": "function", + "name": "getAllItems", + "inputs": [], + "outputs": [ + { + "name": "items", + "type": "tuple[]", + "components": [ + {"name": "id", "type": "uint256"}, + {"name": "name", "type": "string"} + ] + } + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Find the contract class + let contract_class = types.iter().find(|c| c.name == "TestContract").unwrap(); + + // Find the getAllItems method + let method = contract_class + .methods + .iter() + .find(|m| m.name == "getAllItems") + .expect("Should have getAllItems method"); + + // The return type should be an Array of the struct type + let return_type = method + .return_type + .as_ref() + .expect("Should have return type"); + let return_type_str = return_type.to_string(); + assert!( + return_type_str.starts_with("Array<"), + "Return type should be Array<...>, got: {}", + return_type_str + ); + } + + /// Test that array types in events are handled correctly. + #[test] + fn test_array_types_in_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "Airdropped", + "inputs": [ + {"name": "sender", "type": "address", "indexed": true}, + {"name": "recipients", "type": "address[]", "indexed": false}, + {"name": "amounts", "type": "uint256[]", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Find the params class + let params_class = types + .iter() + .find(|c| c.name == "Airdropped__Params") + .expect("Should have Airdropped__Params class"); + + // Check that the array getters exist and have correct return types + let method_names: Vec<&str> = params_class + .methods + .iter() + .map(|m| m.name.as_str()) + .collect(); + + assert!( + method_names.iter().any(|n| n.contains("recipients")), + "Should have recipients getter, found: {:?}", + method_names + ); + assert!( + method_names.iter().any(|n| n.contains("amounts")), + "Should have amounts getter, found: {:?}", + method_names + ); + + // Verify the recipients getter returns Array
+ let recipients_getter = params_class + .methods + .iter() + .find(|m| m.name.contains("recipients")) + .expect("Should have recipients getter"); + let return_type_str = recipients_getter + .return_type + .as_ref() + .expect("Should have return type") + .to_string(); + assert!( + return_type_str.contains("Array
"), + "recipients should return Array
, got: {}", + return_type_str + ); + + // Verify the amounts getter returns Array + let amounts_getter = params_class + .methods + .iter() + .find(|m| m.name.contains("amounts")) + .expect("Should have amounts getter"); + let amounts_type_str = amounts_getter + .return_type + .as_ref() + .expect("Should have return type") + .to_string(); + assert!( + amounts_type_str.contains("Array"), + "amounts should return Array, got: {}", + amounts_type_str + ); + } + + /// Test that 2D array types (matrices) are handled correctly in functions. + #[test] + fn test_matrix_types_in_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "getMatrix", + "inputs": [], + "outputs": [ + {"name": "data", "type": "uint256[][]"} + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Find the contract class + let contract_class = types.iter().find(|c| c.name == "TestContract").unwrap(); + + // Find the getMatrix method + let method = contract_class + .methods + .iter() + .find(|m| m.name == "getMatrix") + .expect("Should have getMatrix method"); + + // The return type should be Array> + let return_type = method + .return_type + .as_ref() + .expect("Should have return type"); + let return_type_str = return_type.to_string(); + assert!( + return_type_str.contains("Array>"), + "Return type should be Array>, got: {}", + return_type_str + ); + } +} diff --git a/gnd/src/codegen/mod.rs b/gnd/src/codegen/mod.rs new file mode 100644 index 00000000000..4c8f5fdbd98 --- /dev/null +++ b/gnd/src/codegen/mod.rs @@ -0,0 +1,20 @@ +//! Code generation for subgraph AssemblyScript types. +//! +//! This module generates AssemblyScript types from: +//! - GraphQL schema (entity classes) +//! - Contract ABIs (event and call bindings) +//! - Data source templates + +mod abi; +mod schema; +mod template; +mod types; +mod typescript; + +pub use abi::AbiCodeGenerator; +pub use schema::SchemaCodeGenerator; +pub use template::{Template, TemplateCodeGenerator, TemplateKind}; +pub use typescript::{ + ArrayType, Class, ClassMember, GENERATED_FILE_NOTE, Method, ModuleImports, NamedType, + NullableType, Param, StaticMethod, +}; diff --git a/gnd/src/codegen/schema.rs b/gnd/src/codegen/schema.rs new file mode 100644 index 00000000000..a8a276f1ca0 --- /dev/null +++ b/gnd/src/codegen/schema.rs @@ -0,0 +1,1289 @@ +//! Schema code generation. +//! +//! Generates AssemblyScript entity classes from GraphQL schemas. + +use anyhow::{Result, anyhow}; +use graphql_tools::parser::schema::{ + Definition, Document, Field, ObjectType, Type, TypeDefinition, +}; + +use super::types::{asc_type_for_value, value_from_asc, value_to_asc}; +use super::typescript::{ + self as ts, ArrayType, Class, Method, ModuleImports, NamedType, NullableType, Param, + StaticMethod, TypeExpr, +}; +use crate::shared::handle_reserved_word; + +/// Type of the ID field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdFieldKind { + String, + Bytes, + Int8, +} + +impl IdFieldKind { + /// Get the AssemblyScript type name for this ID type. + pub fn type_name(&self) -> &'static str { + match self { + IdFieldKind::String => "string", + IdFieldKind::Bytes => "Bytes", + IdFieldKind::Int8 => "i64", + } + } + + /// Get the GraphQL type name for this ID type. + pub fn gql_type_name(&self) -> &'static str { + match self { + IdFieldKind::String => "String", + IdFieldKind::Bytes => "Bytes", + IdFieldKind::Int8 => "Int8", + } + } + + /// Get the code to create a Value from the ID. + pub fn value_from(&self) -> &'static str { + match self { + IdFieldKind::String => "Value.fromString(id)", + IdFieldKind::Bytes => "Value.fromBytes(id)", + IdFieldKind::Int8 => "Value.fromI64(id)", + } + } + + /// Get the ValueKind for this ID type. + pub fn value_kind(&self) -> &'static str { + match self { + IdFieldKind::String => "ValueKind.STRING", + IdFieldKind::Bytes => "ValueKind.BYTES", + IdFieldKind::Int8 => "ValueKind.INT8", + } + } + + /// Get the code to convert a Value to a string representation. + pub fn value_to_string(&self) -> &'static str { + match self { + IdFieldKind::String => "id.toString()", + IdFieldKind::Bytes => "id.toBytes().toHexString()", + IdFieldKind::Int8 => "id.toI64().toString()", + } + } + + /// Get the code to convert the ID to a string. + pub fn id_to_string_code(self) -> &'static str { + match self { + IdFieldKind::String => "id", + IdFieldKind::Bytes => "id.toHexString()", + IdFieldKind::Int8 => "id.toString()", + } + } + + /// Determine the ID field kind from a type name. + pub fn from_type_name(type_name: &str) -> Self { + match type_name { + "Bytes" => IdFieldKind::Bytes, + "Int8" => IdFieldKind::Int8, + _ => IdFieldKind::String, + } + } + + /// Returns true if this ID type supports auto-increment (optional constructor arg). + pub fn supports_auto(&self) -> bool { + matches!(self, IdFieldKind::Int8 | IdFieldKind::Bytes) + } + + /// Get the constructor parameter type (with nullability for auto-increment types). + pub fn constructor_param_type(&self) -> &'static str { + match self { + IdFieldKind::String => "string", + IdFieldKind::Bytes => "Bytes | null", + IdFieldKind::Int8 => "i64", // primitive, can't be nullable + } + } + + /// Get the default value for constructor parameter (for auto-increment types). + pub fn constructor_default(&self) -> Option<&'static str> { + match self { + IdFieldKind::String => None, + IdFieldKind::Bytes => Some("null"), + IdFieldKind::Int8 => Some("i64.MIN_VALUE"), + } + } + + /// Get the condition to check if id was provided (not auto-increment). + pub fn auto_check_condition(&self) -> &'static str { + match self { + IdFieldKind::String => "", // N/A + // For Bytes we end up generating 'if (id) { .. }' Using 'id != + // null' crashes the AssemblyScript compiler 0.19.23 because id + // has type 'Bytes | null' and the compiler doesn't strip the + // null from the type in the body of the if + IdFieldKind::Bytes => "id", + IdFieldKind::Int8 => "id != i64.MIN_VALUE", + } + } +} + +/// Get the base type name from a GraphQL type (stripping NonNull and List wrappers). +fn get_base_type_name(ty: &Type<'_, String>) -> String { + match ty { + Type::NamedType(name) => name.clone(), + Type::NonNullType(inner) => get_base_type_name(inner), + Type::ListType(inner) => get_base_type_name(inner), + } +} + +/// Check if a type is nullable (not wrapped in NonNull). +fn is_nullable(ty: &Type<'_, String>) -> bool { + !matches!(ty, Type::NonNullType(_)) +} + +/// Count the list nesting depth of a type. +/// `String` -> 0, `[String]` -> 1, `[[String]]` -> 2, etc. +fn list_depth(ty: &Type<'_, String>) -> u8 { + match ty { + Type::ListType(inner) => 1 + list_depth(inner), + Type::NonNullType(inner) => list_depth(inner), + Type::NamedType(_) => 0, + } +} + +/// Check if the innermost list members are nullable. +/// For `[String]` returns true, for `[String!]` returns false. +/// For non-list types, returns false. +fn is_list_member_nullable(ty: &Type<'_, String>) -> bool { + match ty { + Type::ListType(inner) => { + // Check the immediate inner type + is_nullable(inner) + } + Type::NonNullType(inner) => is_list_member_nullable(inner), + Type::NamedType(_) => false, + } +} + +/// Check if a field has the @derivedFrom directive. +fn is_derived_field(field: &Field<'_, String>) -> bool { + field.directives.iter().any(|d| d.name == "derivedFrom") +} + +/// Check if an object type has the @entity directive. +fn is_entity_type(obj: &ObjectType<'_, String>) -> bool { + obj.directives.iter().any(|d| d.name == "entity") +} + +/// Collected entity info for code generation. +struct EntityInfo { + name: String, + id_kind: IdFieldKind, + fields: Vec, +} + +/// Collected field info. +struct FieldInfo { + name: String, + is_derived: bool, + base_type: String, + is_nullable: bool, + /// The nesting depth of list wrappers. 0 = scalar, 1 = [T], 2 = [[T]], etc. + list_depth: u8, + /// Whether list members are nullable. Only meaningful when list_depth > 0. + member_nullable: bool, +} + +/// Schema code generator. +pub struct SchemaCodeGenerator { + entities: Vec, + entity_names: std::collections::HashSet, + /// Maps entity name to its ID field kind, for resolving entity reference types. + entity_id_kinds: std::collections::HashMap, +} + +impl SchemaCodeGenerator { + /// Create a new schema code generator from a parsed GraphQL document. + /// + /// Returns an error if the schema contains invalid patterns like non-nullable + /// lists with nullable members (e.g., `[Something]!`). + pub fn new(document: &Document<'_, String>) -> Result { + let mut entities = Vec::new(); + let mut entity_names = std::collections::HashSet::new(); + let mut entity_id_kinds = std::collections::HashMap::new(); + + // First pass: collect entity names and their ID types + for def in &document.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def + && is_entity_type(obj) + { + entity_names.insert(obj.name.clone()); + let id_field = obj.fields.iter().find(|f| f.name == "id"); + let id_kind = id_field + .map(|f| IdFieldKind::from_type_name(&get_base_type_name(&f.field_type))) + .unwrap_or(IdFieldKind::String); + entity_id_kinds.insert(obj.name.clone(), id_kind); + } + } + + // Second pass: collect entity info + for def in &document.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def + && is_entity_type(obj) + { + let name = obj.name.clone(); + + // Find ID field + let id_field = obj.fields.iter().find(|f| f.name == "id"); + let id_kind = id_field + .map(|f| IdFieldKind::from_type_name(&get_base_type_name(&f.field_type))) + .unwrap_or(IdFieldKind::String); + + // Collect field info + let fields: Vec<_> = obj + .fields + .iter() + .map(|f| FieldInfo { + name: f.name.clone(), + is_derived: is_derived_field(f), + base_type: get_base_type_name(&f.field_type), + is_nullable: is_nullable(&f.field_type), + list_depth: list_depth(&f.field_type), + member_nullable: is_list_member_nullable(&f.field_type), + }) + .collect(); + + entities.push(EntityInfo { + name, + id_kind, + fields, + }); + } + } + + // Validate: non-nullable lists must have non-nullable members + for entity in &entities { + for field in &entity.fields { + if field.list_depth > 0 && !field.is_nullable && field.member_nullable { + return Err(anyhow!( + "Codegen can't generate code for GraphQL field '{}' of type '[{}]!' since the inner type is nullable.\n\ + Suggestion: add an '!' to the inner type, e.g., '[{}!]!'", + field.name, + field.base_type, + field.base_type + )); + } + } + } + + Ok(Self { + entities, + entity_names, + entity_id_kinds, + }) + } + + /// Generate module imports for the schema file. + pub fn generate_module_imports(&self) -> Vec { + vec![ModuleImports::new( + vec![ + "TypedMap".to_string(), + "Entity".to_string(), + "Value".to_string(), + "ValueKind".to_string(), + "store".to_string(), + "Bytes".to_string(), + "BigInt".to_string(), + "BigDecimal".to_string(), + "Int8".to_string(), + ], + "@graphprotocol/graph-ts", + )] + } + + /// Generate entity classes from the schema. + pub fn generate_types(&self, generate_store_methods: bool) -> Vec { + self.entities + .iter() + .map(|entity| self.generate_entity_type(entity, generate_store_methods)) + .collect() + } + + /// Generate derived loaders for fields with @derivedFrom. + pub fn generate_derived_loaders(&self) -> Vec { + let mut loaders = Vec::new(); + let mut seen_types = std::collections::HashSet::new(); + + for entity in &self.entities { + for field in &entity.fields { + if field.is_derived && !seen_types.contains(&field.base_type) { + // Only generate loaders for entity types, not interfaces + if self.entity_names.contains(&field.base_type) { + seen_types.insert(field.base_type.clone()); + loaders.push(self.generate_derived_loader(&field.base_type)); + } + } + } + } + + loaders + } + + fn generate_entity_type(&self, entity: &EntityInfo, generate_store_methods: bool) -> Class { + let mut klass = ts::klass(&entity.name).exported().extends("Entity"); + + // Generate constructor + klass.add_method(self.generate_constructor(&entity.id_kind)); + + // Generate store methods + if generate_store_methods { + for method in self.generate_store_methods(&entity.name, &entity.id_kind) { + match method { + StoreMethod::Regular(m) => klass.add_method(m), + StoreMethod::Static(m) => klass.add_static_method(m), + } + } + } + + // Generate field getters and setters + for field in &entity.fields { + if let Some(getter) = self.generate_field_getter(&entity.name, field, &entity.id_kind) { + klass.add_method(getter); + } + if let Some(setter) = self.generate_field_setter(field) { + klass.add_method(setter); + } + } + + klass + } + + fn generate_constructor(&self, id_kind: &IdFieldKind) -> Method { + if id_kind.supports_auto() { + // For Int8 and Bytes, make id optional to support auto-increment + let param_type = id_kind.constructor_param_type(); + let default_value = id_kind.constructor_default().unwrap(); + let check_condition = id_kind.auto_check_condition(); + + Method::new( + "constructor", + vec![Param::with_default( + "id", + TypeExpr::Raw(param_type.to_string()), + default_value, + )], + None, + format!( + r#" + super() + if ({}) {{ + this.set('id', {}) + }}"#, + check_condition, + id_kind.value_from() + ), + ) + .with_doc("Leaving out the id argument uses an autoincrementing id.") + } else { + // For String IDs, keep the existing behavior (required parameter) + Method::new( + "constructor", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + None, + format!( + r#" + super() + this.set('id', {})"#, + id_kind.value_from() + ), + ) + } + } + + fn generate_store_methods(&self, entity_name: &str, id_kind: &IdFieldKind) -> Vec { + // Generate save() method - different for auto-increment vs string IDs + let save_method = if id_kind.supports_auto() { + // For Int8 and Bytes, check if id is null/unset and use "auto" as the key + Method::new( + "save", + vec![], + Some(NamedType::new("void").into()), + format!( + r#" + let id = this.get('id') + if (id == null || id.kind == ValueKind.NULL) {{ + store.set('{}', 'auto', this) + }} else {{ + assert(id.kind == {}, + `Entities of type {} must have an ID of type {} but the id '${{id.displayData()}}' is of type ${{id.displayKind()}}`) + store.set('{}', {}, this) + }}"#, + entity_name, + id_kind.value_kind(), + entity_name, + id_kind.gql_type_name(), + entity_name, + id_kind.value_to_string() + ), + ) + } else { + // For String IDs, keep the existing behavior (require ID) + Method::new( + "save", + vec![], + Some(NamedType::new("void").into()), + format!( + r#" + let id = this.get('id') + assert(id != null, + 'Cannot save {} entity without an ID') + if (id) {{ + assert(id.kind == {}, + `Entities of type {} must have an ID of type {} but the id '${{id.displayData()}}' is of type ${{id.displayKind()}}`) + store.set('{}', {}, this) + }}"#, + entity_name, + id_kind.value_kind(), + entity_name, + id_kind.gql_type_name(), + entity_name, + id_kind.value_to_string() + ), + ) + }; + + vec![ + StoreMethod::Regular(save_method), + // loadInBlock() static method + StoreMethod::Static(StaticMethod::new( + "loadInBlock", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + NullableType::new(NamedType::new(entity_name)), + format!( + r#" + return changetype<{} | null>(store.get_in_block('{}', {}))"#, + entity_name, + entity_name, + id_kind.id_to_string_code() + ), + )), + // load() static method + StoreMethod::Static(StaticMethod::new( + "load", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + NullableType::new(NamedType::new(entity_name)), + format!( + r#" + return changetype<{} | null>(store.get('{}', {}))"#, + entity_name, + entity_name, + id_kind.id_to_string_code() + ), + )), + ] + } + + fn generate_field_getter( + &self, + entity_name: &str, + field: &FieldInfo, + id_kind: &IdFieldKind, + ) -> Option { + let safe_name = handle_reserved_word(&field.name); + + // Handle derived fields + if field.is_derived { + return self.generate_derived_field_getter(entity_name, field, &safe_name, id_kind); + } + + let value_type = self.value_type_from_field(field); + let return_type = self.type_from_field(field); + let nullable = field.is_nullable; + + let primitive_default = match &return_type { + TypeExpr::Named(t) => t.get_primitive_default(), + _ => None, + }; + + let get_code = if nullable { + format!( + r#" + let value = this.get('{}') + if (!value || value.kind == ValueKind.NULL) {{ + return null + }} else {{ + return {} + }}"#, + field.name, + value_to_asc("value", &value_type) + ) + } else { + let null_handling = match primitive_default { + Some(default) => format!("return {}", default), + None => "throw new Error('Cannot return null for a required field.')".to_string(), + }; + format!( + r#" + let value = this.get('{}') + if (!value || value.kind == ValueKind.NULL) {{ + {} + }} else {{ + return {} + }}"#, + field.name, + null_handling, + value_to_asc("value", &value_type) + ) + }; + + Some(Method::new( + format!("get {}", safe_name), + vec![], + Some(return_type), + get_code, + )) + } + + fn generate_derived_field_getter( + &self, + entity_name: &str, + field: &FieldInfo, + safe_name: &str, + id_kind: &IdFieldKind, + ) -> Option { + let loader_name = format!("{}Loader", field.base_type); + + let id_conversion = match id_kind { + IdFieldKind::Bytes => "this.get('id')!.toBytes().toHexString()", + _ => "this.get('id')!.toString()", + }; + + Some(Method::new( + format!("get {}", safe_name), + vec![], + Some(NamedType::new(&loader_name).into()), + format!( + r#" + return new {}('{}', {}, '{}')"#, + loader_name, entity_name, id_conversion, field.name + ), + )) + } + + fn generate_field_setter(&self, field: &FieldInfo) -> Option { + // No setters for derived fields + if field.is_derived { + return None; + } + + let safe_name = handle_reserved_word(&field.name); + let value_type = self.value_type_from_field(field); + let param_type = self.type_from_field(field); + let nullable = field.is_nullable; + + let set_code = if nullable { + let inner_type = match ¶m_type { + TypeExpr::Nullable(n) => n.inner.to_string(), + other => other.to_string(), + }; + format!( + r#" + if (!value) {{ + this.unset('{}') + }} else {{ + this.set('{}', {}) + }}"#, + field.name, + field.name, + value_from_asc(&format!("<{}>value", inner_type), &value_type) + ) + } else { + format!( + r#" + this.set('{}', {})"#, + field.name, + value_from_asc("value", &value_type) + ) + }; + + Some(Method::new( + format!("set {}", safe_name), + vec![Param::new("value", param_type)], + None, + set_code, + )) + } + + fn generate_derived_loader(&self, type_name: &str) -> Class { + let loader_name = format!("{}Loader", type_name); + let mut klass = ts::klass(&loader_name).exported().extends("Entity"); + + // Add members + klass.add_member(ts::klass_member("_entity", "string")); + klass.add_member(ts::klass_member("_field", "string")); + klass.add_member(ts::klass_member("_id", "string")); + + // Add constructor + klass.add_method(Method::new( + "constructor", + vec![ + Param::new("entity", NamedType::new("string")), + Param::new("id", NamedType::new("string")), + Param::new("field", NamedType::new("string")), + ], + None, + r#" + super(); + this._entity = entity; + this._id = id; + this._field = field;"# + .to_string(), + )); + + // Add load() method + klass.add_method(Method::new( + "load", + vec![], + Some(TypeExpr::Raw(format!("{}[]", type_name))), + format!( + r#" + let value = store.loadRelated(this._entity, this._id, this._field); + return changetype<{}[]>(value);"#, + type_name + ), + )); + + klass + } + + /// Get the value type string for a field. + /// + /// Returns the GraphQL-style value type string: + /// - Scalars: `String`, `Int`, `BigInt`, etc. + /// - Arrays: `[String]`, `[Int]`, etc. + /// - Nested arrays: `[[String]]`, `[[Int]]`, etc. + /// - Entity references are converted to the referenced entity's ID type + fn value_type_from_field(&self, field: &FieldInfo) -> String { + let base = if let Some(id_kind) = self.entity_id_kinds.get(&field.base_type) { + id_kind.gql_type_name().to_string() + } else { + field.base_type.clone() + }; + + // Wrap with brackets for each level of list nesting + let mut result = base; + for _ in 0..field.list_depth { + result = format!("[{}]", result); + } + result + } + + /// Convert field info to an AssemblyScript TypeExpr. + /// + /// Creates the correct type expression including nested arrays: + /// - Scalars: `string`, `i32`, `BigInt`, etc. + /// - Arrays: `Array`, `Array`, etc. + /// - Nested arrays: `Array>`, etc. + fn type_from_field(&self, field: &FieldInfo) -> TypeExpr { + let type_name = if let Some(id_kind) = self.entity_id_kinds.get(&field.base_type) { + id_kind.type_name() + } else { + asc_type_for_value(&field.base_type) + }; + + let named = NamedType::new(type_name); + + if field.list_depth > 0 { + // Use ArrayType::with_depth to create nested array types + let array_type = ArrayType::with_depth(named, field.list_depth); + if field.is_nullable { + NullableType::new(array_type).into() + } else { + array_type + } + } else if field.is_nullable && !named.is_primitive() { + NullableType::new(named).into() + } else { + named.into() + } + } +} + +enum StoreMethod { + Regular(Method), + Static(StaticMethod), +} + +#[cfg(test)] +mod tests { + use super::*; + + use graphql_tools::parser::parse_schema; + + #[test] + fn test_simple_entity() { + let schema = r#" + type Transfer @entity { + id: ID! + from: Bytes! + to: Bytes! + value: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let transfer = &classes[0]; + assert_eq!(transfer.name, "Transfer"); + assert_eq!(transfer.extends, Some("Entity".to_string())); + assert!(transfer.export); + } + + #[test] + fn test_nullable_field() { + let schema = r#" + type Token @entity { + id: ID! + name: String + symbol: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + // Check that we have methods for nullable and non-nullable fields + let token = &classes[0]; + let method_names: Vec<_> = token.methods.iter().map(|m| m.name.as_str()).collect(); + assert!(method_names.contains(&"get name")); + assert!(method_names.contains(&"set name")); + assert!(method_names.contains(&"get symbol")); + assert!(method_names.contains(&"set symbol")); + } + + #[test] + fn test_id_field_types() { + assert_eq!(IdFieldKind::String.type_name(), "string"); + assert_eq!(IdFieldKind::Bytes.type_name(), "Bytes"); + assert_eq!(IdFieldKind::Int8.type_name(), "i64"); + } + + #[test] + fn test_auto_increment_support() { + // String IDs don't support auto-increment + assert!(!IdFieldKind::String.supports_auto()); + assert!(IdFieldKind::String.constructor_default().is_none()); + + // Int8 IDs support auto-increment + assert!(IdFieldKind::Int8.supports_auto()); + assert_eq!(IdFieldKind::Int8.constructor_param_type(), "i64"); + assert_eq!( + IdFieldKind::Int8.constructor_default(), + Some("i64.MIN_VALUE") + ); + assert_eq!( + IdFieldKind::Int8.auto_check_condition(), + "id != i64.MIN_VALUE" + ); + + // Bytes IDs support auto-increment + assert!(IdFieldKind::Bytes.supports_auto()); + assert_eq!(IdFieldKind::Bytes.constructor_param_type(), "Bytes | null"); + assert_eq!(IdFieldKind::Bytes.constructor_default(), Some("null")); + assert_eq!(IdFieldKind::Bytes.auto_check_condition(), "id"); + } + + #[test] + fn test_int8_id_auto_increment_codegen() { + let schema = r#" + type Counter @entity { + id: Int8! + value: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let counter = &classes[0]; + let output = counter.to_string(); + + // Constructor should have optional id with sentinel default + assert!( + output.contains("constructor(id: i64 = i64.MIN_VALUE)"), + "Int8 ID constructor should have default sentinel value, got: {}", + output + ); + + // Constructor should have doc comment + assert!( + output.contains("Leaving out the id argument uses an autoincrementing id"), + "Constructor should have auto-increment doc comment" + ); + + // Constructor body should conditionally set id + assert!( + output.contains("if (id != i64.MIN_VALUE)"), + "Constructor should check for sentinel value" + ); + + // save() method should use "auto" when id is null + assert!( + output.contains("store.set('Counter', 'auto', this)"), + "save() should use 'auto' key for auto-increment, got: {}", + output + ); + } + + #[test] + fn test_bytes_id_auto_increment_codegen() { + let schema = r#" + type Event @entity { + id: Bytes! + data: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let event = &classes[0]; + let output = event.to_string(); + + // Constructor should have nullable id with null default + assert!( + output.contains("constructor(id: Bytes | null = null)"), + "Bytes ID constructor should have null default, got: {}", + output + ); + + // Constructor should have doc comment + assert!( + output.contains("Leaving out the id argument uses an autoincrementing id"), + "Constructor should have auto-increment doc comment" + ); + + // Constructor body should conditionally set id + assert!( + output.contains("if (id)"), + "Constructor should check for null" + ); + + // save() method should use "auto" when id is null + assert!( + output.contains("store.set('Event', 'auto', this)"), + "save() should use 'auto' key for auto-increment, got: {}", + output + ); + } + + #[test] + fn test_string_id_no_auto_increment() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let user = &classes[0]; + let output = user.to_string(); + + // Constructor should have required id (no default) + assert!( + output.contains("constructor(id: string)"), + "String ID constructor should have required parameter, got: {}", + output + ); + + // Constructor should NOT have auto-increment doc comment + assert!( + !output.contains("autoincrementing"), + "String ID should not mention auto-increment" + ); + + // save() method should NOT use "auto" + assert!( + !output.contains("'auto'"), + "String ID save() should not use 'auto' key" + ); + + // save() should require ID + assert!( + output.contains("Cannot save User entity without an ID"), + "String ID save() should require ID" + ); + } + + #[test] + fn test_entity_reference() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + type Post @entity { + id: ID! + author: User! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + // The Post.author field should be treated as a string (entity ID reference) + assert!(generator.entity_names.contains("User")); + assert!(generator.entity_names.contains("Post")); + } + + #[test] + fn test_simple_array_field() { + let schema = r#" + type Token @entity { + id: ID! + holders: [String!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let token = &classes[0]; + let output = token.to_string(); + + // Verify array field getter/setter are generated + assert!( + output.contains("get holders()"), + "Should have holders getter" + ); + assert!( + output.contains("set holders("), + "Should have holders setter" + ); + + // Check the type is Array + assert!( + output.contains("Array"), + "Array field should use Array type" + ); + } + + #[test] + fn test_nested_array_field() { + let schema = r#" + type Matrix @entity { + id: ID! + stringMatrix: [[String!]!]! + intMatrix: [[Int!]!] + bigIntMatrix: [[BigInt!]!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let matrix = &classes[0]; + let output = matrix.to_string(); + + // Verify nested array field getter/setter are generated + assert!( + output.contains("get stringMatrix()"), + "Should have stringMatrix getter" + ); + assert!( + output.contains("set stringMatrix("), + "Should have stringMatrix setter" + ); + + // Check the type is Array> + assert!( + output.contains("Array>"), + "Nested array field should use Array> type, got: {}", + output + ); + + // Check that toStringMatrix() and fromStringMatrix() are used + assert!( + output.contains("toStringMatrix()"), + "Should use toStringMatrix() for nested string arrays" + ); + assert!( + output.contains("fromStringMatrix("), + "Should use Value.fromStringMatrix() for nested string arrays" + ); + + // Check BigInt matrix uses correct methods + assert!( + output.contains("Array>"), + "BigInt matrix should use Array> type" + ); + assert!( + output.contains("toBigIntMatrix()"), + "Should use toBigIntMatrix() for nested BigInt arrays" + ); + assert!( + output.contains("fromBigIntMatrix("), + "Should use Value.fromBigIntMatrix() for nested BigInt arrays" + ); + + // Check nullable nested array has correct type + assert!( + output.contains("Array> | null"), + "Nullable nested array should be Array> | null" + ); + } + + #[test] + fn test_list_depth() { + use graphql_tools::parser::parse_schema; + + // Helper to get list depth from schema field type + fn get_field_list_depth(schema_str: &str) -> u8 { + let doc = parse_schema::(schema_str).unwrap(); + for def in &doc.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def { + for field in &obj.fields { + if field.name == "field" { + return list_depth(&field.field_type); + } + } + } + } + panic!("Field not found"); + } + + // Scalar + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: String! }"), + 0 + ); + + // Simple array + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [String!]! }"), + 1 + ); + + // Nested array (matrix) + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [[String!]!]! }"), + 2 + ); + + // Triple nested array + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [[[String!]!]!]! }"), + 3 + ); + } + + #[test] + fn test_value_type_from_field_nested() { + let schema = r#" + type Test @entity { + id: ID! + scalar: String! + array: [String!]! + matrix: [[String!]!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + // Find the entity + let entity = &generator.entities[0]; + + // Find each field and check its value type + let scalar_field = entity.fields.iter().find(|f| f.name == "scalar").unwrap(); + let array_field = entity.fields.iter().find(|f| f.name == "array").unwrap(); + let matrix_field = entity.fields.iter().find(|f| f.name == "matrix").unwrap(); + + assert_eq!(generator.value_type_from_field(scalar_field), "String"); + assert_eq!(generator.value_type_from_field(array_field), "[String]"); + assert_eq!(generator.value_type_from_field(matrix_field), "[[String]]"); + } + + #[test] + fn test_bytes_id_entity_reference() { + let schema = r#" + type Token @entity { + id: Bytes! + name: String! + } + type Balance @entity { + id: ID! + token: Token! + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let balance = classes.iter().find(|c| c.name == "Balance").unwrap(); + let output = balance.to_string(); + + // Getter should return Bytes and use toBytes() + assert!( + output.contains("value.toBytes()"), + "Bytes-ID entity reference getter should use toBytes(), got: {}", + output + ); + + // Setter should use Value.fromBytes() + assert!( + output.contains("Value.fromBytes("), + "Bytes-ID entity reference setter should use Value.fromBytes(), got: {}", + output + ); + + // Return type should be Bytes, not string + assert!( + output.contains("get token(): Bytes"), + "Bytes-ID entity reference getter should return Bytes, got: {}", + output + ); + } + + #[test] + fn test_int8_id_entity_reference() { + let schema = r#" + type Counter @entity { + id: Int8! + value: BigInt! + } + type Snapshot @entity { + id: ID! + counter: Counter! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let snapshot = classes.iter().find(|c| c.name == "Snapshot").unwrap(); + let output = snapshot.to_string(); + + // Getter should return i64 and use toI64() + assert!( + output.contains("value.toI64()"), + "Int8-ID entity reference getter should use toI64(), got: {}", + output + ); + + // Setter should use Value.fromI64() + assert!( + output.contains("Value.fromI64("), + "Int8-ID entity reference setter should use Value.fromI64(), got: {}", + output + ); + + // Return type should be i64 + assert!( + output.contains("get counter(): i64"), + "Int8-ID entity reference getter should return i64, got: {}", + output + ); + } + + #[test] + fn test_mixed_id_entity_references() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + type Token @entity { + id: Bytes! + owner: User! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let token = classes.iter().find(|c| c.name == "Token").unwrap(); + let output = token.to_string(); + + // Token.owner references User which has String ID + assert!( + output.contains("get owner(): string"), + "Reference to String-ID entity should use string type, got: {}", + output + ); + assert!( + output.contains("value.toString()"), + "Reference to String-ID entity should use toString(), got: {}", + output + ); + } + + #[test] + fn test_nullable_bytes_id_entity_reference() { + let schema = r#" + type Token @entity { + id: Bytes! + name: String! + } + type Balance @entity { + id: ID! + token: Token + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let balance = classes.iter().find(|c| c.name == "Balance").unwrap(); + let output = balance.to_string(); + + // Nullable Bytes reference should be `Bytes | null` + assert!( + output.contains("get token(): Bytes | null"), + "Nullable Bytes-ID reference should return Bytes | null, got: {}", + output + ); + } + + #[test] + fn test_derived_field_with_bytes_id_parent() { + let schema = r#" + type Token @entity { + id: Bytes! + balances: [Balance!]! @derivedFrom(field: "token") + } + type Balance @entity { + id: ID! + token: Token! + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let token = classes.iter().find(|c| c.name == "Token").unwrap(); + let output = token.to_string(); + + // Derived field getter on Bytes-ID entity should use toBytes().toHexString() + assert!( + output.contains("this.get('id')!.toBytes().toHexString()"), + "Derived field on Bytes-ID entity should use toBytes().toHexString(), got: {}", + output + ); + } +} diff --git a/gnd/src/codegen/template.rs b/gnd/src/codegen/template.rs new file mode 100644 index 00000000000..72a27f9e733 --- /dev/null +++ b/gnd/src/codegen/template.rs @@ -0,0 +1,286 @@ +//! Data source template code generation. +//! +//! Generates AssemblyScript classes for subgraph templates that allow +//! dynamic data source creation at runtime. + +use super::typescript::{self as ts, Class, ModuleImports, Param, StaticMethod}; + +/// The kind of a data source template. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemplateKind { + /// Ethereum contract template + Ethereum, + /// IPFS file template + FileIpfs, + /// Arweave file template + FileArweave, +} + +impl TemplateKind { + /// Parse a template kind from a string (e.g., "ethereum/contract", "file/ipfs"). + pub fn from_str_kind(kind: &str) -> Option { + match kind { + "ethereum/contract" | "ethereum" => Some(TemplateKind::Ethereum), + "file/ipfs" => Some(TemplateKind::FileIpfs), + "file/arweave" => Some(TemplateKind::FileArweave), + _ => None, + } + } +} + +/// A data source template from the subgraph manifest. +pub struct Template { + /// The name of the template. + pub name: String, + /// The kind of template. + pub kind: TemplateKind, +} + +impl Template { + /// Create a new template. + pub fn new(name: impl Into, kind: TemplateKind) -> Self { + Self { + name: name.into(), + kind, + } + } +} + +const GRAPH_TS_MODULE: &str = "@graphprotocol/graph-ts"; + +/// Template code generator. +pub struct TemplateCodeGenerator { + templates: Vec