From 5447cf4843d3e9f2cbb7e8a7dc2b440568490a2f Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:55:23 +0100 Subject: [PATCH 001/579] feat(webapp): admin UI for feature flag overrides (#3291) Adds a dialog to the admin orgs page for viewing and editing per-org feature flag overrides. Flags are introspected from the catalog so the UI stays in sync with available flags automatically. Also adds a new tab for global flags. Refactors featureFlags.server.ts to split catalog definition (shared) from server-only runtime (flags(), makeSetMultipleFlags). The shared module exports flag metadata and validation so both the UI and API routes can use it without pulling in server dependencies. --- .server-changes/admin-feature-flags-dialog.md | 6 + apps/webapp/CLAUDE.md | 9 + .../components/admin/FeatureFlagsDialog.tsx | 290 +++++++++ .../app/components/admin/FlagControls.tsx | 120 ++++ .../OrganizationsPresenter.server.ts | 3 +- .../presenters/v3/RegionsPresenter.server.ts | 3 +- .../route.tsx | 2 +- .../app/routes/admin.api.v1.feature-flags.ts | 5 +- ...i.v1.orgs.$organizationId.feature-flags.ts | 8 +- ...i.v2.orgs.$organizationId.feature-flags.ts | 132 ++++ .../webapp/app/routes/admin.feature-flags.tsx | 565 ++++++++++++++++++ apps/webapp/app/routes/admin.orgs.tsx | 48 +- apps/webapp/app/routes/admin.tsx | 4 + .../route.tsx | 2 +- .../runsRepository/runsRepository.server.ts | 3 +- apps/webapp/app/v3/canAccessAi.server.ts | 3 +- .../webapp/app/v3/canAccessAiModels.server.ts | 3 +- .../v3/canAccessPrivateConnections.server.ts | 3 +- apps/webapp/app/v3/canAccessQuery.server.ts | 3 +- .../app/v3/eventRepository/index.server.ts | 3 +- apps/webapp/app/v3/featureFlags.server.ts | 57 +- apps/webapp/app/v3/featureFlags.ts | 93 +++ .../computeTemplateCreation.server.ts | 3 +- .../worker/workerGroupService.server.ts | 3 +- apps/webapp/package.json | 1 + pnpm-lock.yaml | 15 +- 26 files changed, 1301 insertions(+), 86 deletions(-) create mode 100644 .server-changes/admin-feature-flags-dialog.md create mode 100644 apps/webapp/app/components/admin/FeatureFlagsDialog.tsx create mode 100644 apps/webapp/app/components/admin/FlagControls.tsx create mode 100644 apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts create mode 100644 apps/webapp/app/routes/admin.feature-flags.tsx create mode 100644 apps/webapp/app/v3/featureFlags.ts diff --git a/.server-changes/admin-feature-flags-dialog.md b/.server-changes/admin-feature-flags-dialog.md new file mode 100644 index 00000000000..2517e21a3b8 --- /dev/null +++ b/.server-changes/admin-feature-flags-dialog.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Add admin UI for viewing and editing feature flags (org-level overrides and global defaults). diff --git a/apps/webapp/CLAUDE.md b/apps/webapp/CLAUDE.md index 2acd5c02d77..00ed6aaf78b 100644 --- a/apps/webapp/CLAUDE.md +++ b/apps/webapp/CLAUDE.md @@ -98,3 +98,12 @@ The `app/v3/` directory name is misleading - most code is actively used by V2. O - `app/v3/sharedSocketConnection.ts` Some services (e.g., `cancelTaskRun.server.ts`, `batchTriggerV3.server.ts`) branch on `RunEngineVersion` to support both V1 and V2. When editing these, only modify V2 code paths. + +## Prisma Query Patterns + +- **Always use `findFirst` instead of `findUnique`.** Prisma's `findUnique` has an implicit DataLoader that batches concurrent calls into a single `IN` query. This batching cannot be disabled and has active bugs even in Prisma 6.x: uppercase UUIDs returning null (#25484, confirmed 6.4.1), composite key SQL correctness issues (#22202), and 5-10x worse performance than manual DataLoader (#6573, open since 2021). `findFirst` is never batched and avoids this entire class of issues. + +## React Patterns + +- Only use `useCallback`/`useMemo` for context provider values, expensive derived data that is a dependency elsewhere, or stable refs required by a dependency array. Don't wrap ordinary event handlers or trivial computations. +- Use named constants for sentinel/placeholder values (e.g. `const UNSET_VALUE = "__unset__"`) instead of raw string literals scattered across comparisons. diff --git a/apps/webapp/app/components/admin/FeatureFlagsDialog.tsx b/apps/webapp/app/components/admin/FeatureFlagsDialog.tsx new file mode 100644 index 00000000000..df8669d36dd --- /dev/null +++ b/apps/webapp/app/components/admin/FeatureFlagsDialog.tsx @@ -0,0 +1,290 @@ +import { useFetcher } from "@remix-run/react"; +import { useEffect, useState } from "react"; +import stableStringify from "json-stable-stringify"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogDescription, + DialogFooter, +} from "~/components/primitives/Dialog"; +import { Button } from "~/components/primitives/Buttons"; +import { Callout } from "~/components/primitives/Callout"; +import { LockClosedIcon } from "@heroicons/react/20/solid"; +import { CheckboxWithLabel } from "~/components/primitives/Checkbox"; +import { cn } from "~/utils/cn"; +import { FEATURE_FLAG, ORG_LOCKED_FLAGS, type FlagControlType } from "~/v3/featureFlags"; +import { + UNSET_VALUE, + BooleanControl, + EnumControl, + StringControl, + WorkerGroupControl, + type WorkerGroup, +} from "./FlagControls"; + +type LoaderData = { + org: { id: string; title: string; slug: string }; + orgFlags: Record; + globalFlags: Record; + controlTypes: Record; + workerGroupName?: string; + workerGroups?: WorkerGroup[]; + isManagedCloud?: boolean; +}; + +type ActionData = { + success?: boolean; + error?: string; +}; + +type FeatureFlagsDialogProps = { + orgId: string | null; + orgTitle: string; + open: boolean; + onOpenChange: (open: boolean) => void; +}; + +export function FeatureFlagsDialog({ + orgId, + orgTitle, + open, + onOpenChange, +}: FeatureFlagsDialogProps) { + const loadFetcher = useFetcher(); + const saveFetcher = useFetcher(); + + const [overrides, setOverrides] = useState>({}); + const [initialOverrides, setInitialOverrides] = useState>({}); + const [saveError, setSaveError] = useState(null); + const [unlocked, setUnlocked] = useState(false); + + const isLocked = (key: string) => !unlocked && ORG_LOCKED_FLAGS.includes(key); + + useEffect(() => { + if (open && orgId) { + setSaveError(null); + setOverrides({}); + setInitialOverrides({}); + loadFetcher.load(`/admin/api/v2/orgs/${orgId}/feature-flags`); + } + }, [open, orgId]); + + useEffect(() => { + if (loadFetcher.data) { + const loaded = loadFetcher.data.orgFlags ?? {}; + setOverrides({ ...loaded }); + setInitialOverrides({ ...loaded }); + } + }, [loadFetcher.data]); + + useEffect(() => { + if (saveFetcher.data?.success) { + onOpenChange(false); + } else if (saveFetcher.data?.error) { + setSaveError(saveFetcher.data.error); + } + }, [saveFetcher.data]); + + const isDirty = stableStringify(overrides) !== stableStringify(initialOverrides); + + const setFlagValue = (key: string, value: unknown) => { + setOverrides((prev) => ({ ...prev, [key]: value })); + }; + + const unsetFlag = (key: string) => { + setOverrides((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + }; + + const handleSave = () => { + if (!orgId) return; + const body = Object.keys(overrides).length === 0 ? null : overrides; + saveFetcher.submit(JSON.stringify(body), { + method: "POST", + action: `/admin/api/v2/orgs/${orgId}/feature-flags`, + encType: "application/json", + }); + }; + + const data = loadFetcher.data; + const isLoading = loadFetcher.state === "loading"; + const isSaving = saveFetcher.state === "submitting"; + + const jsonPreview = + Object.keys(overrides).length === 0 ? "null" : JSON.stringify(overrides, null, 2); + + const sortedFlagKeys = data ? Object.keys(data.controlTypes).sort() : []; + + return ( + + + Feature flags - {orgTitle} + + Org-level overrides. Unset flags inherit from global defaults. + + + {data && ( +
+ +
+ )} + +
+ {isLoading ? ( +
Loading flags...
+ ) : data ? ( +
+ {sortedFlagKeys.map((key) => { + const control = data.controlTypes[key]; + const locked = isLocked(key); + const globalValue = data.globalFlags[key as keyof typeof data.globalFlags]; + const isWorkerGroup = key === FEATURE_FLAG.defaultWorkerInstanceGroupId; + const globalDisplay = + isWorkerGroup && data.workerGroupName && globalValue !== undefined + ? `${data.workerGroupName} (${String(globalValue).slice(0, 8)}...)` + : globalValue !== undefined + ? String(globalValue) + : "unset"; + + if (locked) { + return ( +
+
+
{key}
+
global: {globalDisplay}
+
+ +
+ ); + } + + const isOverridden = key in overrides; + + return ( +
+
+
+ {key} +
+
global: {globalDisplay}
+
+ +
+ + + {isWorkerGroup && data.workerGroups ? ( + { + if (val === UNSET_VALUE) { + unsetFlag(key); + } else { + setFlagValue(key, val); + } + }} + dimmed={!isOverridden} + /> + ) : control.type === "boolean" ? ( + setFlagValue(key, val)} + dimmed={!isOverridden} + /> + ) : control.type === "enum" ? ( + { + if (val === UNSET_VALUE) { + unsetFlag(key); + } else { + setFlagValue(key, val); + } + }} + dimmed={!isOverridden} + /> + ) : control.type === "string" ? ( + { + if (val === "") { + unsetFlag(key); + } else { + setFlagValue(key, val); + } + }} + dimmed={!isOverridden} + /> + ) : null} +
+
+ ); + })} +
+ ) : null} +
+ + {data && ( +
+ + Preview JSON + + 
+              {jsonPreview}
+
+
+ )} + + {saveError && {saveError}} + + + + + + + + ); +} diff --git a/apps/webapp/app/components/admin/FlagControls.tsx b/apps/webapp/app/components/admin/FlagControls.tsx new file mode 100644 index 00000000000..b08f925dd90 --- /dev/null +++ b/apps/webapp/app/components/admin/FlagControls.tsx @@ -0,0 +1,120 @@ +import { Switch } from "~/components/primitives/Switch"; +import { Select, SelectItem } from "~/components/primitives/Select"; +import { Input } from "~/components/primitives/Input"; +import { cn } from "~/utils/cn"; + +export const UNSET_VALUE = "__unset__"; + +export function BooleanControl({ + value, + onChange, + dimmed, +}: { + value: boolean | undefined; + onChange: (val: boolean) => void; + dimmed: boolean; +}) { + return ( + + ); +} + +export function EnumControl({ + value, + options, + onChange, + dimmed, +}: { + value: string | undefined; + options: string[]; + onChange: (val: string) => void; + dimmed: boolean; +}) { + const items = [UNSET_VALUE, ...options]; + + return ( + + ); +} + +export type WorkerGroup = { id: string; name: string }; + +export function WorkerGroupControl({ + value, + workerGroups, + onChange, + dimmed, +}: { + value: string | undefined; + workerGroups: WorkerGroup[]; + onChange: (val: string) => void; + dimmed: boolean; +}) { + const items = [UNSET_VALUE, ...workerGroups.map((wg) => wg.id)]; + + return ( + + ); +} + +export function StringControl({ + value, + onChange, + dimmed, +}: { + value: string; + onChange: (val: string) => void; + dimmed: boolean; +}) { + return ( + onChange(e.target.value)} + placeholder="unset" + className={cn("w-40", dimmed && "opacity-50")} + /> + ); +} diff --git a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts index d19a2878653..591dba5753c 100644 --- a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts +++ b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts @@ -11,7 +11,8 @@ import { import { sortEnvironments } from "~/utils/environmentSort"; import { defaultAvatar, parseAvatar } from "~/components/primitives/Avatar"; import { env } from "~/env.server"; -import { flags, validatePartialFeatureFlags } from "~/v3/featureFlags.server"; +import { flags } from "~/v3/featureFlags.server"; +import { validatePartialFeatureFlags } from "~/v3/featureFlags"; export class OrganizationsPresenter { #prismaClient: PrismaClient; diff --git a/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts b/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts index 7a35fb6fb9b..f72b8d2fc53 100644 --- a/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts @@ -1,6 +1,7 @@ import { type Project } from "~/models/project.server"; import { type User } from "~/models/user.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; import { BasePresenter } from "./basePresenter.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index 0c7f75fe262..80a5c6ef232 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -37,7 +37,7 @@ import { ResizablePanelGroup, } from "~/components/primitives/Resizable"; import { Button } from "~/components/primitives/Buttons"; -import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags"; // Valid log levels for filtering const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts index d0e1dd6b274..b10c983b14d 100644 --- a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts @@ -1,7 +1,8 @@ import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; -import { makeSetMultipleFlags, validatePartialFeatureFlags } from "~/v3/featureFlags.server"; +import { makeSetMultipleFlags } from "~/v3/featureFlags.server"; +import { validatePartialFeatureFlags } from "~/v3/featureFlags"; export async function action({ request }: ActionFunctionArgs) { // Next authenticate the request @@ -11,7 +12,7 @@ export async function action({ request }: ActionFunctionArgs) { return json({ error: "Invalid or Missing API key" }, { status: 401 }); } - const user = await prisma.user.findUnique({ + const user = await prisma.user.findFirst({ where: { id: authenticationResult.userId, }, diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts index 96bf340fe0e..bb0671355bd 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts @@ -2,7 +2,7 @@ import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server- import { z } from "zod"; import { prisma } from "~/db.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; -import { validatePartialFeatureFlags } from "~/v3/featureFlags.server"; +import { validatePartialFeatureFlags } from "~/v3/featureFlags"; const ParamsSchema = z.object({ organizationId: z.string(), @@ -15,7 +15,7 @@ async function authenticateAdmin(request: Request) { return { error: json({ error: "Invalid or Missing API key" }, { status: 401 }) }; } - const user = await prisma.user.findUnique({ + const user = await prisma.user.findFirst({ where: { id: authenticationResult.userId, }, @@ -41,7 +41,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const { organizationId } = ParamsSchema.parse(params); - const organization = await prisma.organization.findUnique({ + const organization = await prisma.organization.findFirst({ where: { id: organizationId, }, @@ -78,7 +78,7 @@ export async function action({ request, params }: ActionFunctionArgs) { const { organizationId } = ParamsSchema.parse(params); - const organization = await prisma.organization.findUnique({ + const organization = await prisma.organization.findFirst({ where: { id: organizationId, }, diff --git a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts new file mode 100644 index 00000000000..6081febb526 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts @@ -0,0 +1,132 @@ +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { Prisma } from "@trigger.dev/database"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { requireUser } from "~/services/session.server"; +import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, validatePartialFeatureFlags, getAllFlagControlTypes } from "~/v3/featureFlags"; +import { featuresForRequest } from "~/features.server"; + +// Session-auth route for the admin feature flags dialog. +// Uses replace semantics: the action writes the full flag set (or null to clear). +// Compare with v1 (admin.api.v1.orgs.$organizationId.feature-flags.ts) which +// uses PAT auth and merge semantics for programmatic use. + +const ParamsSchema = z.object({ + organizationId: z.string(), +}); + +export async function loader({ request, params }: LoaderFunctionArgs) { + const user = await requireUser(request); + if (!user.admin) { + throw new Response("Unauthorized", { status: 403 }); + } + + const { organizationId } = ParamsSchema.parse(params); + + const [organization, globalFlags, workerGroups] = await Promise.all([ + prisma.organization.findFirst({ + where: { id: organizationId }, + select: { + id: true, + title: true, + slug: true, + featureFlags: true, + }, + }), + getGlobalFlags(), + prisma.workerInstanceGroup.findMany({ + select: { id: true, name: true }, + orderBy: { name: "asc" }, + }), + ]); + + if (!organization) { + throw new Response("Organization not found", { status: 404 }); + } + + const orgFlagsResult = organization.featureFlags + ? validatePartialFeatureFlags(organization.featureFlags as Record) + : ({ success: false } as const); + + const orgFlags = orgFlagsResult.success ? orgFlagsResult.data : {}; + const controlTypes = getAllFlagControlTypes(); + + // Resolve worker group name for display + const workerGroupId = (globalFlags as Record)?.[ + FEATURE_FLAG.defaultWorkerInstanceGroupId + ]; + let workerGroupName: string | undefined; + if (typeof workerGroupId === "string") { + const wg = await prisma.workerInstanceGroup.findFirst({ + where: { id: workerGroupId }, + select: { name: true }, + }); + workerGroupName = wg?.name; + } + + const { isManagedCloud } = featuresForRequest(request); + + return json({ + org: { + id: organization.id, + title: organization.title, + slug: organization.slug, + }, + orgFlags, + globalFlags, + controlTypes, + workerGroupName, + workerGroups, + isManagedCloud, + }); +} + +export async function action({ request, params }: ActionFunctionArgs) { + const user = await requireUser(request); + if (!user.admin) { + throw new Response("Unauthorized", { status: 403 }); + } + + const { organizationId } = ParamsSchema.parse(params); + + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + + let featureFlags: typeof Prisma.JsonNull | Record; + + if ( + body === null || + (typeof body === "object" && !Array.isArray(body) && Object.keys(body).length === 0) + ) { + featureFlags = Prisma.JsonNull; + } else { + const validationResult = validatePartialFeatureFlags(body as Record); + if (!validationResult.success) { + return json( + { error: "Invalid feature flags", details: validationResult.error.issues }, + { status: 400 } + ); + } + featureFlags = validationResult.data; + } + + try { + await prisma.organization.update({ + where: { id: organizationId }, + data: { featureFlags: featureFlags as Prisma.InputJsonValue }, + }); + } catch (e) { + if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2025") { + throw new Response("Organization not found", { status: 404 }); + } + throw e; + } + + return json({ success: true }); +} diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx new file mode 100644 index 00000000000..65e975880cb --- /dev/null +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -0,0 +1,565 @@ +import { useFetcher } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { useEffect, useState } from "react"; +import stableStringify from "json-stable-stringify"; +import { json } from "@remix-run/server-runtime"; +import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { LockClosedIcon } from "@heroicons/react/20/solid"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { requireUser } from "~/services/session.server"; +import { + FEATURE_FLAG, + GLOBAL_LOCKED_FLAGS, + type FlagControlType, + getAllFlagControlTypes, + validatePartialFeatureFlags, +} from "~/v3/featureFlags"; +import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; +import { featuresForRequest } from "~/features.server"; +import { Button } from "~/components/primitives/Buttons"; +import { Callout } from "~/components/primitives/Callout"; +import { CheckboxWithLabel } from "~/components/primitives/Checkbox"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogDescription, + DialogFooter, +} from "~/components/primitives/Dialog"; +import { cn } from "~/utils/cn"; +import { + UNSET_VALUE, + BooleanControl, + EnumControl, + StringControl, + WorkerGroupControl, + type WorkerGroup, +} from "~/components/admin/FlagControls"; + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const user = await requireUser(request); + if (!user.admin) { + return redirect("/"); + } + + const [globalFlags, workerGroups] = await Promise.all([ + getGlobalFlags(), + prisma.workerInstanceGroup.findMany({ + select: { id: true, name: true }, + orderBy: { name: "asc" }, + }), + ]); + const controlTypes = getAllFlagControlTypes(); + + // Resolve env-based defaults for locked flags + const resolvedDefaults: Record = { + [FEATURE_FLAG.runsListRepository]: "clickhouse", + [FEATURE_FLAG.taskEventRepository]: env.EVENT_REPOSITORY_DEFAULT_STORE, + }; + + // Look up worker group name if the flag is set + const workerGroupId = (globalFlags as Record)?.[ + FEATURE_FLAG.defaultWorkerInstanceGroupId + ]; + const workerGroupName = + typeof workerGroupId === "string" + ? workerGroups.find((wg) => wg.id === workerGroupId)?.name + : undefined; + + const { isManagedCloud } = featuresForRequest(request); + + return typedjson({ + globalFlags, + controlTypes, + resolvedDefaults, + workerGroupName, + workerGroups, + isManagedCloud, + }); +}; + +export const action = async ({ request }: ActionFunctionArgs) => { + const user = await requireUser(request); + if (!user.admin) { + throw new Response("Unauthorized", { status: 403 }); + } + + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const payloadSchema = z.object({ flags: z.record(z.unknown()) }); + const parsed = payloadSchema.safeParse(body); + if (!parsed.success) { + return json({ error: "Invalid payload" }, { status: 400 }); + } + + const { isManagedCloud } = featuresForRequest(request); + + // On managed cloud, reject if payload includes locked flags + if (isManagedCloud) { + const lockedInPayload = Object.keys(parsed.data.flags).filter((key) => + GLOBAL_LOCKED_FLAGS.includes(key) + ); + if (lockedInPayload.length > 0) { + return json( + { error: `Cannot modify locked flags: ${lockedInPayload.join(", ")}` }, + { status: 400 } + ); + } + } + + const validationResult = validatePartialFeatureFlags(parsed.data.flags); + if (!validationResult.success) { + return json( + { error: "Invalid feature flags", details: validationResult.error.issues }, + { status: 400 } + ); + } + + const validatedFlags = validationResult.data as Record; + const controlTypes = getAllFlagControlTypes(); + const catalogKeys = Object.keys(controlTypes); + + const keysToDelete: string[] = []; + const upsertOps: ReturnType[] = []; + + for (const key of catalogKeys) { + if (key in validatedFlags) { + upsertOps.push( + prisma.featureFlag.upsert({ + where: { key }, + create: { key, value: validatedFlags[key] as any }, + update: { value: validatedFlags[key] as any }, + }) + ); + } else { + // On cloud, never delete locked flags (they're not in the payload + // because the UI doesn't include them). Locally, delete everything + // the user didn't include - full control. + const isProtected = isManagedCloud && GLOBAL_LOCKED_FLAGS.includes(key); + if (!isProtected) { + keysToDelete.push(key); + } + } + } + + await prisma.$transaction([ + ...upsertOps, + ...(keysToDelete.length > 0 + ? [prisma.featureFlag.deleteMany({ where: { key: { in: keysToDelete } } })] + : []), + ]); + + return json({ success: true }); +}; + +export default function AdminFeatureFlagsRoute() { + const { + globalFlags, + controlTypes, + resolvedDefaults, + workerGroupName, + workerGroups, + isManagedCloud, + } = useTypedLoaderData(); + const saveFetcher = useFetcher<{ success?: boolean; error?: string }>(); + + const [values, setValues] = useState>({}); + const [initialValues, setInitialValues] = useState>({}); + const [saveError, setSaveError] = useState(null); + const [confirmOpen, setConfirmOpen] = useState(false); + const [unlocked, setUnlocked] = useState(false); + + const isLocked = (key: string) => !unlocked && GLOBAL_LOCKED_FLAGS.includes(key); + + useEffect(() => { + const loaded = (globalFlags ?? {}) as Record; + // Only track editable flags in state + const editable: Record = {}; + for (const [key, value] of Object.entries(loaded)) { + if (!isLocked(key)) { + editable[key] = value; + } + } + setValues({ ...editable }); + setInitialValues({ ...editable }); + }, [globalFlags, unlocked]); + + useEffect(() => { + if (saveFetcher.data?.success) { + setSaveError(null); + setConfirmOpen(false); + } else if (saveFetcher.data?.error) { + setSaveError(saveFetcher.data.error); + } + }, [saveFetcher.data]); + + const isDirty = stableStringify(values) !== stableStringify(initialValues); + const isSaving = saveFetcher.state === "submitting"; + + const setFlagValue = (key: string, value: unknown) => { + setValues((prev) => ({ ...prev, [key]: value })); + }; + + const unsetFlag = (key: string) => { + setValues((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + }; + + const handleSave = () => { + saveFetcher.submit(JSON.stringify({ flags: values }), { + method: "POST", + encType: "application/json", + }); + }; + + const typedControlTypes = controlTypes as Record; + const typedResolvedDefaults = resolvedDefaults as Record; + const allFlags = (globalFlags ?? {}) as Record; + const sortedFlagKeys = Object.keys(typedControlTypes).sort(); + const workerGroupMap = new Map((workerGroups as WorkerGroup[]).map((wg) => [wg.id, wg.name])); + + const resolveWorkerGroupDisplay = (id: string) => { + const name = workerGroupMap.get(id); + return name ? `${name} (${id.slice(0, 8)}...)` : id; + }; + + return ( +
+
+

+ Global defaults for all organizations. Org-level overrides take precedence. When not set, + each consumer uses its own default. +

+ +
+ +
+ +
+ {sortedFlagKeys.map((key) => { + const control = typedControlTypes[key]; + const locked = isLocked(key); + + if (locked) { + return ( + + ); + } + + const isSet = key in values; + const isWorkerGroup = key === FEATURE_FLAG.defaultWorkerInstanceGroupId; + + return ( +
+
+
+ {isWorkerGroup ? "defaultWorkerInstanceGroup" : key} +
+
+ {isSet + ? isWorkerGroup + ? resolveWorkerGroupDisplay(values[key] as string) + : `value: ${String(values[key])}` + : typedResolvedDefaults[key] + ? `${typedResolvedDefaults[key]} (from env)` + : "not set"} +
+
+ +
+ + + {isWorkerGroup ? ( + { + if (val === UNSET_VALUE) { + unsetFlag(key); + } else { + setFlagValue(key, val); + } + }} + dimmed={!isSet} + /> + ) : ( + <> + {control.type === "boolean" && ( + setFlagValue(key, val)} + dimmed={!isSet} + /> + )} + + {control.type === "enum" && ( + { + if (val === UNSET_VALUE) { + unsetFlag(key); + } else { + setFlagValue(key, val); + } + }} + dimmed={!isSet} + /> + )} + + {control.type === "string" && ( + { + if (val === "") { + unsetFlag(key); + } else { + setFlagValue(key, val); + } + }} + dimmed={!isSet} + /> + )} + + )} +
+
+ ); + })} +
+ + {saveError && {saveError}} + +
+ {isDirty && ( + + )} + +
+
+ + +
+ ); +} + +// --- Locked Flag Row --- + +function LockedFlagRow({ + flagKey, + value, + resolvedDefault, + workerGroupName, +}: { + flagKey: string; + value: unknown; + resolvedDefault: string | undefined; + workerGroupName: string | undefined; +}) { + const isSet = value !== undefined; + const isWorkerGroup = flagKey === FEATURE_FLAG.defaultWorkerInstanceGroupId; + + let displayValue: string; + if (isSet) { + if (isWorkerGroup && workerGroupName) { + displayValue = `${workerGroupName} (${String(value).slice(0, 8)}...)`; + } else { + displayValue = String(value); + } + } else if (resolvedDefault) { + displayValue = `${resolvedDefault} (from env)`; + } else { + displayValue = "not set (required)"; + } + + return ( +
+
+
+ {isWorkerGroup ? "defaultWorkerInstanceGroup" : flagKey} +
+
{displayValue}
+
+ + +
+ ); +} + +// --- Confirmation Dialog with Diff --- + +function ConfirmDialog({ + open, + onOpenChange, + initialValues, + newValues, + controlTypes, + lockedKeys, + onConfirm, + isSaving, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + initialValues: Record; + newValues: Record; + controlTypes: Record; + lockedKeys: readonly string[]; + onConfirm: () => void; + isSaving: boolean; +}) { + const editableKeys = Object.keys(controlTypes) + .filter((key) => !lockedKeys.includes(key)) + .sort(); + + type Change = + | { key: string; type: "added"; newVal: string } + | { key: string; type: "removed"; oldVal: string } + | { key: string; type: "changed"; oldVal: string; newVal: string }; + + const changes = editableKeys.flatMap((key) => { + const wasSet = key in initialValues; + const isSet = key in newValues; + const oldVal = initialValues[key]; + const newVal = newValues[key]; + + if (!wasSet && !isSet) return []; + if (wasSet && isSet && stableStringify(oldVal) === stableStringify(newVal)) return []; + + if (!wasSet && isSet) { + return [{ key, type: "added" as const, newVal: String(newVal) }]; + } + if (wasSet && !isSet) { + return [{ key, type: "removed" as const, oldVal: String(oldVal) }]; + } + return [ + { + key, + type: "changed" as const, + oldVal: String(oldVal), + newVal: String(newVal), + }, + ]; + }); + + return ( + + + Confirm feature flag changes + + These changes affect all organizations globally. Please review carefully. + + +
+ {changes.length === 0 ? ( +

No changes to apply.

+ ) : ( + changes.map((change) => ( +
+
{change.key}
+ {change.type === "added" && ( +
+ {change.newVal}
+ )} + {change.type === "removed" && ( +
- {change.oldVal} (unset)
+ )} + {change.type === "changed" && ( + <> +
- {change.oldVal}
+
+ {change.newVal}
+ + )} +
+ )) + )} +
+ + + + + + + + ); +} diff --git a/apps/webapp/app/routes/admin.orgs.tsx b/apps/webapp/app/routes/admin.orgs.tsx index afdcf5e97c9..6f496362947 100644 --- a/apps/webapp/app/routes/admin.orgs.tsx +++ b/apps/webapp/app/routes/admin.orgs.tsx @@ -2,7 +2,9 @@ import { MagnifyingGlassIcon } from "@heroicons/react/20/solid"; import { Form } from "@remix-run/react"; import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson"; +import { useState } from "react"; import { z } from "zod"; +import { FeatureFlagsDialog } from "~/components/admin/FeatureFlagsDialog"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { CopyableText } from "~/components/primitives/CopyableText"; import { Input } from "~/components/primitives/Input"; @@ -46,6 +48,15 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { export default function AdminDashboardRoute() { const { organizations, filters, page, pageCount } = useTypedLoaderData(); + const [flagsOrgId, setFlagsOrgId] = useState(null); + const [flagsOpen, setFlagsOpen] = useState(false); + const flagsOrgTitle = organizations.find((o) => o.id === flagsOrgId)?.title ?? ""; + + const openFlagsDialog = (orgId: string) => { + setFlagsOrgId(orgId); + setFlagsOpen(true); + }; + return (
{org.v3Enabled ? "✅" : ""} {org.deletedAt ? "☠️" : ""} - - Impersonate - +
+ + + Impersonate + +
); @@ -135,6 +153,12 @@ export default function AdminDashboardRoute() { +
); } diff --git a/apps/webapp/app/routes/admin.tsx b/apps/webapp/app/routes/admin.tsx index c03de5fc537..4cd2deca533 100644 --- a/apps/webapp/app/routes/admin.tsx +++ b/apps/webapp/app/routes/admin.tsx @@ -36,6 +36,10 @@ export default function Page() { label: "LLM Models", to: "/admin/llm-models", }, + { + label: "Feature Flags", + to: "/admin/feature-flags", + }, { label: "Notifications", to: "/admin/notifications", diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.can-view-logs-page/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.can-view-logs-page/route.tsx index 2d205d90c10..f55307c7c34 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.can-view-logs-page/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.can-view-logs-page/route.tsx @@ -2,7 +2,7 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { typedjson } from "remix-typedjson"; import { requireUser } from "~/services/session.server"; import { prisma } from "~/db.server"; -import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags"; import { OrganizationParamsSchema } from "~/utils/pathBuilder"; async function hasLogsPageAccess( diff --git a/apps/webapp/app/services/runsRepository/runsRepository.server.ts b/apps/webapp/app/services/runsRepository/runsRepository.server.ts index 99e5e00ce6e..0bcc73eab6e 100644 --- a/apps/webapp/app/services/runsRepository/runsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/runsRepository.server.ts @@ -8,7 +8,8 @@ import parseDuration from "parse-duration"; import { z } from "zod"; import { timeFilters } from "~/components/runs/v3/SharedFilters"; import { type PrismaClient, type PrismaClientOrTransaction } from "~/db.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; import { startActiveSpan } from "~/v3/tracer.server"; import { logger } from "../logger.server"; import { ClickHouseRunsRepository } from "./clickhouseRunsRepository.server"; diff --git a/apps/webapp/app/v3/canAccessAi.server.ts b/apps/webapp/app/v3/canAccessAi.server.ts index d7957c4dcc7..61e9258e0b7 100644 --- a/apps/webapp/app/v3/canAccessAi.server.ts +++ b/apps/webapp/app/v3/canAccessAi.server.ts @@ -1,6 +1,7 @@ import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; export async function canAccessAi(options: { userId: string; diff --git a/apps/webapp/app/v3/canAccessAiModels.server.ts b/apps/webapp/app/v3/canAccessAiModels.server.ts index a3a489ce881..d1988ba7f89 100644 --- a/apps/webapp/app/v3/canAccessAiModels.server.ts +++ b/apps/webapp/app/v3/canAccessAiModels.server.ts @@ -1,6 +1,7 @@ import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; export async function canAccessAiModels(options: { userId: string; diff --git a/apps/webapp/app/v3/canAccessPrivateConnections.server.ts b/apps/webapp/app/v3/canAccessPrivateConnections.server.ts index ffbd75b2ca5..a801a4d2f24 100644 --- a/apps/webapp/app/v3/canAccessPrivateConnections.server.ts +++ b/apps/webapp/app/v3/canAccessPrivateConnections.server.ts @@ -1,6 +1,7 @@ import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; export async function canAccessPrivateConnections(options: { organizationSlug: string; diff --git a/apps/webapp/app/v3/canAccessQuery.server.ts b/apps/webapp/app/v3/canAccessQuery.server.ts index 87a248725b5..13b7602d937 100644 --- a/apps/webapp/app/v3/canAccessQuery.server.ts +++ b/apps/webapp/app/v3/canAccessQuery.server.ts @@ -1,6 +1,7 @@ import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; export async function canAccessQuery(options: { userId: string; diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index 2f457e23593..70ea6440321 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -7,7 +7,8 @@ import { import { IEventRepository, TraceEventOptions } from "./eventRepository.types"; import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; -import { FEATURE_FLAG, flag } from "../featureFlags.server"; +import { FEATURE_FLAG } from "../featureFlags"; +import { flag } from "../featureFlags.server"; import { getTaskEventStore } from "../taskEventStore.server"; export function resolveEventRepositoryForStore(store: string | undefined): IEventRepository { diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index cc977da7dcf..03353c914e6 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -1,31 +1,10 @@ -import { z } from "zod"; +import { type z } from "zod"; import { prisma, type PrismaClientOrTransaction } from "~/db.server"; - -export const FEATURE_FLAG = { - defaultWorkerInstanceGroupId: "defaultWorkerInstanceGroupId", - runsListRepository: "runsListRepository", - taskEventRepository: "taskEventRepository", - hasQueryAccess: "hasQueryAccess", - hasLogsPageAccess: "hasLogsPageAccess", - hasAiAccess: "hasAiAccess", - hasAiModelsAccess: "hasAiModelsAccess", - hasComputeAccess: "hasComputeAccess", - hasPrivateConnections: "hasPrivateConnections", -} as const; - -const FeatureFlagCatalog = { - [FEATURE_FLAG.defaultWorkerInstanceGroupId]: z.string(), - [FEATURE_FLAG.runsListRepository]: z.enum(["clickhouse", "postgres"]), - [FEATURE_FLAG.taskEventRepository]: z.enum(["clickhouse", "clickhouse_v2", "postgres"]), - [FEATURE_FLAG.hasQueryAccess]: z.coerce.boolean(), - [FEATURE_FLAG.hasLogsPageAccess]: z.coerce.boolean(), - [FEATURE_FLAG.hasAiAccess]: z.coerce.boolean(), - [FEATURE_FLAG.hasAiModelsAccess]: z.coerce.boolean(), - [FEATURE_FLAG.hasComputeAccess]: z.coerce.boolean(), - [FEATURE_FLAG.hasPrivateConnections]: z.coerce.boolean(), -}; - -type FeatureFlagKey = keyof typeof FeatureFlagCatalog; +import { + type FeatureFlagKey, + FeatureFlagCatalog, + FeatureFlagCatalogSchema, +} from "~/v3/featureFlags"; export type FlagsOptions = { key: T; @@ -43,7 +22,7 @@ export function makeFlag(_prisma: PrismaClientOrTransaction = prisma) { async function flag( opts: FlagsOptions ): Promise | undefined> { - const value = await _prisma.featureFlag.findUnique({ + const value = await _prisma.featureFlag.findFirst({ where: { key: opts.key, }, @@ -146,28 +125,6 @@ export const flag = makeFlag(); export const flags = makeFlags(); export const setFlag = makeSetFlag(); -// Create a Zod schema from the existing catalog -export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog); -export type FeatureFlagCatalog = z.infer; - -// Utility function to validate a feature flag value -export function validateFeatureFlagValue( - key: T, - value: unknown -): z.SafeParseReturnType> { - return FeatureFlagCatalog[key].safeParse(value); -} - -// Utility function to validate all feature flags at once -export function validateAllFeatureFlags(values: Record) { - return FeatureFlagCatalogSchema.safeParse(values); -} - -// Utility function to validate partial feature flags (all keys optional) -export function validatePartialFeatureFlags(values: Record) { - return FeatureFlagCatalogSchema.partial().safeParse(values); -} - // Utility function to set multiple feature flags at once export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma) { return async function setMultipleFlags( diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts new file mode 100644 index 00000000000..7571d27f67b --- /dev/null +++ b/apps/webapp/app/v3/featureFlags.ts @@ -0,0 +1,93 @@ +import { z } from "zod"; + +export const FEATURE_FLAG = { + defaultWorkerInstanceGroupId: "defaultWorkerInstanceGroupId", + runsListRepository: "runsListRepository", + taskEventRepository: "taskEventRepository", + hasQueryAccess: "hasQueryAccess", + hasLogsPageAccess: "hasLogsPageAccess", + hasAiAccess: "hasAiAccess", + hasAiModelsAccess: "hasAiModelsAccess", + hasComputeAccess: "hasComputeAccess", + hasPrivateConnections: "hasPrivateConnections", +} as const; + +export const FeatureFlagCatalog = { + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: z.string(), + [FEATURE_FLAG.runsListRepository]: z.enum(["clickhouse", "postgres"]), + [FEATURE_FLAG.taskEventRepository]: z.enum(["clickhouse", "clickhouse_v2", "postgres"]), + [FEATURE_FLAG.hasQueryAccess]: z.coerce.boolean(), + [FEATURE_FLAG.hasLogsPageAccess]: z.coerce.boolean(), + [FEATURE_FLAG.hasAiAccess]: z.coerce.boolean(), + [FEATURE_FLAG.hasAiModelsAccess]: z.coerce.boolean(), + [FEATURE_FLAG.hasComputeAccess]: z.coerce.boolean(), + [FEATURE_FLAG.hasPrivateConnections]: z.coerce.boolean(), +}; + +export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; + +// Infrastructure flags that are read-only on the global flags page. +// Shown with current/resolved value but no controls. +export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [ + FEATURE_FLAG.defaultWorkerInstanceGroupId, + FEATURE_FLAG.runsListRepository, + FEATURE_FLAG.taskEventRepository, +]; + +// Flags that are read-only on the org-level dialog. +// Shown with global value but no controls (org can't override these). +export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [ + FEATURE_FLAG.defaultWorkerInstanceGroupId, + FEATURE_FLAG.runsListRepository, + FEATURE_FLAG.taskEventRepository, +]; + +// Create a Zod schema from the existing catalog +export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog); +export type FeatureFlagCatalog = z.infer; + +// Utility function to validate a feature flag value +export function validateFeatureFlagValue( + key: T, + value: unknown +): z.SafeParseReturnType> { + return FeatureFlagCatalog[key].safeParse(value); +} + +// Utility function to validate all feature flags at once +export function validateAllFeatureFlags(values: Record) { + return FeatureFlagCatalogSchema.safeParse(values); +} + +// Utility function to validate partial feature flags (all keys optional) +export function validatePartialFeatureFlags(values: Record) { + return FeatureFlagCatalogSchema.partial().safeParse(values); +} + +// Utility types for catalog-driven UI rendering +export type FlagControlType = + | { type: "boolean" } + | { type: "enum"; options: string[] } + | { type: "string" }; + +export function getFlagControlType(schema: z.ZodTypeAny): FlagControlType { + const typeName = schema._def.typeName; + + if (typeName === "ZodBoolean") { + return { type: "boolean" }; + } + + if (typeName === "ZodEnum") { + return { type: "enum", options: schema._def.values as string[] }; + } + + return { type: "string" }; +} + +export function getAllFlagControlTypes(): Record { + const result: Record = {}; + for (const [key, schema] of Object.entries(FeatureFlagCatalog)) { + result[key] = getFlagControlType(schema); + } + return result; +} diff --git a/apps/webapp/app/v3/services/computeTemplateCreation.server.ts b/apps/webapp/app/v3/services/computeTemplateCreation.server.ts index 6c144b13448..4daa2667f25 100644 --- a/apps/webapp/app/v3/services/computeTemplateCreation.server.ts +++ b/apps/webapp/app/v3/services/computeTemplateCreation.server.ts @@ -3,7 +3,8 @@ import { machinePresetFromName } from "~/v3/machinePresets.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import type { PrismaClientOrTransaction } from "~/db.server"; -import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { ServiceValidationError } from "./baseService.server"; import { FailDeploymentService } from "./failDeployment.server"; diff --git a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts index 936f8bbd48e..6a900a16fd7 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts @@ -2,7 +2,8 @@ import { WorkerInstanceGroup, WorkerInstanceGroupType } from "@trigger.dev/datab import { WithRunEngine } from "../baseService.server"; import { WorkerGroupTokenService } from "./workerGroupTokenService.server"; import { logger } from "~/services/logger.server"; -import { FEATURE_FLAG, makeFlag, makeSetFlag } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag, makeSetFlag } from "~/v3/featureFlags.server"; export class WorkerGroupService extends WithRunEngine { private readonly defaultNamePrefix = "worker_group"; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 0752ac8fe03..c1b75596cf8 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -162,6 +162,7 @@ "ioredis": "^5.3.2", "isbot": "^3.6.5", "jose": "^5.4.0", + "json-stable-stringify": "^1.3.0", "jsonpointer": "^5.0.1", "lodash.omit": "^4.5.0", "lucide-react": "^0.229.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be0fe1a6f6b..15477faf4f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -629,6 +629,9 @@ importers: jose: specifier: ^5.4.0 version: 5.4.0 + json-stable-stringify: + specifier: ^1.3.0 + version: 1.3.0 jsonpointer: specifier: ^5.0.1 version: 5.0.1 @@ -1131,7 +1134,7 @@ importers: version: 18.3.1 react-email: specifier: ^2.1.1 - version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0) + version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0) resend: specifier: ^3.2.0 version: 3.2.0 @@ -39434,7 +39437,7 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0): + react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0): dependencies: '@babel/parser': 7.24.1 '@radix-ui/colors': 1.0.1 @@ -39471,8 +39474,8 @@ snapshots: react: 18.3.1 react-dom: 18.2.0(react@18.3.1) shelljs: 0.8.5 - socket.io: 4.7.3 - socket.io-client: 4.7.3 + socket.io: 4.7.3(bufferutil@4.0.9) + socket.io-client: 4.7.3(bufferutil@4.0.9) sonner: 1.3.1(react-dom@18.2.0(react@18.3.1))(react@18.3.1) source-map-js: 1.0.2 stacktrace-parser: 0.1.10 @@ -40699,7 +40702,7 @@ snapshots: - supports-color - utf-8-validate - socket.io-client@4.7.3: + socket.io-client@4.7.3(bufferutil@4.0.9): dependencies: '@socket.io/component-emitter': 3.1.0 debug: 4.3.7(supports-color@10.0.0) @@ -40728,7 +40731,7 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.7.3: + socket.io@4.7.3(bufferutil@4.0.9): dependencies: accepts: 1.3.8 base64id: 2.0.0 From b07567888dc9c0a56dbb1753b4983c9d70500501 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 30 Mar 2026 21:59:43 +0100 Subject: [PATCH 002/579] fix(llm-catalog): refresh default model pricing on sync (#3281) - Rebuild llm_pricing_tiers and llm_prices in syncLlmCatalog for source=default - Add vitest config, sync regression tests, and pin vitest 3.1.4 - Update pnpm-lock.yaml for the new devDependency --- .../llm-model-catalog/package.json | 5 + .../llm-model-catalog/src/sync.test.ts | 203 ++++++++++++++++++ .../llm-model-catalog/src/sync.ts | 80 +++++-- .../llm-model-catalog/tsconfig.json | 2 +- .../llm-model-catalog/vitest.config.ts | 16 ++ pnpm-lock.yaml | 9 +- 6 files changed, 296 insertions(+), 19 deletions(-) create mode 100644 internal-packages/llm-model-catalog/src/sync.test.ts create mode 100644 internal-packages/llm-model-catalog/vitest.config.ts diff --git a/internal-packages/llm-model-catalog/package.json b/internal-packages/llm-model-catalog/package.json index be27ce3529d..ac5cdafc0a6 100644 --- a/internal-packages/llm-model-catalog/package.json +++ b/internal-packages/llm-model-catalog/package.json @@ -9,7 +9,12 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*" }, + "devDependencies": { + "@internal/testcontainers": "workspace:*", + "vitest": "3.1.4" + }, "scripts": { + "test": "vitest --sequence.concurrent=false --no-file-parallelism", "typecheck": "tsc --noEmit", "generate": "node scripts/generate.mjs", "sync-prices": "bash scripts/sync-model-prices.sh && node scripts/generate.mjs", diff --git a/internal-packages/llm-model-catalog/src/sync.test.ts b/internal-packages/llm-model-catalog/src/sync.test.ts new file mode 100644 index 00000000000..d2138564ef6 --- /dev/null +++ b/internal-packages/llm-model-catalog/src/sync.test.ts @@ -0,0 +1,203 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { defaultModelPrices } from "./defaultPrices.js"; +import { modelCatalog } from "./modelCatalog.js"; +import { syncLlmCatalog } from "./sync.js"; + +function getGpt4oDefinition() { + const def = defaultModelPrices.find((m) => m.modelName === "gpt-4o"); + if (def === undefined) { + throw new Error("expected gpt-4o in defaultModelPrices"); + } + return def; +} + +const gpt4oDef = getGpt4oDefinition(); + +function getGeminiProDefinition() { + const def = defaultModelPrices.find((m) => m.modelName === "gemini-pro"); + if (def === undefined) { + throw new Error("expected gemini-pro in defaultModelPrices"); + } + return def; +} + +const geminiProDef = getGeminiProDefinition(); + +/** If sync used `catalog?.baseModelName ?? existing.baseModelName`, sync would keep this string instead of clearing to null. */ +const STALE_BASE_MODEL_NAME = "wrong-base-model-sentinel"; + +const STALE_INPUT_PRICE = 0.099; +const STALE_OUTPUT_PRICE = 0.088; + +async function createGpt4oWithStalePricing( + prisma: PrismaClient, + source: "default" | "admin" +) { + const model = await prisma.llmModel.create({ + data: { + friendlyId: generateFriendlyId("llm_model"), + projectId: null, + modelName: gpt4oDef.modelName, + matchPattern: "^stale-pattern$", + startDate: gpt4oDef.startDate ? new Date(gpt4oDef.startDate) : null, + source, + provider: "stale-provider", + description: "stale description", + contextWindow: 111, + maxOutputTokens: 222, + capabilities: ["stale-cap"], + isHidden: true, + baseModelName: "stale-base", + }, + }); + + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: "Standard", + isDefault: true, + priority: 0, + conditions: [], + prices: { + create: [ + { modelId: model.id, usageType: "input", price: STALE_INPUT_PRICE }, + { modelId: model.id, usageType: "output", price: STALE_OUTPUT_PRICE }, + ], + }, + }, + }); + + return model; +} + +async function createGeminiProWithStaleBaseModelName(prisma: PrismaClient) { + const catalogEntry = modelCatalog[geminiProDef.modelName]; + expect(catalogEntry).toBeDefined(); + expect(catalogEntry.baseModelName).toBeNull(); + + const model = await prisma.llmModel.create({ + data: { + friendlyId: generateFriendlyId("llm_model"), + projectId: null, + modelName: geminiProDef.modelName, + matchPattern: "^stale-gemini-pattern$", + startDate: geminiProDef.startDate ? new Date(geminiProDef.startDate) : null, + source: "default", + provider: "stale-provider", + description: "stale description", + contextWindow: 111, + maxOutputTokens: 222, + capabilities: ["stale-cap"], + isHidden: true, + baseModelName: STALE_BASE_MODEL_NAME, + }, + }); + + const tier = geminiProDef.pricingTiers[0]; + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + + return model; +} + +async function loadGpt4oWithTiers(prisma: PrismaClient) { + return prisma.llmModel.findFirst({ + where: { projectId: null, modelName: gpt4oDef.modelName }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + }); +} + +function expectBundledGpt4oPricing(model: NonNullable>>) { + expect(model.matchPattern).toBe(gpt4oDef.matchPattern); + expect(model.pricingTiers).toHaveLength(gpt4oDef.pricingTiers.length); + + const dbTier = model.pricingTiers[0]; + const defTier = gpt4oDef.pricingTiers[0]; + expect(dbTier.name).toBe(defTier.name); + expect(dbTier.isDefault).toBe(defTier.isDefault); + expect(dbTier.priority).toBe(defTier.priority); + + const priceByType = new Map(dbTier.prices.map((p) => [p.usageType, Number(p.price)])); + for (const [usageType, expected] of Object.entries(defTier.prices)) { + expect(priceByType.get(usageType)).toBeCloseTo(expected, 12); + } + expect(priceByType.size).toBe(Object.keys(defTier.prices).length); +} + +describe("syncLlmCatalog", () => { + postgresTest( + "rebuilds gpt-4o pricing tiers from bundled defaults when source is default", + async ({ prisma }) => { + await createGpt4oWithStalePricing(prisma, "default"); + + const result = await syncLlmCatalog(prisma); + + expect(result.modelsUpdated).toBe(1); + expect(result.modelsSkipped).toBe(defaultModelPrices.length - 1); + + const after = await loadGpt4oWithTiers(prisma); + expect(after).not.toBeNull(); + expectBundledGpt4oPricing(after!); + } + ); + + postgresTest( + "does not replace pricing tiers when model source is not default", + async ({ prisma }) => { + await createGpt4oWithStalePricing(prisma, "admin"); + + const result = await syncLlmCatalog(prisma); + + expect(result.modelsUpdated).toBe(0); + expect(result.modelsSkipped).toBeGreaterThanOrEqual(1); + + const after = await loadGpt4oWithTiers(prisma); + expect(after).not.toBeNull(); + expect(after!.matchPattern).toBe("^stale-pattern$"); + expect(after!.pricingTiers).toHaveLength(1); + const prices = after!.pricingTiers[0].prices; + const input = prices.find((p) => p.usageType === "input"); + const output = prices.find((p) => p.usageType === "output"); + expect(Number(input?.price)).toBeCloseTo(STALE_INPUT_PRICE, 12); + expect(Number(output?.price)).toBeCloseTo(STALE_OUTPUT_PRICE, 12); + expect(prices).toHaveLength(2); + } + ); + + postgresTest( + "clears baseModelName when bundled catalog has null (regression for nullish-coalescing merge)", + async ({ prisma }) => { + await createGeminiProWithStaleBaseModelName(prisma); + + await syncLlmCatalog(prisma); + + const after = await prisma.llmModel.findFirst({ + where: { projectId: null, modelName: geminiProDef.modelName }, + }); + expect(after).not.toBeNull(); + expect(after!.baseModelName).toBeNull(); + } + ); +}); diff --git a/internal-packages/llm-model-catalog/src/sync.ts b/internal-packages/llm-model-catalog/src/sync.ts index b600e39a692..aa0611dbda2 100644 --- a/internal-packages/llm-model-catalog/src/sync.ts +++ b/internal-packages/llm-model-catalog/src/sync.ts @@ -1,6 +1,27 @@ -import type { PrismaClient } from "@trigger.dev/database"; +import type { Prisma, PrismaClient } from "@trigger.dev/database"; import { defaultModelPrices } from "./defaultPrices.js"; import { modelCatalog } from "./modelCatalog.js"; +import type { DefaultModelDefinition } from "./types.js"; + +function pricingTierCreateData( + modelId: string, + tier: DefaultModelDefinition["pricingTiers"][number] +): Prisma.LlmPricingTierUncheckedCreateInput { + return { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, + }; +} export async function syncLlmCatalog(prisma: PrismaClient): Promise<{ modelsUpdated: number; @@ -31,24 +52,49 @@ export async function syncLlmCatalog(prisma: PrismaClient): Promise<{ const catalog = modelCatalog[modelDef.modelName]; - await prisma.llmModel.update({ - where: { id: existing.id }, - data: { - // Update match pattern and start date from Langfuse (may have changed) - matchPattern: modelDef.matchPattern, - startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, - // Update catalog metadata - provider: catalog?.provider ?? existing.provider, - description: catalog?.description ?? existing.description, - contextWindow: catalog?.contextWindow ?? existing.contextWindow, - maxOutputTokens: catalog?.maxOutputTokens ?? existing.maxOutputTokens, - capabilities: catalog?.capabilities ?? existing.capabilities, - isHidden: catalog?.isHidden ?? existing.isHidden, - baseModelName: catalog?.baseModelName ?? existing.baseModelName, - }, + const applied = await prisma.$transaction(async (tx) => { + const updateResult = await tx.llmModel.updateMany({ + where: { id: existing.id, source: "default" }, + data: { + // Update match pattern and start date from Langfuse (may have changed) + matchPattern: modelDef.matchPattern, + startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, + // Update catalog metadata + provider: catalog?.provider ?? existing.provider, + description: catalog?.description ?? existing.description, + contextWindow: + catalog?.contextWindow === undefined ? existing.contextWindow : catalog.contextWindow, + maxOutputTokens: + catalog?.maxOutputTokens === undefined + ? existing.maxOutputTokens + : catalog.maxOutputTokens, + capabilities: catalog?.capabilities ?? existing.capabilities, + isHidden: catalog?.isHidden ?? existing.isHidden, + baseModelName: + catalog?.baseModelName === undefined + ? existing.baseModelName + : catalog.baseModelName, + }, + }); + + if (updateResult.count !== 1) { + return false; + } + + await tx.llmPricingTier.deleteMany({ where: { modelId: existing.id } }); + + for (const tier of modelDef.pricingTiers) { + await tx.llmPricingTier.create({ + data: pricingTierCreateData(existing.id, tier), + }); + } + + return true; }); - modelsUpdated++; + if (applied) { + modelsUpdated++; + } } return { modelsUpdated, modelsSkipped }; diff --git a/internal-packages/llm-model-catalog/tsconfig.json b/internal-packages/llm-model-catalog/tsconfig.json index c64cf33133b..7c980007eef 100644 --- a/internal-packages/llm-model-catalog/tsconfig.json +++ b/internal-packages/llm-model-catalog/tsconfig.json @@ -15,5 +15,5 @@ "strict": true, "resolveJsonModule": true }, - "exclude": ["node_modules"] + "exclude": ["node_modules", "**/*.test.ts"] } diff --git a/internal-packages/llm-model-catalog/vitest.config.ts b/internal-packages/llm-model-catalog/vitest.config.ts new file mode 100644 index 00000000000..474961216bf --- /dev/null +++ b/internal-packages/llm-model-catalog/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + isolate: true, + fileParallelism: false, + poolOptions: { + threads: { + singleThread: true, + }, + }, + testTimeout: 120_000, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 15477faf4f8..d460bdb1096 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1160,6 +1160,13 @@ importers: '@trigger.dev/database': specifier: workspace:* version: link:../database + devDependencies: + '@internal/testcontainers': + specifier: workspace:* + version: link:../testcontainers + vitest: + specifier: 3.1.4 + version: 3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) internal-packages/otlp-importer: dependencies: @@ -42293,7 +42300,7 @@ snapshots: vite-node@3.1.4(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: cac: 6.7.14 - debug: 4.4.1 + debug: 4.4.3(supports-color@10.0.0) es-module-lexer: 1.7.0 pathe: 2.0.3 vite: 5.4.21(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) From 0e63f8317e7c8b472d8c9c7bb36ad3986d0238c7 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:25:07 +0100 Subject: [PATCH 003/579] feat: add ttl support at task and config levels (#3196) Add TTL (time-to-live) defaults at task-level and config-level, with precedence: per-trigger > task > config > dev default (10m). Docs PR: #3200 (merge after packages are released) --- .changeset/quiet-dogs-fly.md | 7 ++ apps/webapp/CLAUDE.md | 11 +++ .../app/runEngine/concerns/queues.server.ts | 69 +++++++++++++++---- .../runEngine/services/triggerTask.server.ts | 26 ++++--- apps/webapp/app/runEngine/types.ts | 2 +- .../services/createBackgroundWorker.server.ts | 4 +- .../migration.sql | 2 + .../database/prisma/schema.prisma | 2 + .../src/entryPoints/dev-index-worker.ts | 14 ++++ .../src/entryPoints/managed-index-worker.ts | 14 ++++ packages/core/src/v3/config.ts | 19 +++++ packages/core/src/v3/schemas/resources.ts | 1 + packages/core/src/v3/schemas/schemas.ts | 1 + packages/core/src/v3/types/tasks.ts | 23 +++++++ packages/trigger-sdk/src/v3/shared.ts | 2 + references/hello-world/src/trigger/example.ts | 1 + .../hello-world/src/trigger/schedule.ts | 1 + references/hello-world/trigger.config.ts | 1 + 18 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 .changeset/quiet-dogs-fly.md create mode 100644 internal-packages/database/prisma/migrations/20260309120000_add_ttl_to_background_worker_task/migration.sql diff --git a/.changeset/quiet-dogs-fly.md b/.changeset/quiet-dogs-fly.md new file mode 100644 index 00000000000..e6017304760 --- /dev/null +++ b/.changeset/quiet-dogs-fly.md @@ -0,0 +1,7 @@ +--- +"@trigger.dev/sdk": patch +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Add support for setting TTL (time-to-live) defaults at the task level and globally in trigger.config.ts, with per-trigger overrides still taking precedence diff --git a/apps/webapp/CLAUDE.md b/apps/webapp/CLAUDE.md index 00ed6aaf78b..b0f5e09b829 100644 --- a/apps/webapp/CLAUDE.md +++ b/apps/webapp/CLAUDE.md @@ -99,6 +99,17 @@ The `app/v3/` directory name is misleading - most code is actively used by V2. O Some services (e.g., `cancelTaskRun.server.ts`, `batchTriggerV3.server.ts`) branch on `RunEngineVersion` to support both V1 and V2. When editing these, only modify V2 code paths. +## Performance: Trigger Hot Path + +The `triggerTask.server.ts` service is the **highest-throughput code path** in the system. Every API trigger call goes through it. Keep it fast: + +- **Do NOT add database queries** to `triggerTask.server.ts` or `batchTriggerV3.server.ts`. Task defaults (TTL, etc.) are resolved via `backgroundWorkerTask.findFirst()` in the queue concern (`queues.server.ts`) - one query per request, in mutually exclusive branches depending on locked/non-locked path. Piggyback on the existing query instead of adding new ones. +- **Two-stage resolution pattern**: Task metadata is resolved in two stages by design: + 1. **Trigger time** (`triggerTask.server.ts`): Only TTL is resolved from task defaults. Everything else uses whatever the caller provides. + 2. **Dequeue time** (`dequeueSystem.ts`): Full `BackgroundWorkerTask` is loaded and retry config, machine config, maxDuration, etc. are resolved against task defaults. +- If you need to add a new task-level default, **add it to the existing `select` clause** in the `backgroundWorkerTask.findFirst()` query — do NOT add a second query. If the default doesn't need to be known at trigger time, resolve it at dequeue time instead. +- Batch triggers (`batchTriggerV3.server.ts`) follow the same pattern — keep batch paths equally fast. + ## Prisma Query Patterns - **Always use `findFirst` instead of `findUnique`.** Prisma's `findUnique` has an implicit DataLoader that batches concurrent calls into a single `IN` query. This batching cannot be disabled and has active bugs even in Prisma 6.x: uppercase UUIDs returning null (#25484, confirmed 6.4.1), composite key SQL correctness issues (#22202), and 5-10x worse performance than manual DataLoader (#6573, open since 2021). `findFirst` is never batched and avoids this entire class of issues. diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index 12b0b29c5ff..4a4dcc8e98f 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -73,19 +73,20 @@ export class DefaultQueueManager implements QueueManager { ): Promise { let queueName: string; let lockedQueueId: string | undefined; + let taskTtl: string | null | undefined; // Determine queue name based on lockToVersion and provided options if (lockedBackgroundWorker) { // Task is locked to a specific worker version const specifiedQueueName = extractQueueName(request.body.options?.queue); + if (specifiedQueueName) { - // A specific queue name is provided + // A specific queue name is provided, validate it exists for the locked worker const specifiedQueue = await this.prisma.taskQueue.findFirst({ - // Validate it exists for the locked worker where: { name: specifiedQueueName, runtimeEnvironmentId: request.environment.id, - workers: { some: { id: lockedBackgroundWorker.id } }, // Ensure the queue is associated with any task of the locked worker + workers: { some: { id: lockedBackgroundWorker.id } }, }, }); @@ -95,11 +96,26 @@ export class DefaultQueueManager implements QueueManager { }'.` ); } + // Use the validated queue name directly queueName = specifiedQueue.name; lockedQueueId = specifiedQueue.id; + + // Only fetch task for TTL if caller didn't provide a per-trigger TTL + if (request.body.options?.ttl === undefined) { + const lockedTask = await this.prisma.backgroundWorkerTask.findFirst({ + where: { + workerId: lockedBackgroundWorker.id, + runtimeEnvironmentId: request.environment.id, + slug: request.taskId, + }, + select: { ttl: true }, + }); + + taskTtl = lockedTask?.ttl; + } } else { - // No specific queue name provided, use the default queue for the task on the locked worker + // No queue override - fetch task with queue to get both default queue and TTL const lockedTask = await this.prisma.backgroundWorkerTask.findFirst({ where: { workerId: lockedBackgroundWorker.id, @@ -118,6 +134,8 @@ export class DefaultQueueManager implements QueueManager { ); } + taskTtl = lockedTask.ttl; + if (!lockedTask.queue) { // This case should ideally be prevented by earlier checks or schema constraints, // but handle it defensively. @@ -131,6 +149,7 @@ export class DefaultQueueManager implements QueueManager { }'.` ); } + // Use the task's default queue name queueName = lockedTask.queue.name; lockedQueueId = lockedTask.queue.id; @@ -145,7 +164,9 @@ export class DefaultQueueManager implements QueueManager { } // Get queue name using the helper for non-locked case (handles provided name or finds default) - queueName = await this.getQueueName(request); + const taskInfo = await this.getTaskQueueInfo(request); + queueName = taskInfo.queueName; + taskTtl = taskInfo.taskTtl; } // Sanitize the final determined queue name once @@ -161,21 +182,27 @@ export class DefaultQueueManager implements QueueManager { return { queueName, lockedQueueId, + taskTtl, }; } - async getQueueName(request: TriggerTaskRequest): Promise { + private async getTaskQueueInfo( + request: TriggerTaskRequest + ): Promise<{ queueName: string; taskTtl?: string | null }> { const { taskId, environment, body } = request; const { queue } = body.options ?? {}; // Use extractQueueName to handle double-wrapped queue objects - const queueName = extractQueueName(queue); - if (queueName) { - return queueName; - } + const overriddenQueueName = extractQueueName(queue); const defaultQueueName = `task/${taskId}`; + // When caller provides both a queue override and a per-trigger TTL, + // we don't need any DB queries - the per-trigger TTL takes precedence + if (overriddenQueueName && body.options?.ttl !== undefined) { + return { queueName: overriddenQueueName, taskTtl: undefined }; + } + // Find the current worker for the environment const worker = await findCurrentWorkerFromEnvironment(environment, this.prisma); @@ -185,7 +212,21 @@ export class DefaultQueueManager implements QueueManager { environmentId: environment.id, }); - return defaultQueueName; + return { queueName: overriddenQueueName ?? defaultQueueName, taskTtl: undefined }; + } + + // When queue is overridden, we only need TTL from the task (no queue join needed) + if (overriddenQueueName) { + const task = await this.prisma.backgroundWorkerTask.findFirst({ + where: { + workerId: worker.id, + runtimeEnvironmentId: environment.id, + slug: taskId, + }, + select: { ttl: true }, + }); + + return { queueName: overriddenQueueName, taskTtl: task?.ttl }; } const task = await this.prisma.backgroundWorkerTask.findFirst({ @@ -205,7 +246,7 @@ export class DefaultQueueManager implements QueueManager { environmentId: environment.id, }); - return defaultQueueName; + return { queueName: defaultQueueName, taskTtl: undefined }; } if (!task.queue) { @@ -215,10 +256,10 @@ export class DefaultQueueManager implements QueueManager { queueConfig: task.queueConfig, }); - return defaultQueueName; + return { queueName: defaultQueueName, taskTtl: task.ttl }; } - return task.queue.name ?? defaultQueueName; + return { queueName: task.queue.name ?? defaultQueueName, taskTtl: task.ttl }; } async validateQueueLimits( diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index 1fe3b839e39..d87bfa67db9 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -191,11 +191,6 @@ export class RunEngineTriggerTaskService { } } - const ttl = - typeof body.options?.ttl === "number" - ? stringifyDuration(body.options?.ttl) - : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); - // Get parent run if specified const parentRun = body.options?.parentRunId ? await this.prisma.taskRun.findFirst({ @@ -251,10 +246,23 @@ export class RunEngineTriggerTaskService { }) : undefined; - const { queueName, lockedQueueId } = await this.queueConcern.resolveQueueProperties( - triggerRequest, - lockedToBackgroundWorker ?? undefined - ); + const { queueName, lockedQueueId, taskTtl } = + await this.queueConcern.resolveQueueProperties( + triggerRequest, + lockedToBackgroundWorker ?? undefined + ); + + // Resolve TTL with precedence: per-trigger > task-level > dev default + let ttl: string | undefined; + + if (body.options?.ttl !== undefined) { + ttl = + typeof body.options.ttl === "number" + ? stringifyDuration(body.options.ttl) + : body.options.ttl; + } else { + ttl = taskTtl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); + } if (!options.skipChecks) { const queueSizeGuard = await this.queueConcern.validateQueueLimits( diff --git a/apps/webapp/app/runEngine/types.ts b/apps/webapp/app/runEngine/types.ts index 3fc8d8034b7..310e0b32e5a 100644 --- a/apps/webapp/app/runEngine/types.ts +++ b/apps/webapp/app/runEngine/types.ts @@ -48,6 +48,7 @@ export type QueueValidationResult = export type QueueProperties = { queueName: string; lockedQueueId?: string; + taskTtl?: string | null; }; export type LockedBackgroundWorker = Pick< @@ -61,7 +62,6 @@ export interface QueueManager { request: TriggerTaskRequest, lockedBackgroundWorker?: LockedBackgroundWorker ): Promise; - getQueueName(request: TriggerTaskRequest): Promise; validateQueueLimits( env: AuthenticatedEnvironment, queueName: string, diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index c191ae5f397..1b51ec04aee 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -6,7 +6,7 @@ import { QueueManifest, TaskResource, } from "@trigger.dev/core/v3"; -import { BackgroundWorkerId } from "@trigger.dev/core/v3/isomorphic"; +import { BackgroundWorkerId, stringifyDuration } from "@trigger.dev/core/v3/isomorphic"; import type { BackgroundWorker, TaskQueue, TaskQueueType } from "@trigger.dev/database"; import cronstrue from "cronstrue"; import { $transaction, Prisma, PrismaClientOrTransaction } from "~/db.server"; @@ -286,6 +286,8 @@ async function createWorkerTask( triggerSource: task.triggerSource === "schedule" ? "SCHEDULED" : "STANDARD", fileId: tasksToBackgroundFiles?.get(task.id) ?? null, maxDurationInSeconds: task.maxDuration ? clampMaxDuration(task.maxDuration) : null, + ttl: + typeof task.ttl === "number" ? stringifyDuration(task.ttl) ?? null : task.ttl ?? null, queueId: queue.id, payloadSchema: task.payloadSchema as any, }, diff --git a/internal-packages/database/prisma/migrations/20260309120000_add_ttl_to_background_worker_task/migration.sql b/internal-packages/database/prisma/migrations/20260309120000_add_ttl_to_background_worker_task/migration.sql new file mode 100644 index 00000000000..569fb94643b --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260309120000_add_ttl_to_background_worker_task/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."BackgroundWorkerTask" ADD COLUMN IF NOT EXISTS "ttl" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 9b2388f87b1..142fcb6dff7 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -648,6 +648,8 @@ model BackgroundWorkerTask { maxDurationInSeconds Int? + ttl String? + triggerSource TaskTriggerSource @default(STANDARD) payloadSchema Json? diff --git a/packages/cli-v3/src/entryPoints/dev-index-worker.ts b/packages/cli-v3/src/entryPoints/dev-index-worker.ts index 3328f553ca6..0b8ee54b0a1 100644 --- a/packages/cli-v3/src/entryPoints/dev-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-index-worker.ts @@ -132,6 +132,20 @@ if (typeof config.maxDuration === "number") { }); } +// If the config has a TTL, we need to apply it to all tasks that don't have a TTL +if (config.ttl !== undefined) { + tasks = tasks.map((task) => { + if (task.ttl === undefined) { + return { + ...task, + ttl: config.ttl, + } satisfies TaskManifest; + } + + return task; + }); +} + // If the config has a machine preset, we need to apply it to all tasks that don't have a machine preset if (typeof config.machine === "string") { tasks = tasks.map((task) => { diff --git a/packages/cli-v3/src/entryPoints/managed-index-worker.ts b/packages/cli-v3/src/entryPoints/managed-index-worker.ts index f300e7ac716..efc226b99e0 100644 --- a/packages/cli-v3/src/entryPoints/managed-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-index-worker.ts @@ -132,6 +132,20 @@ if (typeof config.maxDuration === "number") { }); } +// If the config has a TTL, we need to apply it to all tasks that don't have a TTL +if (config.ttl !== undefined) { + tasks = tasks.map((task) => { + if (task.ttl === undefined) { + return { + ...task, + ttl: config.ttl, + } satisfies TaskManifest; + } + + return task; + }); +} + // If the config has a machine preset, we need to apply it to all tasks that don't have a machine preset if (typeof config.machine === "string") { tasks = tasks.map((task) => { diff --git a/packages/core/src/v3/config.ts b/packages/core/src/v3/config.ts index bf1e6c687c8..40334f04280 100644 --- a/packages/core/src/v3/config.ts +++ b/packages/core/src/v3/config.ts @@ -176,6 +176,25 @@ export type TriggerConfig = { */ maxDuration: number; + /** + * Set a default time-to-live (TTL) for all task runs in the project. If a run is not executed within this time, it will be removed from the queue and never execute. + * + * This can be a string like "1h" (1 hour), "30m" (30 minutes), "1d" (1 day), or a number of seconds. + * + * You can override this on a per-task basis by setting the `ttl` option on the task definition, or per-trigger by setting the `ttl` option when triggering. + * + * @example + * + * ```ts + * export default defineConfig({ + * project: "my-project", + * maxDuration: 3600, + * ttl: "1h", + * }); + * ``` + */ + ttl?: string | number; + /** * Enable console logging while running the dev CLI. This will print out logs from console.log, console.warn, and console.error. By default all logs are sent to the trigger.dev backend, and not logged to the console. */ diff --git a/packages/core/src/v3/schemas/resources.ts b/packages/core/src/v3/schemas/resources.ts index 9a9a86e4e91..e681c728416 100644 --- a/packages/core/src/v3/schemas/resources.ts +++ b/packages/core/src/v3/schemas/resources.ts @@ -13,6 +13,7 @@ export const TaskResource = z.object({ triggerSource: z.string().optional(), schedule: ScheduleMetadata.optional(), maxDuration: z.number().optional(), + ttl: z.string().or(z.number().nonnegative().int()).optional(), // JSONSchema type - using z.unknown() for runtime validation to accept JSONSchema7 payloadSchema: z.unknown().optional(), }); diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index 2d79d017426..4ec559ebf41 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -189,6 +189,7 @@ const taskMetadata = { triggerSource: z.string().optional(), schedule: ScheduleMetadata.optional(), maxDuration: z.number().optional(), + ttl: z.string().or(z.number().nonnegative().int()).optional(), payloadSchema: z.unknown().optional(), }; diff --git a/packages/core/src/v3/types/tasks.ts b/packages/core/src/v3/types/tasks.ts index 184aebe78a2..d04d088ef1a 100644 --- a/packages/core/src/v3/types/tasks.ts +++ b/packages/core/src/v3/types/tasks.ts @@ -278,6 +278,29 @@ type CommonTaskOptions< */ maxDuration?: number; + /** + * Set a default time-to-live for runs of this task. If the run is not executed within this time, it will be removed from the queue and never execute. + * + * This can be a string like "1h" (1 hour), "30m" (30 minutes), "1d" (1 day), or a number of seconds. + * + * If omitted it will use the value in your `trigger.config.ts` file, if set. + * + * You can override this on a per-trigger basis by setting the `ttl` option when triggering the task. + * + * @example + * + * ```ts + * export const myTask = task({ + * id: "my-task", + * ttl: "10m", + * run: async (payload) => { + * //... + * }, + * }); + * ``` + */ + ttl?: string | number; + /** This gets called when a task is triggered. It's where you put the code you want to execute. * * @param payload - The payload that is passed to your task when it's triggered. This must be JSON serializable. diff --git a/packages/trigger-sdk/src/v3/shared.ts b/packages/trigger-sdk/src/v3/shared.ts index c03732c12ea..c69bceeb535 100644 --- a/packages/trigger-sdk/src/v3/shared.ts +++ b/packages/trigger-sdk/src/v3/shared.ts @@ -236,6 +236,7 @@ export function createTask< retry: params.retry ? { ...defaultRetryOptions, ...params.retry } : undefined, machine: typeof params.machine === "string" ? { preset: params.machine } : params.machine, maxDuration: params.maxDuration, + ttl: params.ttl, payloadSchema: params.jsonSchema, fns: { run: params.run, @@ -367,6 +368,7 @@ export function createSchemaTask< retry: params.retry ? { ...defaultRetryOptions, ...params.retry } : undefined, machine: typeof params.machine === "string" ? { preset: params.machine } : params.machine, maxDuration: params.maxDuration, + ttl: params.ttl, fns: { run: params.run, parsePayload, diff --git a/references/hello-world/src/trigger/example.ts b/references/hello-world/src/trigger/example.ts index 8ecfdb033ed..5e04f57144f 100644 --- a/references/hello-world/src/trigger/example.ts +++ b/references/hello-world/src/trigger/example.ts @@ -5,6 +5,7 @@ import { fixedLengthTask } from "./batches.js"; export const helloWorldTask = task({ id: "hello-world", + ttl: "10m", retry: { maxAttempts: 3, minTimeoutInMs: 500, diff --git a/references/hello-world/src/trigger/schedule.ts b/references/hello-world/src/trigger/schedule.ts index 29c363d64ca..46a7d9a18ce 100644 --- a/references/hello-world/src/trigger/schedule.ts +++ b/references/hello-world/src/trigger/schedule.ts @@ -3,6 +3,7 @@ import { schedules } from "@trigger.dev/sdk/v3"; export const simpleSchedule = schedules.task({ id: "simple-schedule", cron: "0 0 * * *", + ttl: "30m", run: async (payload, { ctx }) => { return { message: "Hello, world!", diff --git a/references/hello-world/trigger.config.ts b/references/hello-world/trigger.config.ts index 2c3751b041f..e0b875cd6d1 100644 --- a/references/hello-world/trigger.config.ts +++ b/references/hello-world/trigger.config.ts @@ -11,6 +11,7 @@ export default defineConfig({ }, logLevel: "debug", maxDuration: 3600, + ttl: "1h", retries: { enabledInDev: true, default: { From a3407287c9563f9a27a03879be6dbe8baf8e4019 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 31 Mar 2026 10:16:49 +0100 Subject: [PATCH 004/579] feat(engine): enqueue fast path; skip the queue under certain conditions (#3299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Currently, every triggered run follows a two-step path through Redis: 1. **Enqueue** — A Lua script atomically adds the message to a queue sorted set (ordered by priority-adjusted timestamp) 2. **Dequeue** — A debounced `processQueueForWorkerQueue` job fires ~500ms later, checks concurrency limits, removes the message from the sorted set, and pushes it to a worker queue (Redis list) where workers pick it up via `BLPOP` This means every run pays at least ~500ms of latency between being triggered and being available for a worker to execute, even when the queue is empty and concurrency is wide open. ### What changed The enqueue Lua scripts now atomically decide whether to **skip the queue sorted set entirely** and push directly to the worker queue. This happens inside the same Lua script that handles normal enqueue, so the decision is atomic with respect to concurrency bookkeeping. A run takes the **fast path** when all of these are true: - **Fast path is enabled** for this worker queue (gated per `WorkerInstanceGroup`) - **No available messages** in the queue (`ZRANGEBYSCORE` finds nothing with score ≤ now) — this respects priority ordering and allows fast path even when the queue has future-scored messages (e.g. nacked retries with delay) - **Environment concurrency** has capacity - **Queue concurrency** has capacity (including per-concurrency-key limits for CK queues) When the fast path is taken: - The message is stored and pushed directly to the worker queue (`RPUSH`) - Concurrency slots are claimed (`SADD` to the same sets used by the normal dequeue path) - The `processQueueForWorkerQueue` job is **not scheduled** (no work to do) - TTL sorted set is skipped (the `expireRun` worker job handles TTL independently) When any condition fails, the existing slow path runs unchanged. ### Rollout gating - **Development environments**: Fast path is always enabled - **Production environments**: Gated by a new `enableFastPath` boolean on `WorkerInstanceGroup` (defaults to `false`), allowing region-by-region rollout ### Rolling deploy safety Each process registers its own Lua scripts via `defineCommand` (identified by SHA hash). Old and new processes never share scripts. The Redis data structures are fully compatible in both directions — ack, nack, and release operations work identically regardless of which path a message took. ## Test plan - [x] Fast path taken when queue is empty and concurrency available - [x] Slow path when `enableFastPath` is false - [x] Slow path when queue has available messages (respects priority ordering) - [x] Fast path when queue only has future-scored messages - [x] Slow path when env concurrency is full - [x] Fast-path message can be acknowledged correctly - [x] Fast-path message can be nacked and re-enqueued to the queue sorted set - [x] Run all existing run-queue tests (ack, nack, CK, concurrency sweeper, dequeue) to verify no regressions - [x] Typecheck passes for run-engine and webapp --- .server-changes/enqueue-fast-path.md | 6 + .../app/runEngine/concerns/queues.server.ts | 9 +- .../runEngine/services/triggerTask.server.ts | 8 +- apps/webapp/app/runEngine/types.ts | 2 +- .../migration.sql | 2 + .../database/prisma/schema.prisma | 4 + .../run-engine/src/engine/index.ts | 2 + .../src/engine/systems/enqueueSystem.ts | 4 + .../run-engine/src/engine/types.ts | 3 + .../run-engine/src/run-queue/index.ts | 423 ++++++++++++++++-- .../run-queue/tests/enqueueMessage.test.ts | 359 ++++++++++++++- 11 files changed, 762 insertions(+), 60 deletions(-) create mode 100644 .server-changes/enqueue-fast-path.md create mode 100644 internal-packages/database/prisma/migrations/20260330210247_add_enable_fast_path_to_worker_instance_group/migration.sql diff --git a/.server-changes/enqueue-fast-path.md b/.server-changes/enqueue-fast-path.md new file mode 100644 index 00000000000..65ff0dbaca8 --- /dev/null +++ b/.server-changes/enqueue-fast-path.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Reduce run start latency by skipping the intermediate queue when concurrency is available. This optimization is rolled out per-region and enabled automatically for development environments. diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index 4a4dcc8e98f..eb00bf1c586 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -295,9 +295,9 @@ export class DefaultQueueManager implements QueueManager { async getWorkerQueue( environment: AuthenticatedEnvironment, regionOverride?: string - ): Promise { + ): Promise<{ masterQueue: string; enableFastPath: boolean } | undefined> { if (environment.type === "DEVELOPMENT") { - return environment.id; + return { masterQueue: environment.id, enableFastPath: true }; } const workerGroupService = new WorkerGroupService({ @@ -320,7 +320,10 @@ export class DefaultQueueManager implements QueueManager { throw new ServiceValidationError("No worker group found"); } - return workerGroup.masterQueue; + return { + masterQueue: workerGroup.masterQueue, + enableFastPath: workerGroup.enableFastPath, + }; } } diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index d87bfa67db9..c8a6dca2c93 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -298,7 +298,12 @@ export class RunEngineTriggerTaskService { const depth = parentRun ? parentRun.depth + 1 : 0; - const workerQueue = await this.queueConcern.getWorkerQueue(environment, body.options?.region); + const workerQueueResult = await this.queueConcern.getWorkerQueue( + environment, + body.options?.region + ); + const workerQueue = workerQueueResult?.masterQueue; + const enableFastPath = workerQueueResult?.enableFastPath ?? false; // Build annotations for this run const triggerSource = options.triggerSource ?? "api"; @@ -352,6 +357,7 @@ export class RunEngineTriggerTaskService { queue: queueName, lockedQueueId, workerQueue, + enableFastPath, isTest: body.options?.test ?? false, delayUntil, queuedAt: delayUntil ? undefined : new Date(), diff --git a/apps/webapp/app/runEngine/types.ts b/apps/webapp/app/runEngine/types.ts index 310e0b32e5a..d5e61d01889 100644 --- a/apps/webapp/app/runEngine/types.ts +++ b/apps/webapp/app/runEngine/types.ts @@ -70,7 +70,7 @@ export interface QueueManager { getWorkerQueue( env: AuthenticatedEnvironment, regionOverride?: string - ): Promise; + ): Promise<{ masterQueue: string; enableFastPath: boolean } | undefined>; } export interface PayloadProcessor { diff --git a/internal-packages/database/prisma/migrations/20260330210247_add_enable_fast_path_to_worker_instance_group/migration.sql b/internal-packages/database/prisma/migrations/20260330210247_add_enable_fast_path_to_worker_instance_group/migration.sql new file mode 100644 index 00000000000..b7ca1576264 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260330210247_add_enable_fast_path_to_worker_instance_group/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."WorkerInstanceGroup" ADD COLUMN "enableFastPath" BOOLEAN NOT NULL DEFAULT false; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 142fcb6dff7..a0ff9aee690 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1333,6 +1333,10 @@ model WorkerInstanceGroup { workloadType WorkloadType @default(CONTAINER) + /// When true, runs enqueued to this worker queue may skip the intermediate queue + /// and be pushed directly to the worker queue when concurrency is available. + enableFastPath Boolean @default(false) + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt } diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index e459dd85697..a1446d54b20 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -462,6 +462,7 @@ export class RunEngine { cliVersion, concurrencyKey, workerQueue, + enableFastPath, queue, lockedQueueId, isTest, @@ -799,6 +800,7 @@ export class RunEngine { tx: prisma, skipRunLock: true, includeTtl: true, + enableFastPath, }); } catch (enqueueError) { this.logger.error("engine.trigger(): failed to schedule TTL or enqueue run", { diff --git a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts index 9856fa855fc..d899aa7a6f3 100644 --- a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts @@ -36,6 +36,7 @@ export class EnqueueSystem { runnerId, skipRunLock, includeTtl = false, + enableFastPath = false, }: { run: TaskRun; env: MinimalAuthenticatedEnvironment; @@ -57,6 +58,8 @@ export class EnqueueSystem { skipRunLock?: boolean; /** When true, include TTL in the queued message (only for first enqueue from trigger). Default false. */ includeTtl?: boolean; + /** When true, allow the queue to push directly to worker queue if concurrency is available. */ + enableFastPath?: boolean; }) { const prisma = tx ?? this.$.prisma; @@ -98,6 +101,7 @@ export class EnqueueSystem { await this.$.runQueue.enqueueMessage({ env, workerQueue, + enableFastPath, message: { runId: run.id, taskIdentifier: run.taskIdentifier, diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index f30a0fb3796..6ecb726e3af 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -181,6 +181,9 @@ export type TriggerParams = { cliVersion?: string; concurrencyKey?: string; workerQueue?: string; + /** When true, the run queue may push directly to the worker queue if concurrency is available. + * Gated per WorkerInstanceGroup (production) or always true (development). */ + enableFastPath?: boolean; queue: string; lockedQueueId?: string; isTest: boolean; diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 9088099ef1a..de0df73ad05 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -618,16 +618,40 @@ export class RunQueue { ); } + /** + * Enqueue a run message for execution. + * + * The Lua script atomically decides between two paths: + * + * **Fast path** (when `enableFastPath` is true and conditions are met): + * Skips the queue sorted set and pushes directly to the worker queue (Redis list). + * This eliminates the ~500ms debounce delay from the processQueueForWorkerQueue job. + * Conditions: no available messages in the queue, env concurrency available, queue concurrency available. + * + * **Slow path** (default, or when fast-path conditions aren't met): + * Adds to the queue sorted set, then a debounced processQueueForWorkerQueue job moves it + * to the worker queue after checking concurrency limits. + * + * Both paths are atomic within their respective Lua scripts. The fast path claims concurrency + * slots (SADD to currentConcurrency sets) identically to how the dequeue Lua does, so + * ack/nack/release operations work the same regardless of which path was taken. + * + * @param enableFastPath - Gated per WorkerInstanceGroup. Dev environments always true. + * Production regions opt in via the `enableFastPath` flag on WorkerInstanceGroup. + */ public async enqueueMessage({ env, message, workerQueue, skipDequeueProcessing = false, + enableFastPath = false, }: { env: MinimalAuthenticatedEnvironment; message: InputPayload; workerQueue: string; skipDequeueProcessing?: boolean; + /** When true, the Lua script may push directly to the worker queue if concurrency is available. */ + enableFastPath?: boolean; }) { return await this.#trace( "enqueueMessage", @@ -656,8 +680,28 @@ export class RunQueue { attempt: 0, }; - if (!skipDequeueProcessing) { - // This will move the message to the worker queue so it can be dequeued + // Pass TTL info to enqueue so it can be added atomically + const ttlInfo = + message.ttlExpiresAt && this.options.ttlSystem + ? { + ttlExpiresAt: message.ttlExpiresAt, + ttlQueueKey: this.keys.ttlQueueKeyForShard(this.#getTtlShardForQueue(queueKey)), + ttlMember: `${queueKey}|${message.runId}|${message.orgId}`, + } + : undefined; + + // Enqueue the message. The Lua script atomically decides whether to fast-path + // (push directly to worker queue) or slow-path (add to queue sorted set). + const fastPathTaken = await this.#callEnqueueMessage( + messagePayload, + ttlInfo, + enableFastPath + ); + + span.setAttribute("fastPath", fastPathTaken); + + if (!fastPathTaken && !skipDequeueProcessing) { + // Slow path: schedule the dequeue job to move the message from queue to worker queue await this.worker.enqueueOnce({ id: dedupQueueKey, // dedupe by environment and base queue (CK wildcard for CK queues) job: "processQueueForWorkerQueue", @@ -670,18 +714,6 @@ export class RunQueue { availableAt: new Date(Date.now() + (this.options.processWorkerQueueDebounceMs ?? 500)), // 500ms from now }); } - - // Pass TTL info to enqueue so it can be added atomically - const ttlInfo = - message.ttlExpiresAt && this.options.ttlSystem - ? { - ttlExpiresAt: message.ttlExpiresAt, - ttlQueueKey: this.keys.ttlQueueKeyForShard(this.#getTtlShardForQueue(queueKey)), - ttlMember: `${queueKey}|${message.runId}|${message.orgId}`, - } - : undefined; - - await this.#callEnqueueMessage(messagePayload, ttlInfo); }, { kind: SpanKind.PRODUCER, @@ -1711,14 +1743,38 @@ export class RunQueue { }); } + /** + * Calls the appropriate Lua enqueue script variant (plain, TTL, CK, or TTL+CK). + * + * Each variant receives the same fast-path keys/args in addition to its own specific keys. + * The Lua script returns 1 (fast path taken) or 0 (slow path taken). + * + * All four variants share this KEYS layout: + * KEYS[1..N] = variant-specific keys (queue, message, concurrency, env, master, TTL, CK index) + * KEYS[N+1] = workerQueueKey (fast-path: RPUSH target) + * KEYS[N+2] = queueConcurrencyLimitKey + * KEYS[N+3] = envConcurrencyLimitKey + * KEYS[N+4] = envConcurrencyLimitBurstFactorKey + * + * And this ARGV layout (appended after variant-specific args): + * messageKeyValue (fast-path: value to RPUSH into worker queue) + * defaultEnvConcurrencyLimit (fallback if no Redis key set) + * defaultEnvConcurrencyBurstFactor + * currentTime (ms timestamp for ZRANGEBYSCORE availability check) + * enableFastPath ('1' or '0') + * + * @returns true if the fast path was taken (message pushed directly to worker queue) + */ async #callEnqueueMessage( message: OutputPayloadV2, ttlInfo?: { ttlExpiresAt: number; ttlQueueKey: string; ttlMember: string; - } - ) { + }, + enableFastPath: boolean = false + ): Promise { + // --- Slow-path keys (used by all variants) --- const queueKey = message.queue; const messageKey = this.keys.messageKey(message.orgId, message.runId); const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKeyFromQueue(message.queue); @@ -1731,10 +1787,25 @@ export class RunQueue { this.shardCount ); + // --- Fast-path keys (appended to each variant's KEYS) --- + const workerQueueKey = this.keys.workerQueueKey(message.workerQueue); + const queueConcurrencyLimitKey = this.keys.queueConcurrencyLimitKeyFromQueue(message.queue); + const envConcurrencyLimitKey = this.keys.envConcurrencyLimitKeyFromQueue(message.queue); + const envConcurrencyLimitBurstFactorKey = + this.keys.envConcurrencyLimitBurstFactorKeyFromQueue(message.queue); + // The value stored in the worker queue list — used to look up the message payload on dequeue + const messageKeyValue = messageKey; + const queueName = message.queue; const messageId = message.runId; const messageData = JSON.stringify(message); const messageScore = String(message.timestamp); + const currentTime = String(Date.now()); + const enableFastPathArg = enableFastPath ? "1" : "0"; + const defaultEnvConcurrencyLimit = String(this.options.defaultEnvConcurrency); + const defaultEnvConcurrencyBurstFactor = String( + this.options.defaultEnvConcurrencyBurstFactor ?? 1.0 + ); this.logger.debug("Calling enqueueMessage", { queueKey, @@ -1749,17 +1820,21 @@ export class RunQueue { messageData, messageScore, masterQueueKey, + enableFastPath, ttlInfo, service: this.name, }); + let result: number; + // Use CK-aware enqueue for messages with concurrency keys if (message.concurrencyKey) { const ckIndexKey = this.keys.ckIndexKeyFromQueue(message.queue); const ckWildcardName = this.keys.toCkWildcard(message.queue); if (ttlInfo) { - await this.redis.enqueueMessageWithTtlCk( + result = await this.redis.enqueueMessageWithTtlCk( + // keys masterQueueKey, queueKey, messageKey, @@ -1770,16 +1845,27 @@ export class RunQueue { envQueueKey, ttlInfo.ttlQueueKey, ckIndexKey, + workerQueueKey, + queueConcurrencyLimitKey, + envConcurrencyLimitKey, + envConcurrencyLimitBurstFactorKey, + // args queueName, messageId, messageData, messageScore, ttlInfo.ttlMember, String(ttlInfo.ttlExpiresAt), - ckWildcardName + ckWildcardName, + messageKeyValue, + defaultEnvConcurrencyLimit, + defaultEnvConcurrencyBurstFactor, + currentTime, + enableFastPathArg ); } else { - await this.redis.enqueueMessageCk( + result = await this.redis.enqueueMessageCk( + // keys masterQueueKey, queueKey, messageKey, @@ -1789,16 +1875,27 @@ export class RunQueue { envCurrentDequeuedKey, envQueueKey, ckIndexKey, + workerQueueKey, + queueConcurrencyLimitKey, + envConcurrencyLimitKey, + envConcurrencyLimitBurstFactorKey, + // args queueName, messageId, messageData, messageScore, - ckWildcardName + ckWildcardName, + messageKeyValue, + defaultEnvConcurrencyLimit, + defaultEnvConcurrencyBurstFactor, + currentTime, + enableFastPathArg ); } } else if (ttlInfo) { // Use the TTL-aware enqueue that atomically adds to both queues - await this.redis.enqueueMessageWithTtl( + result = await this.redis.enqueueMessageWithTtl( + // keys masterQueueKey, queueKey, messageKey, @@ -1808,15 +1905,26 @@ export class RunQueue { envCurrentDequeuedKey, envQueueKey, ttlInfo.ttlQueueKey, + workerQueueKey, + queueConcurrencyLimitKey, + envConcurrencyLimitKey, + envConcurrencyLimitBurstFactorKey, + // args queueName, messageId, messageData, messageScore, ttlInfo.ttlMember, - String(ttlInfo.ttlExpiresAt) + String(ttlInfo.ttlExpiresAt), + messageKeyValue, + defaultEnvConcurrencyLimit, + defaultEnvConcurrencyBurstFactor, + currentTime, + enableFastPathArg ); } else { - await this.redis.enqueueMessage( + result = await this.redis.enqueueMessage( + // keys masterQueueKey, queueKey, messageKey, @@ -1825,12 +1933,24 @@ export class RunQueue { queueCurrentDequeuedKey, envCurrentDequeuedKey, envQueueKey, + workerQueueKey, + queueConcurrencyLimitKey, + envConcurrencyLimitKey, + envConcurrencyLimitBurstFactorKey, + // args queueName, messageId, messageData, - messageScore + messageScore, + messageKeyValue, + defaultEnvConcurrencyLimit, + defaultEnvConcurrencyBurstFactor, + currentTime, + enableFastPathArg ); } + + return result === 1; } async #callDequeueMessagesFromQueue({ @@ -2794,8 +2914,24 @@ end `, }); + // Enqueue message with optional fast path. + // + // Returns 1 (fast path: message pushed directly to worker queue) or 0 (slow path: message + // added to queue sorted set, needs processQueueForWorkerQueue to move it later). + // + // Fast-path conditions (all must be true): + // 1. enableFastPath == '1' (gated per WorkerInstanceGroup) + // 2. No available messages in the queue (ZRANGEBYSCORE finds nothing with score <= now) + // 3. Env concurrency has capacity (SCARD < limit * burstFactor) + // 4. Queue concurrency has capacity (SCARD < min(queueLimit, envLimit)) + // + // The fast path performs the same concurrency bookkeeping (SADD to currentConcurrency sets) + // as dequeueMessagesFromQueue, so ack/nack/release work identically for both paths. + // + // When enableFastPath == '0', the script skips the fast-path check entirely and behaves + // identically to the pre-fast-path version (with the addition of returning 0). this.redis.defineCommand("enqueueMessage", { - numberOfKeys: 8, + numberOfKeys: 12, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -2805,12 +2941,51 @@ local envCurrentConcurrencyKey = KEYS[5] local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] +-- Fast-path keys (KEYS 9-12) +local workerQueueKey = KEYS[9] +local queueConcurrencyLimitKey = KEYS[10] +local envConcurrencyLimitKey = KEYS[11] +local envConcurrencyLimitBurstFactorKey = KEYS[12] local queueName = ARGV[1] local messageId = ARGV[2] local messageData = ARGV[3] local messageScore = ARGV[4] +-- Fast-path args (ARGV 5-9) +local messageKeyValue = ARGV[5] +local defaultEnvConcurrencyLimit = ARGV[6] +local defaultEnvConcurrencyBurstFactor = ARGV[7] +local currentTime = ARGV[8] +local enableFastPath = ARGV[9] + +-- Fast path: check if we can skip the queue and go directly to worker queue +if enableFastPath == '1' then + local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) + if #available == 0 then + local envCurrent = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') + local envLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + local envBurstFactor = tonumber(redis.call('GET', envConcurrencyLimitBurstFactorKey) or defaultEnvConcurrencyBurstFactor) + local envLimitWithBurst = math.floor(envLimit * envBurstFactor) + + if envCurrent < envLimitWithBurst then + local queueCurrent = tonumber(redis.call('SCARD', queueCurrentConcurrencyKey) or '0') + local queueLimit = math.min( + tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), + envLimit + ) + + if queueCurrent < queueLimit then + redis.call('SET', messageKey, messageData) + redis.call('SADD', queueCurrentConcurrencyKey, messageId) + redis.call('SADD', envCurrentConcurrencyKey, messageId) + redis.call('RPUSH', workerQueueKey, messageKeyValue) + return 1 + end + end + end +end +-- Slow path: normal enqueue -- Write the message to the message key redis.call('SET', messageKey, messageData) @@ -2834,12 +3009,17 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) + +return 0 `, }); - // Enqueue with TTL tracking - atomically adds to both normal queue and TTL sorted set + // Enqueue with TTL tracking. Same fast-path logic as enqueueMessage. + // On fast path, the TTL sorted set is intentionally skipped — the expireRun worker job + // (scheduled independently before enqueue) handles TTL expiry. This mirrors what + // dequeueMessagesFromQueue does: it removes from the TTL set when dequeuing. this.redis.defineCommand("enqueueMessageWithTtl", { - numberOfKeys: 9, + numberOfKeys: 13, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -2850,6 +3030,11 @@ local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local ttlQueueKey = KEYS[9] +-- Fast-path keys (KEYS 10-13) +local workerQueueKey = KEYS[10] +local queueConcurrencyLimitKey = KEYS[11] +local envConcurrencyLimitKey = KEYS[12] +local envConcurrencyLimitBurstFactorKey = KEYS[13] local queueName = ARGV[1] local messageId = ARGV[2] @@ -2857,7 +3042,42 @@ local messageData = ARGV[3] local messageScore = ARGV[4] local ttlMember = ARGV[5] local ttlScore = ARGV[6] +-- Fast-path args (ARGV 7-11) +local messageKeyValue = ARGV[7] +local defaultEnvConcurrencyLimit = ARGV[8] +local defaultEnvConcurrencyBurstFactor = ARGV[9] +local currentTime = ARGV[10] +local enableFastPath = ARGV[11] + +-- Fast path: check if we can skip the queue and go directly to worker queue +if enableFastPath == '1' then + local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) + if #available == 0 then + local envCurrent = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') + local envLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + local envBurstFactor = tonumber(redis.call('GET', envConcurrencyLimitBurstFactorKey) or defaultEnvConcurrencyBurstFactor) + local envLimitWithBurst = math.floor(envLimit * envBurstFactor) + + if envCurrent < envLimitWithBurst then + local queueCurrent = tonumber(redis.call('SCARD', queueCurrentConcurrencyKey) or '0') + local queueLimit = math.min( + tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), + envLimit + ) + + if queueCurrent < queueLimit then + redis.call('SET', messageKey, messageData) + redis.call('SADD', queueCurrentConcurrencyKey, messageId) + redis.call('SADD', envCurrentConcurrencyKey, messageId) + redis.call('RPUSH', workerQueueKey, messageKeyValue) + -- Skip TTL sorted set: the expireRun worker job handles TTL expiry independently + return 1 + end + end + end +end +-- Slow path: normal enqueue -- Write the message to the message key redis.call('SET', messageKey, messageData) @@ -2884,12 +3104,16 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) + +return 0 `, }); - // CK-aware enqueue: adds to CK index + master queue with :ck:* member + // CK-aware enqueue: adds to CK index + master queue with :ck:* member. + // Same fast-path logic as enqueueMessage. For CK queues, the fast-path checks the + // per-CK sub-queue for available messages and the per-CK concurrency limit. this.redis.defineCommand("enqueueMessageCk", { - numberOfKeys: 9, + numberOfKeys: 13, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -2900,13 +3124,53 @@ local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local ckIndexKey = KEYS[9] +-- Fast-path keys (KEYS 10-13) +local workerQueueKey = KEYS[10] +local queueConcurrencyLimitKey = KEYS[11] +local envConcurrencyLimitKey = KEYS[12] +local envConcurrencyLimitBurstFactorKey = KEYS[13] local queueName = ARGV[1] local messageId = ARGV[2] local messageData = ARGV[3] local messageScore = ARGV[4] local ckWildcardName = ARGV[5] +-- Fast-path args (ARGV 6-10) +local messageKeyValue = ARGV[6] +local defaultEnvConcurrencyLimit = ARGV[7] +local defaultEnvConcurrencyBurstFactor = ARGV[8] +local currentTime = ARGV[9] +local enableFastPath = ARGV[10] + +-- Fast path: check if we can skip the queue and go directly to worker queue +if enableFastPath == '1' then + local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) + if #available == 0 then + local envCurrent = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') + local envLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + local envBurstFactor = tonumber(redis.call('GET', envConcurrencyLimitBurstFactorKey) or defaultEnvConcurrencyBurstFactor) + local envLimitWithBurst = math.floor(envLimit * envBurstFactor) + + if envCurrent < envLimitWithBurst then + -- For CK queues, check per-CK concurrency (same key as queue concurrency) + local queueCurrent = tonumber(redis.call('SCARD', queueCurrentConcurrencyKey) or '0') + local queueLimit = math.min( + tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), + envLimit + ) + + if queueCurrent < queueLimit then + redis.call('SET', messageKey, messageData) + redis.call('SADD', queueCurrentConcurrencyKey, messageId) + redis.call('SADD', envCurrentConcurrencyKey, messageId) + redis.call('RPUSH', workerQueueKey, messageKeyValue) + return 1 + end + end + end +end +-- Slow path: normal enqueue -- Write the message to the message key redis.call('SET', messageKey, messageData) @@ -2936,12 +3200,15 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) + +return 0 `, }); - // CK-aware enqueue with TTL tracking + // CK-aware enqueue with TTL tracking. Combines CK and TTL behavior with fast path. + // On fast path: skips both the CK sub-queue and TTL sorted set. this.redis.defineCommand("enqueueMessageWithTtlCk", { - numberOfKeys: 10, + numberOfKeys: 14, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -2953,6 +3220,11 @@ local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local ttlQueueKey = KEYS[9] local ckIndexKey = KEYS[10] +-- Fast-path keys (KEYS 11-14) +local workerQueueKey = KEYS[11] +local queueConcurrencyLimitKey = KEYS[12] +local envConcurrencyLimitKey = KEYS[13] +local envConcurrencyLimitBurstFactorKey = KEYS[14] local queueName = ARGV[1] local messageId = ARGV[2] @@ -2961,7 +3233,42 @@ local messageScore = ARGV[4] local ttlMember = ARGV[5] local ttlScore = ARGV[6] local ckWildcardName = ARGV[7] +-- Fast-path args (ARGV 8-12) +local messageKeyValue = ARGV[8] +local defaultEnvConcurrencyLimit = ARGV[9] +local defaultEnvConcurrencyBurstFactor = ARGV[10] +local currentTime = ARGV[11] +local enableFastPath = ARGV[12] + +-- Fast path: check if we can skip the queue and go directly to worker queue +if enableFastPath == '1' then + local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) + if #available == 0 then + local envCurrent = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') + local envLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + local envBurstFactor = tonumber(redis.call('GET', envConcurrencyLimitBurstFactorKey) or defaultEnvConcurrencyBurstFactor) + local envLimitWithBurst = math.floor(envLimit * envBurstFactor) + + if envCurrent < envLimitWithBurst then + local queueCurrent = tonumber(redis.call('SCARD', queueCurrentConcurrencyKey) or '0') + local queueLimit = math.min( + tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), + envLimit + ) + + if queueCurrent < queueLimit then + redis.call('SET', messageKey, messageData) + redis.call('SADD', queueCurrentConcurrencyKey, messageId) + redis.call('SADD', envCurrentConcurrencyKey, messageId) + redis.call('RPUSH', workerQueueKey, messageKeyValue) + -- Skip TTL sorted set: the expireRun worker job handles TTL expiry independently + return 1 + end + end + end +end +-- Slow path: normal enqueue -- Write the message to the message key redis.call('SET', messageKey, messageData) @@ -2994,6 +3301,8 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) + +return 0 `, }); @@ -3870,13 +4179,22 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, envQueueKey: string, + workerQueueKey: string, + queueConcurrencyLimitKey: string, + envConcurrencyLimitKey: string, + envConcurrencyLimitBurstFactorKey: string, //args queueName: string, messageId: string, messageData: string, messageScore: string, - callback?: Callback - ): Result; + messageKeyValue: string, + defaultEnvConcurrencyLimit: string, + defaultEnvConcurrencyBurstFactor: string, + currentTime: string, + enableFastPath: string, + callback?: Callback + ): Result; enqueueMessageWithTtl( //keys @@ -3889,6 +4207,10 @@ declare module "@internal/redis" { envCurrentDequeuedKey: string, envQueueKey: string, ttlQueueKey: string, + workerQueueKey: string, + queueConcurrencyLimitKey: string, + envConcurrencyLimitKey: string, + envConcurrencyLimitBurstFactorKey: string, //args queueName: string, messageId: string, @@ -3896,8 +4218,13 @@ declare module "@internal/redis" { messageScore: string, ttlMember: string, ttlScore: string, - callback?: Callback - ): Result; + messageKeyValue: string, + defaultEnvConcurrencyLimit: string, + defaultEnvConcurrencyBurstFactor: string, + currentTime: string, + enableFastPath: string, + callback?: Callback + ): Result; expireTtlRuns( //keys @@ -4057,14 +4384,23 @@ declare module "@internal/redis" { envCurrentDequeuedKey: string, envQueueKey: string, ckIndexKey: string, + workerQueueKey: string, + queueConcurrencyLimitKey: string, + envConcurrencyLimitKey: string, + envConcurrencyLimitBurstFactorKey: string, //args queueName: string, messageId: string, messageData: string, messageScore: string, ckWildcardName: string, - callback?: Callback - ): Result; + messageKeyValue: string, + defaultEnvConcurrencyLimit: string, + defaultEnvConcurrencyBurstFactor: string, + currentTime: string, + enableFastPath: string, + callback?: Callback + ): Result; enqueueMessageWithTtlCk( //keys @@ -4078,6 +4414,10 @@ declare module "@internal/redis" { envQueueKey: string, ttlQueueKey: string, ckIndexKey: string, + workerQueueKey: string, + queueConcurrencyLimitKey: string, + envConcurrencyLimitKey: string, + envConcurrencyLimitBurstFactorKey: string, //args queueName: string, messageId: string, @@ -4086,8 +4426,13 @@ declare module "@internal/redis" { ttlMember: string, ttlScore: string, ckWildcardName: string, - callback?: Callback - ): Result; + messageKeyValue: string, + defaultEnvConcurrencyLimit: string, + defaultEnvConcurrencyBurstFactor: string, + currentTime: string, + enableFastPath: string, + callback?: Callback + ): Result; dequeueMessagesFromCkQueue( //keys diff --git a/internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts b/internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts index bf4ed87f295..8ce2d68d5de 100644 --- a/internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts @@ -1,7 +1,7 @@ import { assertNonNullable, redisTest } from "@internal/testcontainers"; import { trace } from "@internal/tracing"; import { Logger } from "@trigger.dev/core/logger"; -import { describe } from "node:test"; +import { describe } from "vitest"; import { setTimeout } from "node:timers/promises"; import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; import { RunQueue } from "../index.js"; @@ -39,7 +39,7 @@ const messageDev: InputPayload = { taskIdentifier: "task/my-task", orgId: "o1234", projectId: "p1234", - environmentId: "e4321", + environmentId: "e1234", environmentType: "DEVELOPMENT", queue: "task/my-task", timestamp: Date.now(), @@ -48,24 +48,28 @@ const messageDev: InputPayload = { vi.setConfig({ testTimeout: 60_000 }); -describe("RunQueue.enqueueMessage", () => { - redisTest("should add the message to the queue", async ({ redisContainer }) => { - const queue = new RunQueue({ - ...testOptions, - queueSelectionStrategy: new FairQueueSelectionStrategy({ - redis: { - keyPrefix: "runqueue:test:", - host: redisContainer.getHost(), - port: redisContainer.getPort(), - }, - keys: testOptions.keys, - }), +function createQueue(redisContainer: { getHost: () => string; getPort: () => number }, prefix = "runqueue:test:") { + return new RunQueue({ + ...testOptions, + queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: { - keyPrefix: "runqueue:test:", + keyPrefix: prefix, host: redisContainer.getHost(), port: redisContainer.getPort(), }, - }); + keys: testOptions.keys, + }), + redis: { + keyPrefix: prefix, + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + }); +} + +describe("RunQueue.enqueueMessage", () => { + redisTest("should add the message to the queue", async ({ redisContainer }) => { + const queue = createQueue(redisContainer); try { //initial queue length @@ -127,3 +131,326 @@ describe("RunQueue.enqueueMessage", () => { } }); }); + +describe("RunQueue.enqueueMessage fast path", () => { + redisTest("should fast-path to worker queue when queue is empty and concurrency available", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, "runqueue:fp1:"); + + try { + // Set concurrency limits + await queue.updateEnvConcurrencyLimits(authenticatedEnvDev); + + // Enqueue with fast path enabled + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: messageDev, + workerQueue: authenticatedEnvDev.id, + enableFastPath: true, + }); + + // Queue sorted set should be empty (fast path skips it) + const queueLength = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(queueLength).toBe(0); + + // Queue concurrency should be claimed (operational concurrency) + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(1); + + // Message should be directly in worker queue - dequeue it + const dequeued = await queue.dequeueMessageFromWorkerQueue( + "test_12345", + authenticatedEnvDev.id, + { blockingPop: false } + ); + assertNonNullable(dequeued); + expect(dequeued.messageId).toEqual(messageDev.runId); + expect(dequeued.message.version).toEqual("2"); + } finally { + await queue.quit(); + } + }); + + redisTest("should take slow path when enableFastPath is false", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, "runqueue:fp2:"); + + try { + await queue.updateEnvConcurrencyLimits(authenticatedEnvDev); + + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: messageDev, + workerQueue: authenticatedEnvDev.id, + enableFastPath: false, + }); + + // Message should be in the queue sorted set (slow path) + const queueLength = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(queueLength).toBe(1); + + // No concurrency claimed yet + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(0); + } finally { + await queue.quit(); + } + }); + + redisTest("should take slow path when queue has available messages", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, "runqueue:fp3:"); + + try { + await queue.updateEnvConcurrencyLimits(authenticatedEnvDev); + + // Enqueue a first message (slow path to populate the queue) + const message1: InputPayload = { + ...messageDev, + runId: "r1111", + timestamp: Date.now() - 1000, // in the past, so it's "available" + }; + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: message1, + workerQueue: authenticatedEnvDev.id, + enableFastPath: false, + }); + + // Now enqueue a second message with fast path + const message2: InputPayload = { + ...messageDev, + runId: "r2222", + timestamp: Date.now(), + }; + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: message2, + workerQueue: authenticatedEnvDev.id, + enableFastPath: true, + }); + + // Both messages should be in the queue sorted set (slow path for both) + const queueLength = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(queueLength).toBe(2); + } finally { + await queue.quit(); + } + }); + + redisTest("should fast-path when queue only has future-scored messages", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, "runqueue:fp4:"); + + try { + await queue.updateEnvConcurrencyLimits(authenticatedEnvDev); + + // Enqueue a message with a future timestamp (simulating a nacked retry) + const futureMessage: InputPayload = { + ...messageDev, + runId: "r_future", + timestamp: Date.now() + 60_000, // 60 seconds in the future + }; + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: futureMessage, + workerQueue: authenticatedEnvDev.id, + enableFastPath: false, + }); + + // Queue has 1 message but it's not available (future score) + const queueLength = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(queueLength).toBe(1); + + // Now enqueue a new message with fast path + const newMessage: InputPayload = { + ...messageDev, + runId: "r_new", + timestamp: Date.now(), + }; + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: newMessage, + workerQueue: authenticatedEnvDev.id, + enableFastPath: true, + }); + + // The future message stays in queue, new message went to worker queue + const queueLength2 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(queueLength2).toBe(1); // Only the future message + + // Queue concurrency claimed for the fast-pathed message + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(1); + + // Can dequeue the fast-pathed message from worker queue + const dequeued = await queue.dequeueMessageFromWorkerQueue( + "test_12345", + authenticatedEnvDev.id, + { blockingPop: false } + ); + assertNonNullable(dequeued); + expect(dequeued.messageId).toEqual("r_new"); + } finally { + await queue.quit(); + } + }); + + redisTest("should take slow path when env concurrency is full", async ({ redisContainer }) => { + // Use a low concurrency limit + const lowConcurrencyEnv = { + ...authenticatedEnvDev, + maximumConcurrencyLimit: 1, + concurrencyLimitBurstFactor: new Decimal(1.0), + }; + + const queue = createQueue(redisContainer, "runqueue:fp5:"); + + try { + await queue.updateEnvConcurrencyLimits(lowConcurrencyEnv); + + // First message takes fast path + const message1: InputPayload = { + ...messageDev, + runId: "r_first", + timestamp: Date.now(), + }; + await queue.enqueueMessage({ + env: lowConcurrencyEnv, + message: message1, + workerQueue: lowConcurrencyEnv.id, + enableFastPath: true, + }); + + // Queue concurrency is now 1 (fast path claimed it) + const queueConcurrency = await queue.currentConcurrencyOfQueue( + lowConcurrencyEnv, + messageDev.queue + ); + expect(queueConcurrency).toBe(1); + + // Second message should take slow path (env concurrency full) + const message2: InputPayload = { + ...messageDev, + runId: "r_second", + timestamp: Date.now(), + }; + await queue.enqueueMessage({ + env: lowConcurrencyEnv, + message: message2, + workerQueue: lowConcurrencyEnv.id, + enableFastPath: true, + }); + + // Second message should be in queue sorted set + const queueLength = await queue.lengthOfQueue(lowConcurrencyEnv, messageDev.queue); + expect(queueLength).toBe(1); + + // Queue concurrency unchanged (still 1 from first message) + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + lowConcurrencyEnv, + messageDev.queue + ); + expect(queueConcurrency2).toBe(1); + } finally { + await queue.quit(); + } + }); + + redisTest("fast-path message can be acknowledged correctly", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, "runqueue:fp6:"); + + try { + await queue.updateEnvConcurrencyLimits(authenticatedEnvDev); + + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: messageDev, + workerQueue: authenticatedEnvDev.id, + enableFastPath: true, + }); + + // Verify fast path was taken + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(1); + + // Dequeue from worker queue + const dequeued = await queue.dequeueMessageFromWorkerQueue( + "test_12345", + authenticatedEnvDev.id, + { blockingPop: false } + ); + assertNonNullable(dequeued); + + // Acknowledge the message + await queue.acknowledgeMessage(messageDev.orgId, dequeued.messageId); + + // Queue concurrency should be released + const queueConcurrencyAfter = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrencyAfter).toBe(0); + } finally { + await queue.quit(); + } + }); + + redisTest("fast-path message can be nacked and re-enqueued", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, "runqueue:fp7:"); + + try { + await queue.updateEnvConcurrencyLimits(authenticatedEnvDev); + + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: messageDev, + workerQueue: authenticatedEnvDev.id, + enableFastPath: true, + }); + + // Verify fast path was taken + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(1); + + // Dequeue from worker queue + const dequeued = await queue.dequeueMessageFromWorkerQueue( + "test_12345", + authenticatedEnvDev.id, + { blockingPop: false } + ); + assertNonNullable(dequeued); + + // Nack the message (re-enqueue it) + await queue.nackMessage({ + orgId: messageDev.orgId, + messageId: dequeued.messageId, + retryAt: Date.now() + 1000, + }); + + // Queue concurrency should be released + const queueConcurrencyAfter = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrencyAfter).toBe(0); + + // Message should now be in the queue sorted set + const queueLength = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(queueLength).toBe(1); + } finally { + await queue.quit(); + } + }); +}); From 1307d97f7a324dcd66680adf824306bb1fed5ba0 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 31 Mar 2026 12:53:05 +0200 Subject: [PATCH 005/579] fix(webapp): match environment when searching env variables (#3302) Temporary workaround that enables filtering by environment in the envvars page, without changing any UI. --------- Co-authored-by: Claude --- .server-changes/env-variables-search-by-environment.md | 6 ++++++ apps/webapp/app/hooks/useFuzzyFilter.ts | 4 ++-- .../route.tsx | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .server-changes/env-variables-search-by-environment.md diff --git a/.server-changes/env-variables-search-by-environment.md b/.server-changes/env-variables-search-by-environment.md new file mode 100644 index 00000000000..c3f9ed8bc2a --- /dev/null +++ b/.server-changes/env-variables-search-by-environment.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Extended the search filter on the environment variables page to match on environment type (production, staging, development, preview) and branch name, not just variable name and value. diff --git a/apps/webapp/app/hooks/useFuzzyFilter.ts b/apps/webapp/app/hooks/useFuzzyFilter.ts index 1c0f6048268..3f0797179f2 100644 --- a/apps/webapp/app/hooks/useFuzzyFilter.ts +++ b/apps/webapp/app/hooks/useFuzzyFilter.ts @@ -8,7 +8,7 @@ import { matchSorter } from "match-sorter"; * * @param params - The parameters object * @param params.items - Array of objects to filter - * @param params.keys - Array of object keys to perform the fuzzy search on + * @param params.keys - Array of object keys to perform the fuzzy search on (supports dot-notation for nested properties) * @returns An object containing: * - filterText: The current filter text * - setFilterText: Function to update the filter text @@ -28,7 +28,7 @@ export function useFuzzyFilter({ keys, }: { items: T[]; - keys: Extract[]; + keys: (Extract | (string & {}))[]; }) { const [filterText, setFilterText] = useState(""); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index 2670f0188df..f7f91f33274 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -256,7 +256,7 @@ export default function Page() { const { filterText, setFilterText, filteredItems } = useFuzzyFilter({ items: environmentVariables, - keys: ["key", "value"], + keys: ["key", "value", "environment.type", "environment.branchName"], }); // Add isFirst and isLast to each environment variable From 2ba77d89ddb5657718a5c848db23e1cee7ec6ff4 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 31 Mar 2026 13:23:46 +0100 Subject: [PATCH 006/579] fix: add build step to @internal/compute package (#3303) The @internal/compute package had its main/types pointing to ./src/index.ts with no build step. This works in dev (tsc resolves .ts at compile time) but fails at runtime in Docker because Node.js can't load .ts files directly. Added tsconfig.build.json and build/clean/dev scripts matching the pattern used by schedule-engine and other internal packages. Exports now point to dist/. --- internal-packages/compute/package.json | 20 +++++++++++++++--- internal-packages/compute/tsconfig.build.json | 21 +++++++++++++++++++ pnpm-lock.yaml | 16 +++++++------- 3 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 internal-packages/compute/tsconfig.build.json diff --git a/internal-packages/compute/package.json b/internal-packages/compute/package.json index 004d29990f3..4e221879382 100644 --- a/internal-packages/compute/package.json +++ b/internal-packages/compute/package.json @@ -2,13 +2,27 @@ "name": "@internal/compute", "private": true, "version": "0.0.1", - "main": "./src/index.ts", - "types": "./src/index.ts", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", "type": "module", + "exports": { + ".": { + "@triggerdotdev/source": "./src/index.ts", + "import": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "default": "./dist/src/index.js" + } + }, "dependencies": { "zod": "3.25.76" }, + "devDependencies": { + "rimraf": "6.0.1" + }, "scripts": { - "typecheck": "tsc --noEmit" + "clean": "rimraf dist", + "typecheck": "tsc --noEmit", + "build": "pnpm run clean && tsc -p tsconfig.build.json", + "dev": "tsc --watch -p tsconfig.build.json" } } diff --git a/internal-packages/compute/tsconfig.build.json b/internal-packages/compute/tsconfig.build.json new file mode 100644 index 00000000000..89c87a3dc67 --- /dev/null +++ b/internal-packages/compute/tsconfig.build.json @@ -0,0 +1,21 @@ +{ + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"], + "compilerOptions": { + "composite": true, + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + "outDir": "dist", + "module": "Node16", + "moduleResolution": "Node16", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "strict": true, + "declaration": true + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d460bdb1096..13d33a7df42 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1095,6 +1095,10 @@ importers: zod: specifier: 3.25.76 version: 3.25.76 + devDependencies: + rimraf: + specifier: 6.0.1 + version: 6.0.1 internal-packages/database: dependencies: @@ -15592,10 +15596,6 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.0.0: - resolution: {integrity: sha512-Qv32eSV1RSCfhY3fpPE2GNZ8jgM9X7rdAfemLWqTUxwiyIC4jJ6Sy0fZ8H+oLWevO6i4/bizg7c8d8i6bxrzbA==} - engines: {node: 20 || >=22} - lru-cache@11.2.4: resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} engines: {node: 20 || >=22} @@ -23269,7 +23269,7 @@ snapshots: '@epic-web/test-server@0.1.0(bufferutil@4.0.9)': dependencies: '@hono/node-server': 1.12.2(hono@4.5.11) - '@hono/node-ws': 1.0.4(@hono/node-server@1.12.2(hono@4.11.8))(bufferutil@4.0.9) + '@hono/node-ws': 1.0.4(@hono/node-server@1.12.2(hono@4.5.11))(bufferutil@4.0.9) '@open-draft/deferred-promise': 2.2.0 '@types/ws': 8.5.12 hono: 4.5.11 @@ -24024,7 +24024,7 @@ snapshots: dependencies: hono: 4.11.8 - '@hono/node-ws@1.0.4(@hono/node-server@1.12.2(hono@4.11.8))(bufferutil@4.0.9)': + '@hono/node-ws@1.0.4(@hono/node-server@1.12.2(hono@4.5.11))(bufferutil@4.0.9)': dependencies: '@hono/node-server': 1.12.2(hono@4.5.11) ws: 8.18.3(bufferutil@4.0.9) @@ -36714,8 +36714,6 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.0.0: {} - lru-cache@11.2.4: {} lru-cache@4.1.5: @@ -38546,7 +38544,7 @@ snapshots: path-scurry@2.0.0: dependencies: - lru-cache: 11.0.0 + lru-cache: 11.2.4 minipass: 7.1.2 path-to-regexp@0.1.10: {} From 0977c56efee3abbd248c7d03eb9a42ba9fb825aa Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 31 Mar 2026 19:06:54 +0100 Subject: [PATCH 007/579] Errors (versions) (#3187) - Added versions filtering on the Errors list and page - Added errors stacked bars to the graph on the individual error page --------- Co-authored-by: James Ritchie --- .../webapp/app/assets/icons/SlackMonoIcon.tsx | 10 + .../errors/ConfigureErrorAlerts.tsx | 365 +++++++++ .../components/errors/ErrorStatusBadge.tsx | 34 + .../app/components/errors/ErrorStatusMenu.tsx | 250 ++++++ .../app/components/logs/LogsVersionFilter.tsx | 58 ++ .../OrganizationSettingsSideMenu.tsx | 9 + .../app/components/navigation/SideMenu.tsx | 9 +- .../components/navigation/SideMenuItem.tsx | 5 +- .../app/components/primitives/DateTime.tsx | 16 +- .../app/components/primitives/Popover.tsx | 25 +- .../app/components/primitives/Table.tsx | 14 +- .../app/components/primitives/Toast.tsx | 28 +- .../components/primitives/UnorderedList.tsx | 129 ++++ .../components/primitives/charts/ChartBar.tsx | 1 + .../primitives/charts/ChartLegendCompound.tsx | 6 +- .../primitives/charts/ChartRoot.tsx | 7 + .../app/components/runs/v3/EnabledStatus.tsx | 6 +- .../app/components/runs/v3/RunFilters.tsx | 2 +- apps/webapp/app/models/projectAlert.server.ts | 6 + .../v3/ApiAlertChannelPresenter.server.ts | 5 + .../v3/ErrorAlertChannelPresenter.server.ts | 73 ++ .../v3/ErrorGroupPresenter.server.ts | 163 +++- .../v3/ErrorsListPresenter.server.ts | 182 ++++- .../v3/NewAlertChannelPresenter.server.ts | 1 + ...v.$envParam.alerts.new.connect-to-slack.ts | 1 + .../route.tsx | 16 +- .../route.tsx | 17 +- .../route.tsx | 719 ++++++++++++++---- .../route.tsx | 325 +++++++- ...m.env.$envParam.errors.connect-to-slack.ts | 48 ++ .../route.tsx | 199 ++++- ...zationSlug.settings.integrations.slack.tsx | 259 ++++--- .../routes/storybook.unordered-list/route.tsx | 67 ++ apps/webapp/app/routes/storybook/route.tsx | 4 + apps/webapp/app/utils/pathBuilder.ts | 8 + apps/webapp/app/v3/alertsWorker.server.ts | 48 +- apps/webapp/app/v3/otlpExporter.server.ts | 2 +- .../alerts/createAlertChannel.server.ts | 31 +- .../v3/services/alerts/deliverAlert.server.ts | 9 + .../alerts/deliverErrorGroupAlert.server.ts | 404 ++++++++++ .../alerts/errorAlertEvaluator.server.ts | 484 ++++++++++++ .../alerts/errorGroupWebhook.server.ts | 74 ++ .../v3/services/errorGroupActions.server.ts | 144 ++++ apps/webapp/tailwind.config.js | 2 + apps/webapp/test/errorGroupWebhook.test.ts | 248 ++++++ apps/webapp/test/slackErrorAlerts.test.ts | 403 ++++++++++ apps/webapp/test/webhookErrorAlerts.test.ts | 128 ++++ internal-packages/clickhouse/src/errors.ts | 149 +++- internal-packages/clickhouse/src/index.ts | 9 + .../migration.sql | 53 ++ .../database/prisma/schema.prisma | 98 ++- .../emails/emails/alert-error-group.tsx | 114 +++ internal-packages/emails/src/index.tsx | 16 + packages/core/src/v3/schemas/webhooks.ts | 63 ++ 54 files changed, 5172 insertions(+), 374 deletions(-) create mode 100644 apps/webapp/app/assets/icons/SlackMonoIcon.tsx create mode 100644 apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx create mode 100644 apps/webapp/app/components/errors/ErrorStatusBadge.tsx create mode 100644 apps/webapp/app/components/errors/ErrorStatusMenu.tsx create mode 100644 apps/webapp/app/components/logs/LogsVersionFilter.tsx create mode 100644 apps/webapp/app/components/primitives/UnorderedList.tsx create mode 100644 apps/webapp/app/presenters/v3/ErrorAlertChannelPresenter.server.ts create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.connect-to-slack.ts create mode 100644 apps/webapp/app/routes/storybook.unordered-list/route.tsx create mode 100644 apps/webapp/app/v3/services/alerts/deliverErrorGroupAlert.server.ts create mode 100644 apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts create mode 100644 apps/webapp/app/v3/services/alerts/errorGroupWebhook.server.ts create mode 100644 apps/webapp/app/v3/services/errorGroupActions.server.ts create mode 100644 apps/webapp/test/errorGroupWebhook.test.ts create mode 100644 apps/webapp/test/slackErrorAlerts.test.ts create mode 100644 apps/webapp/test/webhookErrorAlerts.test.ts create mode 100644 internal-packages/database/prisma/migrations/20260306102053_error_group_state/migration.sql create mode 100644 internal-packages/emails/emails/alert-error-group.tsx diff --git a/apps/webapp/app/assets/icons/SlackMonoIcon.tsx b/apps/webapp/app/assets/icons/SlackMonoIcon.tsx new file mode 100644 index 00000000000..666393a229d --- /dev/null +++ b/apps/webapp/app/assets/icons/SlackMonoIcon.tsx @@ -0,0 +1,10 @@ +export function SlackMonoIcon({ className }: { className?: string }) { + return ( + + + + + + + ); +} diff --git a/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx b/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx new file mode 100644 index 00000000000..dc586c89438 --- /dev/null +++ b/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx @@ -0,0 +1,365 @@ +import { conform, list, requestIntent, useFieldList, useForm } from "@conform-to/react"; +import { parse } from "@conform-to/zod"; +import { + EnvelopeIcon, + GlobeAltIcon, + HashtagIcon, + LockClosedIcon, + XMarkIcon, +} from "@heroicons/react/20/solid"; +import { useFetcher, useNavigate } from "@remix-run/react"; +import { SlackIcon } from "@trigger.dev/companyicons"; +import { Fragment, useEffect, useRef, useState } from "react"; +import { z } from "zod"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Callout, variantClasses } from "~/components/primitives/Callout"; +import { useToast } from "~/components/primitives/Toast"; +import { Fieldset } from "~/components/primitives/Fieldset"; +import { FormError } from "~/components/primitives/FormError"; +import { Header2, Header3 } from "~/components/primitives/Headers"; +import { Hint } from "~/components/primitives/Hint"; +import { InlineCode } from "~/components/code/InlineCode"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { Select, SelectItem } from "~/components/primitives/Select"; +import { UnorderedList } from "~/components/primitives/UnorderedList"; +import type { ErrorAlertChannelData } from "~/presenters/v3/ErrorAlertChannelPresenter.server"; +import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { cn } from "~/utils/cn"; +import { organizationSlackIntegrationPath } from "~/utils/pathBuilder"; +import { ExitIcon } from "~/assets/icons/ExitIcon"; +import { TextLink } from "~/components/primitives/TextLink"; +import { BellAlertIcon } from "@heroicons/react/24/solid"; + +export const ErrorAlertsFormSchema = z.object({ + emails: z.preprocess((i) => { + if (typeof i === "string") return i === "" ? [] : [i]; + if (Array.isArray(i)) return i.filter((v) => typeof v === "string" && v !== ""); + return []; + }, z.string().email().array()), + slackChannel: z.string().optional(), + slackIntegrationId: z.string().optional(), + webhooks: z.preprocess((i) => { + if (typeof i === "string") return i === "" ? [] : [i]; + if (Array.isArray(i)) return i.filter((v) => typeof v === "string" && v !== ""); + return []; + }, z.string().url().array()), +}); + +type ConfigureErrorAlertsProps = ErrorAlertChannelData & { + connectToSlackHref?: string; + formAction: string; +}; + +export function ConfigureErrorAlerts({ + emails: existingEmails, + webhooks: existingWebhooks, + slackChannel: existingSlackChannel, + slack, + emailAlertsEnabled, + connectToSlackHref, + formAction, +}: ConfigureErrorAlertsProps) { + const organization = useOrganization(); + const fetcher = useFetcher<{ ok?: boolean }>(); + const navigate = useNavigate(); + const toast = useToast(); + const location = useOptimisticLocation(); + const isSubmitting = fetcher.state !== "idle"; + + const [selectedSlackChannelValue, setSelectedSlackChannelValue] = useState( + existingSlackChannel + ? `${existingSlackChannel.channelId}/${existingSlackChannel.channelName}` + : undefined + ); + + const selectedSlackChannel = + slack.status === "READY" + ? slack.channels?.find((s) => selectedSlackChannelValue === `${s.id}/${s.name}`) + : undefined; + + const closeHref = (() => { + const params = new URLSearchParams(location.search); + params.delete("alerts"); + const qs = params.toString(); + return qs ? `?${qs}` : location.pathname; + })(); + + const hasHandledSuccess = useRef(false); + useEffect(() => { + if (fetcher.state === "idle" && fetcher.data?.ok && !hasHandledSuccess.current) { + hasHandledSuccess.current = true; + toast.success("Alert settings saved"); + navigate(closeHref, { replace: true }); + } + }, [fetcher.state, fetcher.data, closeHref, navigate, toast]); + + const emailFieldValues = useRef( + existingEmails.length > 0 ? [...existingEmails.map((e) => e.email), ""] : [""] + ); + + const webhookFieldValues = useRef( + existingWebhooks.length > 0 ? [...existingWebhooks.map((w) => w.url), ""] : [""] + ); + + const [form, { emails, webhooks, slackChannel, slackIntegrationId }] = useForm({ + id: "configure-error-alerts", + onValidate({ formData }) { + return parse(formData, { schema: ErrorAlertsFormSchema }); + }, + shouldRevalidate: "onSubmit", + defaultValue: { + emails: emailFieldValues.current, + webhooks: webhookFieldValues.current, + }, + }); + + const emailFields = useFieldList(form.ref, emails); + const webhookFields = useFieldList(form.ref, webhooks); + + return ( +
+
+ + Configure alerts + + +
+ + +
+
+
+ Receive alerts when + +
  • An error is seen for the first time
    • +
  • A resolved error re-occurs
    • +
  • An ignored error re-occurs based on settings you configured
    • +     +
    + + {/* Email section */} +
    + Email + {emailAlertsEnabled ? ( + + {emailFields.map((emailField, index) => ( + + { + emailFieldValues.current[index] = e.target.value; + if ( + emailFields.length === emailFieldValues.current.length && + emailFieldValues.current.every((v) => v !== "") + ) { + requestIntent(form.ref.current ?? undefined, list.append(emails.name)); + } + }} + /> + {emailField.error} + + ))} + + ) : ( + + Email integration is not available. Please contact your organization + administrator. + + )} +
    + + {/* Slack section */} +
    + Slack + + + {slack.status === "READY" ? ( + <> + + {selectedSlackChannel && selectedSlackChannel.is_private && ( + + To receive alerts in the{" "} + {selectedSlackChannel.name}{" "} + channel, you need to invite the @Trigger.dev Slack Bot. Go to the channel in + Slack and type:{" "} + /invite @Trigger.dev. + + )} + + + Manage Slack connection + + + + + ) : slack.status === "NOT_CONFIGURED" ? ( + connectToSlackHref ? ( + + + Connect to Slack + + + ) : ( + + Slack is not connected. Connect Slack from the{" "} + Alerts page to enable + Slack notifications. + + ) + ) : slack.status === "TOKEN_REVOKED" || slack.status === "TOKEN_EXPIRED" ? ( + connectToSlackHref ? ( +
    + + The Slack integration in your workspace has been revoked or has expired. + Please re-connect your Slack workspace. + + + + Connect to Slack + + +
    + ) : ( + + The Slack integration in your workspace has been revoked or expired. Please + re-connect from the{" "} + Alerts page. + + ) + ) : slack.status === "FAILED_FETCHING_CHANNELS" ? ( + + Failed loading channels from Slack. Please try again later. + + ) : ( + + Slack integration is not available. Please contact your organization + administrator. + + )} +     +
    + + {/* Webhook section */} +
    + Webhook + + {webhookFields.map((webhookField, index) => ( + + { + webhookFieldValues.current[index] = e.target.value; + if ( + webhookFields.length === webhookFieldValues.current.length && + webhookFieldValues.current.every((v) => v !== "") + ) { + requestIntent(form.ref.current ?? undefined, list.append(webhooks.name)); + } + }} + /> + {webhookField.error} + + ))} + We'll issue POST requests to these URLs with a JSON payload. + +
    + + {form.error} +
    +
    + +
    + + Cancel + + +
    +     +
    + ); +} + +function SlackChannelTitle({ name, is_private }: { name?: string; is_private?: boolean }) { + return ( +
    + {is_private ? : } + {name} +
    + ); +} diff --git a/apps/webapp/app/components/errors/ErrorStatusBadge.tsx b/apps/webapp/app/components/errors/ErrorStatusBadge.tsx new file mode 100644 index 00000000000..571a209ddf1 --- /dev/null +++ b/apps/webapp/app/components/errors/ErrorStatusBadge.tsx @@ -0,0 +1,34 @@ +import { type ErrorGroupStatus } from "@trigger.dev/database"; +import { cn } from "~/utils/cn"; + +const styles: Record = { + UNRESOLVED: "bg-error/10 text-error", + RESOLVED: "bg-success/10 text-success", + IGNORED: "bg-blue-500/10 text-blue-400", +}; + +const labels: Record = { + UNRESOLVED: "Unresolved", + RESOLVED: "Resolved", + IGNORED: "Ignored", +}; + +export function ErrorStatusBadge({ + status, + className, +}: { + status: ErrorGroupStatus; + className?: string; +}) { + return ( + + {labels[status]} + + ); +} diff --git a/apps/webapp/app/components/errors/ErrorStatusMenu.tsx b/apps/webapp/app/components/errors/ErrorStatusMenu.tsx new file mode 100644 index 00000000000..a981c8eee52 --- /dev/null +++ b/apps/webapp/app/components/errors/ErrorStatusMenu.tsx @@ -0,0 +1,250 @@ +import { CheckIcon } from "@heroicons/react/20/solid"; +import { + IconAlarmSnooze as IconAlarmSnoozeBase, + IconArrowBackUp as IconArrowBackUpBase, + IconBugOff as IconBugOffBase, +} from "@tabler/icons-react"; +import { useEffect, useRef, useState } from "react"; +import { type ErrorGroupStatus } from "@trigger.dev/database"; +import { useFetcher } from "@remix-run/react"; +import { Button } from "~/components/primitives/Buttons"; +import { useToast } from "~/components/primitives/Toast"; +import { FormError } from "~/components/primitives/FormError"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { Label } from "~/components/primitives/Label"; +import { PopoverMenuItem } from "~/components/primitives/Popover"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "~/components/primitives/Dialog"; + +const AlarmSnoozeIcon = ({ className }: { className?: string }) => ( + +); +const ArrowBackUpIcon = ({ className }: { className?: string }) => ( + +); +const BugOffIcon = ({ className }: { className?: string }) => ( + +); + +export function statusActionToastMessage(data: Record): string { + switch (data.action) { + case "resolve": + return "Error marked as resolved"; + case "unresolve": + return "Error marked as unresolved"; + case "ignore": { + const duration = data.duration ? Number(data.duration) : undefined; + if (!duration) return "Error ignored indefinitely"; + const hours = duration / (60 * 60 * 1000); + if (hours < 24) return `Error ignored for ${hours} ${hours === 1 ? "hour" : "hours"}`; + const days = hours / 24; + return `Error ignored for ${days} ${days === 1 ? "day" : "days"}`; + } + default: + return "Error status updated"; + } +} + +export function ErrorStatusMenuItems({ + status, + taskIdentifier, + onAction, + onCustomIgnore, +}: { + status: ErrorGroupStatus; + taskIdentifier: string; + onAction: (data: Record) => void; + onCustomIgnore: () => void; +}) { + return ( + <> + {status === "UNRESOLVED" && ( + <> + onAction({ taskIdentifier, action: "resolve" })} + /> + + onAction({ + taskIdentifier, + action: "ignore", + duration: String(60 * 60 * 1000), + }) + } + /> + + onAction({ + taskIdentifier, + action: "ignore", + duration: String(24 * 60 * 60 * 1000), + }) + } + /> + onAction({ taskIdentifier, action: "ignore" })} + /> + + + )} + + {status === "IGNORED" && ( + <> + onAction({ taskIdentifier, action: "resolve" })} + /> + onAction({ taskIdentifier, action: "unresolve" })} + /> + + )} + + {status === "RESOLVED" && ( + onAction({ taskIdentifier, action: "unresolve" })} + /> + )} + + ); +} + +export function CustomIgnoreDialog({ + open, + onOpenChange, + taskIdentifier, + formAction, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + taskIdentifier: string; + formAction?: string; +}) { + const fetcher = useFetcher<{ ok?: boolean }>(); + const isSubmitting = fetcher.state !== "idle"; + const [conditionError, setConditionError] = useState(null); + const toast = useToast(); + const hasHandledSuccess = useRef(false); + + useEffect(() => { + if (fetcher.state === "idle" && fetcher.data?.ok && !hasHandledSuccess.current) { + hasHandledSuccess.current = true; + toast.success("Error ignored with custom condition"); + onOpenChange(false); + } + }, [fetcher.state, fetcher.data, onOpenChange, toast]); + + return ( + + + + + + Custom ignore condition + + + { + e.preventDefault(); + const formData = new FormData(e.currentTarget); + const rate = formData.get("occurrenceRate")?.toString().trim(); + const total = formData.get("totalOccurrences")?.toString().trim(); + + if (!rate && !total) { + setConditionError("At least one unignore condition is required"); + return; + } + + setConditionError(null); + hasHandledSuccess.current = false; + fetcher.submit(e.currentTarget, { method: "post", action: formAction }); + }} + > + + + +
    + + + conditionError && setConditionError(null)} + /> + + + + + conditionError && setConditionError(null)} + /> + + + {conditionError && {conditionError}} + + + + + +
    + + + + + +     +     +     + ); +} diff --git a/apps/webapp/app/components/logs/LogsVersionFilter.tsx b/apps/webapp/app/components/logs/LogsVersionFilter.tsx new file mode 100644 index 00000000000..4cc10545060 --- /dev/null +++ b/apps/webapp/app/components/logs/LogsVersionFilter.tsx @@ -0,0 +1,58 @@ +import * as Ariakit from "@ariakit/react"; +import { SelectTrigger } from "~/components/primitives/Select"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { appliedSummary, FilterMenuProvider } from "~/components/runs/v3/SharedFilters"; +import { filterIcon, VersionsDropdown } from "~/components/runs/v3/RunFilters"; +import { AppliedFilter } from "~/components/primitives/AppliedFilter"; + +const shortcut = { key: "v" }; + +export function LogsVersionFilter() { + const { values, del } = useSearchParams(); + const selectedVersions = values("versions"); + + if (selectedVersions.length === 0 || selectedVersions.every((v) => v === "")) { + return ( + + {(search, setSearch) => ( + + Versions + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + /> + )} + + ); + } + + return ( + + {(search, setSearch) => ( + }> + del(["versions", "cursor", "direction"])} + variant="secondary/small" + /> + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + /> + )} + + ); +} diff --git a/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx b/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx index b3cc17724a3..e274ad20f43 100644 --- a/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx +++ b/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx @@ -83,6 +83,7 @@ export function OrganizationSettingsSideMenu({ name="Usage" icon={ChartBarIcon} activeIconColor="text-indigo-500" + inactiveIconColor="text-indigo-500" to={v3UsagePath(organization)} data-action="usage" /> @@ -90,6 +91,7 @@ export function OrganizationSettingsSideMenu({ name="Billing" icon={CreditCardIcon} activeIconColor="text-emerald-500" + inactiveIconColor="text-emerald-500" to={v3BillingPath(organization)} data-action="billing" badge={ @@ -102,6 +104,7 @@ export function OrganizationSettingsSideMenu({ name="Billing alerts" icon={BellAlertIcon} activeIconColor="text-rose-500" + inactiveIconColor="text-rose-500" to={v3BillingAlertsPath(organization)} data-action="billing-alerts" /> @@ -112,6 +115,7 @@ export function OrganizationSettingsSideMenu({ name="Private Connections" icon={LockClosedIcon} activeIconColor="text-purple-500" + inactiveIconColor="text-purple-500" to={v3PrivateConnectionsPath(organization)} data-action="private-connections" /> @@ -120,6 +124,7 @@ export function OrganizationSettingsSideMenu({ name="Team" icon={UserGroupIcon} activeIconColor="text-amber-500" + inactiveIconColor="text-amber-500" to={organizationTeamPath(organization)} data-action="team" /> @@ -127,6 +132,7 @@ export function OrganizationSettingsSideMenu({ name="Settings" icon={Cog8ToothIcon} activeIconColor="text-orgSettings" + inactiveIconColor="text-orgSettings" to={organizationSettingsPath(organization)} data-action="settings" /> @@ -139,6 +145,8 @@ export function OrganizationSettingsSideMenu({ name="Vercel" icon={VercelLogo} activeIconColor="text-white" + inactiveIconColor="text-white" + iconClassName="size-4 ml-0.5" to={organizationVercelIntegrationPath(organization)} data-action="integrations" /> @@ -146,6 +154,7 @@ export function OrganizationSettingsSideMenu({ name="Slack" icon={SlackIcon} activeIconColor="text-white" + inactiveIconColor="text-white" to={organizationSlackIntegrationPath(organization)} data-action="integrations" /> diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index d64fc96488c..1169343e9d3 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -464,10 +464,7 @@ export function SideMenu({ title="AI" isSideMenuCollapsed={isCollapsed} itemSpacingClassName="space-y-0" - initialCollapsed={getSectionCollapsed( - user.dashboardPreferences.sideMenu, - "ai" - )} + initialCollapsed={getSectionCollapsed(user.dashboardPreferences.sideMenu, "ai")} onCollapseToggle={handleSectionToggle("ai")} > { +export const RelativeDateTime = ({ date, timeZone, capitalize = true }: RelativeDateTimeProps) => { const locales = useLocales(); const userTimeZone = useUserTimeZone(); const realDate = useMemo(() => (typeof date === "string" ? new Date(date) : date), [date]); - const [relativeText, setRelativeText] = useState(() => getRelativeText(realDate)); + const [relativeText, setRelativeText] = useState(() => getRelativeText(realDate, capitalize)); // Every 60s refresh useEffect(() => { const interval = setInterval(() => { - setRelativeText(getRelativeText(realDate)); + setRelativeText(getRelativeText(realDate, capitalize)); }, 60_000); return () => clearInterval(interval); - }, [realDate]); + }, [realDate, capitalize]); // On first render useEffect(() => { - setRelativeText(getRelativeText(realDate)); - }, [realDate]); + setRelativeText(getRelativeText(realDate, capitalize)); + }, [realDate, capitalize]); return ( ["type"]; } >( ( @@ -80,6 +83,9 @@ const PopoverMenuItem = React.forwardRef< onClick, disabled, openInNewTab = false, + name, + value, + type, }, ref ) => { @@ -114,7 +120,6 @@ const PopoverMenuItem = React.forwardRef< return ( @@ -197,6 +205,18 @@ const popoverArrowTriggerVariants = { text: "group-hover:text-text-bright", icon: "text-text-dimmed group-hover:text-text-bright", }, + primary: { + trigger: + "bg-indigo-600 border border-indigo-500 text-text-bright hover:bg-indigo-500 hover:border-indigo-400 disabled:opacity-50 disabled:pointer-events-none", + text: "text-text-bright hover:text-white", + icon: "text-text-bright", + }, + secondary: { + trigger: + "bg-secondary border border-charcoal-600 text-text-bright hover:bg-charcoal-600 hover:border-charcoal-550 disabled:opacity-60 disabled:pointer-events-none", + text: "text-text-bright", + icon: "text-text-bright", + }, tertiary: { trigger: "bg-tertiary text-text-bright hover:bg-charcoal-600", text: "text-text-bright", @@ -245,8 +265,7 @@ function PopoverArrowTrigger({ const popoverVerticalEllipseVariants = { minimal: { - trigger: - "size-6 rounded-[3px] text-text-dimmed hover:bg-tertiary hover:text-text-bright", + trigger: "size-6 rounded-[3px] text-text-dimmed hover:bg-tertiary hover:text-text-bright", icon: "size-5", }, secondary: { diff --git a/apps/webapp/app/components/primitives/Table.tsx b/apps/webapp/app/components/primitives/Table.tsx index dfff784853d..1a30bc82b8a 100644 --- a/apps/webapp/app/components/primitives/Table.tsx +++ b/apps/webapp/app/components/primitives/Table.tsx @@ -431,7 +431,7 @@ export const TableCellMenu = forwardRef< onClick?: (event: React.MouseEvent) => void; visibleButtons?: ReactNode; hiddenButtons?: ReactNode; - popoverContent?: ReactNode; + popoverContent?: ReactNode | ((close: () => void) => ReactNode); children?: ReactNode; isSelected?: boolean; } @@ -451,6 +451,8 @@ export const TableCellMenu = forwardRef< ) => { const [isOpen, setIsOpen] = useState(false); const { variant } = useContext(TableContext); + const resolvedContent = + typeof popoverContent === "function" ? popoverContent(() => setIsOpen(false)) : popoverContent; return ( setIsOpen(open)}> + {resolvedContent && ( + setIsOpen(open)}> -
    {popoverContent}
    + {typeof popoverContent === "function" ? ( + resolvedContent + ) : ( +
    {resolvedContent}
    + )}     )} diff --git a/apps/webapp/app/components/primitives/Toast.tsx b/apps/webapp/app/components/primitives/Toast.tsx index 742715fa6ad..175d5ccb604 100644 --- a/apps/webapp/app/components/primitives/Toast.tsx +++ b/apps/webapp/app/components/primitives/Toast.tsx @@ -1,7 +1,7 @@ import { EnvelopeIcon, ExclamationCircleIcon, XMarkIcon } from "@heroicons/react/20/solid"; import { CheckCircleIcon } from "@heroicons/react/24/solid"; import { useSearchParams } from "@remix-run/react"; -import { useEffect } from "react"; +import { useEffect, useMemo } from "react"; import { useTypedLoaderData } from "remix-typedjson"; import { Toaster, toast } from "sonner"; import { type ToastMessageAction } from "~/models/message.server"; @@ -43,6 +43,32 @@ export function Toast() { return ; } +export function useToast() { + return useMemo( + () => ({ + success(message: string, options?: { title?: string; ephemeral?: boolean }) { + const ephemeral = options?.ephemeral ?? true; + toast.custom( + (t) => ( + + ), + { duration: ephemeral ? defaultToastDuration : permanentToastDuration } + ); + }, + error(message: string, options?: { title?: string; ephemeral?: boolean }) { + const ephemeral = options?.ephemeral ?? true; + toast.custom( + (t) => ( + + ), + { duration: ephemeral ? defaultToastDuration : permanentToastDuration } + ); + }, + }), + [] + ); +} + export function ToastUI({ variant, message, diff --git a/apps/webapp/app/components/primitives/UnorderedList.tsx b/apps/webapp/app/components/primitives/UnorderedList.tsx new file mode 100644 index 00000000000..e65dfe6673f --- /dev/null +++ b/apps/webapp/app/components/primitives/UnorderedList.tsx @@ -0,0 +1,129 @@ +import { cn } from "~/utils/cn"; +import { type ParagraphVariant } from "./Paragraph"; + +const listVariants: Record< + ParagraphVariant, + { text: string; spacing: string; items: string } +> = { + base: { + text: "font-sans text-base font-normal text-text-dimmed", + spacing: "mb-3", + items: "space-y-1 [&>li]:gap-1.5", + }, + "base/bright": { + text: "font-sans text-base font-normal text-text-bright", + spacing: "mb-3", + items: "space-y-1 [&>li]:gap-1.5", + }, + small: { + text: "font-sans text-sm font-normal text-text-dimmed", + spacing: "mb-2", + items: "space-y-0.5 [&>li]:gap-1", + }, + "small/bright": { + text: "font-sans text-sm font-normal text-text-bright", + spacing: "mb-2", + items: "space-y-0.5 [&>li]:gap-1", + }, + "small/dimmed": { + text: "font-sans text-sm font-normal text-text-dimmed", + spacing: "mb-2", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small": { + text: "font-sans text-xs font-normal text-text-dimmed", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/bright": { + text: "font-sans text-xs font-normal text-text-bright", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/dimmed": { + text: "font-sans text-xs font-normal text-text-dimmed", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/dimmed/mono": { + text: "font-mono text-xs font-normal text-text-dimmed", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/mono": { + text: "font-mono text-xs font-normal text-text-dimmed", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/bright/mono": { + text: "font-mono text-xs text-text-bright", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/caps": { + text: "font-sans text-xs uppercase tracking-wider font-normal text-text-dimmed", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-small/bright/caps": { + text: "font-sans text-xs uppercase tracking-wider font-normal text-text-bright", + spacing: "mb-1.5", + items: "space-y-0.5 [&>li]:gap-1", + }, + "extra-extra-small": { + text: "font-sans text-xxs font-normal text-text-dimmed", + spacing: "mb-1", + items: "space-y-0.5 [&>li]:gap-0.5", + }, + "extra-extra-small/bright": { + text: "font-sans text-xxs font-normal text-text-bright", + spacing: "mb-1", + items: "space-y-0.5 [&>li]:gap-0.5", + }, + "extra-extra-small/caps": { + text: "font-sans text-xxs uppercase tracking-wider font-normal text-text-dimmed", + spacing: "mb-1", + items: "space-y-0.5 [&>li]:gap-0.5", + }, + "extra-extra-small/bright/caps": { + text: "font-sans text-xxs uppercase tracking-wider font-normal text-text-bright", + spacing: "mb-1", + items: "space-y-0.5 [&>li]:gap-0.5", + }, + "extra-extra-small/dimmed/caps": { + text: "font-sans text-xxs uppercase tracking-wider font-normal text-text-dimmed", + spacing: "mb-1", + items: "space-y-0.5 [&>li]:gap-0.5", + }, +}; + +type UnorderedListProps = { + variant?: ParagraphVariant; + className?: string; + spacing?: boolean; + children: React.ReactNode; +} & React.HTMLAttributes; + +export function UnorderedList({ + variant = "base", + className, + spacing = false, + children, + ...props +}: UnorderedListProps) { + const v = listVariants[variant]; + return ( +
      li]:flex [&>li]:items-baseline [&>li]:before:shrink-0 [&>li]:before:content-['•']", + v.text, + v.items, + spacing && v.spacing, + className + )} + {...props} + > + {children} +
    + ); +} diff --git a/apps/webapp/app/components/primitives/charts/ChartBar.tsx b/apps/webapp/app/components/primitives/charts/ChartBar.tsx index 28493070c06..0b560747297 100644 --- a/apps/webapp/app/components/primitives/charts/ChartBar.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartBar.tsx @@ -174,6 +174,7 @@ export function ChartBarRenderer({ } labelFormatter={tooltipLabelFormatter} allowEscapeViewBox={{ x: false, y: true }} + animationDuration={0} /> {/* Zoom selection area - rendered before bars to appear behind them */} diff --git a/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx b/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx index 6cf3f7d7f24..7fe77d97e81 100644 --- a/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx @@ -180,7 +180,7 @@ export function ChartLegendCompound({ )} > {currentTotalLabel} - + {currentTotal != null ? ( valueFormatter ? ( valueFormatter(currentTotal) @@ -253,7 +253,7 @@ export function ChartLegendCompound({ /> @@ -350,7 +350,7 @@ function HoveredHiddenItemRow({ item, value, remainingCount, valueFormatter }: H {item.label} {remainingCount > 0 && +{remainingCount} more} - + {value != null ? ( valueFormatter ? ( valueFormatter(value) diff --git a/apps/webapp/app/components/primitives/charts/ChartRoot.tsx b/apps/webapp/app/components/primitives/charts/ChartRoot.tsx index 9a366c9789d..3b2a2c6a3c1 100644 --- a/apps/webapp/app/components/primitives/charts/ChartRoot.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartRoot.tsx @@ -40,6 +40,8 @@ export type ChartRootProps = { onViewAllLegendItems?: () => void; /** When true, constrains legend to max 50% height with scrolling */ legendScrollable?: boolean; + /** Additional className for the legend */ + legendClassName?: string; /** When true, chart fills its parent container height and distributes space between chart and legend */ fillContainer?: boolean; /** Content rendered between the chart and the legend */ @@ -87,6 +89,7 @@ export function ChartRoot({ legendValueFormatter, onViewAllLegendItems, legendScrollable = false, + legendClassName, fillContainer = false, beforeLegend, children, @@ -114,6 +117,7 @@ export function ChartRoot({ legendValueFormatter={legendValueFormatter} onViewAllLegendItems={onViewAllLegendItems} legendScrollable={legendScrollable} + legendClassName={legendClassName} fillContainer={fillContainer} beforeLegend={beforeLegend} > @@ -133,6 +137,7 @@ type ChartRootInnerProps = { legendValueFormatter?: (value: number) => string; onViewAllLegendItems?: () => void; legendScrollable?: boolean; + legendClassName?: string; fillContainer?: boolean; beforeLegend?: React.ReactNode; children: React.ComponentProps["children"]; @@ -148,6 +153,7 @@ function ChartRootInner({ legendValueFormatter, onViewAllLegendItems, legendScrollable = false, + legendClassName, fillContainer = false, beforeLegend, children, @@ -193,6 +199,7 @@ function ChartRootInner({ valueFormatter={legendValueFormatter} onViewAllLegendItems={onViewAllLegendItems} scrollable={legendScrollable} + className={legendClassName} /> )} diff --git a/apps/webapp/app/components/runs/v3/EnabledStatus.tsx b/apps/webapp/app/components/runs/v3/EnabledStatus.tsx index 9e1f7163239..ff902147f19 100644 --- a/apps/webapp/app/components/runs/v3/EnabledStatus.tsx +++ b/apps/webapp/app/components/runs/v3/EnabledStatus.tsx @@ -1,4 +1,4 @@ -import { BoltSlashIcon, CheckCircleIcon } from "@heroicons/react/20/solid"; +import { NoSymbolIcon, CheckIcon } from "@heroicons/react/20/solid"; type EnabledStatusProps = { enabled: boolean; @@ -8,8 +8,8 @@ type EnabledStatusProps = { export function EnabledStatus({ enabled, - enabledIcon = CheckCircleIcon, - disabledIcon = BoltSlashIcon, + enabledIcon = CheckIcon, + disabledIcon = NoSymbolIcon, }: EnabledStatusProps) { const EnabledIcon = enabledIcon; const DisabledIcon = disabledIcon; diff --git a/apps/webapp/app/components/runs/v3/RunFilters.tsx b/apps/webapp/app/components/runs/v3/RunFilters.tsx index f643209b8cb..dc3657b42a9 100644 --- a/apps/webapp/app/components/runs/v3/RunFilters.tsx +++ b/apps/webapp/app/components/runs/v3/RunFilters.tsx @@ -1216,7 +1216,7 @@ function AppliedMachinesFilter() { ); } -function VersionsDropdown({ +export function VersionsDropdown({ trigger, clearSearchValue, searchValue, diff --git a/apps/webapp/app/models/projectAlert.server.ts b/apps/webapp/app/models/projectAlert.server.ts index d2ab0be1d1a..dbcb672ad7d 100644 --- a/apps/webapp/app/models/projectAlert.server.ts +++ b/apps/webapp/app/models/projectAlert.server.ts @@ -32,3 +32,9 @@ export const ProjectAlertSlackStorage = z.object({ }); export type ProjectAlertSlackStorage = z.infer; + +export const ErrorAlertConfig = z.object({ + evaluationIntervalMs: z.number().min(60_000).default(300_000), +}); + +export type ErrorAlertConfig = z.infer; diff --git a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts index 4bc4c776e85..83ab09c177c 100644 --- a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts @@ -17,6 +17,7 @@ export const ApiAlertType = z.enum([ "attempt_failure", "deployment_failure", "deployment_success", + "error_group", ]); export type ApiAlertType = z.infer; @@ -85,6 +86,8 @@ export class ApiAlertChannelPresenter { return "deployment_failure"; case "DEPLOYMENT_SUCCESS": return "deployment_success"; + case "ERROR_GROUP": + return "error_group"; default: assertNever(alertType); } @@ -100,6 +103,8 @@ export class ApiAlertChannelPresenter { return "DEPLOYMENT_FAILURE"; case "deployment_success": return "DEPLOYMENT_SUCCESS"; + case "error_group": + return "ERROR_GROUP"; default: assertNever(alertType); } diff --git a/apps/webapp/app/presenters/v3/ErrorAlertChannelPresenter.server.ts b/apps/webapp/app/presenters/v3/ErrorAlertChannelPresenter.server.ts new file mode 100644 index 00000000000..e2d207555fe --- /dev/null +++ b/apps/webapp/app/presenters/v3/ErrorAlertChannelPresenter.server.ts @@ -0,0 +1,73 @@ +import type { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, + ProjectAlertWebhookProperties, +} from "~/models/projectAlert.server"; +import { BasePresenter } from "./basePresenter.server"; +import { NewAlertChannelPresenter } from "./NewAlertChannelPresenter.server"; +import { env } from "~/env.server"; + +export type ErrorAlertChannelData = Awaited>; + +export class ErrorAlertChannelPresenter extends BasePresenter { + public async call(projectId: string, environmentType: RuntimeEnvironmentType) { + const channels = await this._prisma.projectAlertChannel.findMany({ + where: { + projectId, + alertTypes: { has: "ERROR_GROUP" }, + environmentTypes: { has: environmentType }, + }, + orderBy: { createdAt: "asc" }, + }); + + const emails: Array<{ id: string; email: string }> = []; + const webhooks: Array<{ id: string; url: string }> = []; + let slackChannel: { id: string; channelId: string; channelName: string } | null = null; + + for (const channel of channels) { + switch (channel.type) { + case "EMAIL": { + const parsed = ProjectAlertEmailProperties.safeParse(channel.properties); + if (parsed.success) { + emails.push({ id: channel.id, email: parsed.data.email }); + } + break; + } + case "SLACK": { + if (!channel.enabled) break; + const parsed = ProjectAlertSlackProperties.safeParse(channel.properties); + if (parsed.success) { + slackChannel = { + id: channel.id, + channelId: parsed.data.channelId, + channelName: parsed.data.channelName, + }; + } + break; + } + case "WEBHOOK": { + const parsed = ProjectAlertWebhookProperties.safeParse(channel.properties); + if (parsed.success) { + webhooks.push({ id: channel.id, url: parsed.data.url }); + } + break; + } + } + } + + const slackPresenter = new NewAlertChannelPresenter(this._prisma, this._replica); + const slackResult = await slackPresenter.call(projectId); + + const emailAlertsEnabled = + env.ALERT_FROM_EMAIL !== undefined && env.ALERT_RESEND_API_KEY !== undefined; + + return { + emails, + webhooks, + slackChannel, + slack: slackResult.slack, + emailAlertsEnabled, + }; + } +} diff --git a/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts b/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts index 024ac1e95ea..5e9df362e4c 100644 --- a/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { type ClickHouse, msToClickHouseInterval } from "@internal/clickhouse"; import { TimeGranularity } from "~/utils/timeGranularity"; import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; -import { type PrismaClientOrTransaction } from "@trigger.dev/database"; +import { type ErrorGroupStatus, type PrismaClientOrTransaction } from "@trigger.dev/database"; import { timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { type Direction, DirectionSchema } from "~/components/ListPagination"; import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server"; @@ -27,6 +27,7 @@ export type ErrorGroupOptions = { userId?: string; projectId: string; fingerprint: string; + versions?: string[]; runsPageSize?: number; period?: string; from?: number; @@ -39,6 +40,7 @@ export const ErrorGroupOptionsSchema = z.object({ userId: z.string().optional(), projectId: z.string(), fingerprint: z.string(), + versions: z.array(z.string()).optional(), runsPageSize: z.number().int().positive().max(1000).optional(), period: z.string().optional(), from: z.number().int().nonnegative().optional(), @@ -59,6 +61,21 @@ function parseClickHouseDateTime(value: string): Date { return new Date(value.replace(" ", "T") + "Z"); } +export type ErrorGroupState = { + status: ErrorGroupStatus; + resolvedAt: Date | null; + resolvedInVersion: string | null; + resolvedBy: string | null; + ignoredAt: Date | null; + ignoredUntil: Date | null; + ignoredReason: string | null; + ignoredByUserId: string | null; + ignoredByUserDisplayName: string | null; + ignoredUntilOccurrenceRate: number | null; + ignoredUntilTotalOccurrences: number | null; + ignoredAtOccurrenceCount: number | null; +}; + export type ErrorGroupSummary = { fingerprint: string; errorType: string; @@ -68,10 +85,12 @@ export type ErrorGroupSummary = { firstSeen: Date; lastSeen: Date; affectedVersions: string[]; + state: ErrorGroupState; }; export type ErrorGroupOccurrences = Awaited>; export type ErrorGroupActivity = ErrorGroupOccurrences["data"]; +export type ErrorGroupActivityVersions = ErrorGroupOccurrences["versions"]; export class ErrorGroupPresenter extends BasePresenter { constructor( @@ -89,6 +108,7 @@ export class ErrorGroupPresenter extends BasePresenter { userId, projectId, fingerprint, + versions, runsPageSize = DEFAULT_RUNS_PAGE_SIZE, period, from, @@ -110,23 +130,40 @@ export class ErrorGroupPresenter extends BasePresenter { defaultPeriod: "7d", }); - const [summary, affectedVersions, runList] = await Promise.all([ - this.getSummary(organizationId, projectId, environmentId, fingerprint), + const summary = await this.getSummary(organizationId, projectId, environmentId, fingerprint); + + const [affectedVersions, runList, stateRow] = await Promise.all([ this.getAffectedVersions(organizationId, projectId, environmentId, fingerprint), this.getRunList(organizationId, environmentId, { userId, projectId, fingerprint, + versions, pageSize: runsPageSize, from: time.from.getTime(), to: time.to.getTime(), cursor, direction, }), + this.getState(environmentId, summary?.taskIdentifier, fingerprint), ]); if (summary) { summary.affectedVersions = affectedVersions; + summary.state = stateRow ?? { + status: "UNRESOLVED", + resolvedAt: null, + resolvedInVersion: null, + resolvedBy: null, + ignoredAt: null, + ignoredUntil: null, + ignoredReason: null, + ignoredByUserId: null, + ignoredByUserDisplayName: null, + ignoredUntilOccurrenceRate: null, + ignoredUntilTotalOccurrences: null, + ignoredAtOccurrenceCount: null, + }; } return { @@ -140,8 +177,8 @@ export class ErrorGroupPresenter extends BasePresenter { } /** - * Returns bucketed occurrence counts for a single fingerprint over a time range. - * Granularity is determined automatically from the range span. + * Returns bucketed occurrence counts for a single fingerprint over a time range, + * grouped by task_version for stacked charts. */ public async getOccurrences( organizationId: string, @@ -149,14 +186,17 @@ export class ErrorGroupPresenter extends BasePresenter { environmentId: string, fingerprint: string, from: Date, - to: Date + to: Date, + versions?: string[] ): Promise<{ - data: Array<{ date: Date; count: number }>; + data: Array>; + versions: string[]; }> { const granularityMs = errorGroupGranularity.getTimeGranularityMs(from, to); const intervalExpr = msToClickHouseInterval(granularityMs); - const queryBuilder = this.logsClickhouse.errors.createOccurrencesQueryBuilder(intervalExpr); + const queryBuilder = + this.logsClickhouse.errors.createOccurrencesByVersionQueryBuilder(intervalExpr); queryBuilder.where("organization_id = {organizationId: String}", { organizationId }); queryBuilder.where("project_id = {projectId: String}", { projectId }); @@ -169,7 +209,11 @@ export class ErrorGroupPresenter extends BasePresenter { toTimeMs: to.getTime(), }); - queryBuilder.groupBy("error_fingerprint, bucket_epoch"); + if (versions && versions.length > 0) { + queryBuilder.where("task_version IN {versions: Array(String)}", { versions }); + } + + queryBuilder.groupBy("error_fingerprint, task_version, bucket_epoch"); queryBuilder.orderBy("bucket_epoch ASC"); const [queryError, records] = await queryBuilder.execute(); @@ -186,17 +230,27 @@ export class ErrorGroupPresenter extends BasePresenter { buckets.push(epoch); } - const byBucket = new Map(); + // Collect distinct versions and index results by (epoch, version) + const versionSet = new Set(); + const byBucketVersion = new Map(); for (const row of records ?? []) { - byBucket.set(row.bucket_epoch, (byBucket.get(row.bucket_epoch) ?? 0) + row.count); + const version = row.task_version || "unknown"; + versionSet.add(version); + const key = `${row.bucket_epoch}:${version}`; + byBucketVersion.set(key, (byBucketVersion.get(key) ?? 0) + row.count); } - return { - data: buckets.map((epoch) => ({ - date: new Date(epoch * 1000), - count: byBucket.get(epoch) ?? 0, - })), - }; + const sortedVersions = sortVersionsDescending([...versionSet]); + + const data = buckets.map((epoch) => { + const point: Record = { date: new Date(epoch * 1000) }; + for (const version of sortedVersions) { + point[version] = byBucketVersion.get(`${epoch}:${version}`) ?? 0; + } + return point; + }); + + return { data, versions: sortedVersions }; } private async getSummary( @@ -235,6 +289,20 @@ export class ErrorGroupPresenter extends BasePresenter { firstSeen: parseClickHouseDateTime(record.first_seen), lastSeen: parseClickHouseDateTime(record.last_seen), affectedVersions: [], + state: { + status: "UNRESOLVED" as const, + resolvedAt: null, + resolvedInVersion: null, + resolvedBy: null, + ignoredAt: null, + ignoredUntil: null, + ignoredReason: null, + ignoredByUserId: null, + ignoredByUserDisplayName: null, + ignoredUntilOccurrenceRate: null, + ignoredUntilTotalOccurrences: null, + ignoredAtOccurrenceCount: null, + }, }; } @@ -268,6 +336,65 @@ export class ErrorGroupPresenter extends BasePresenter { return sortVersionsDescending(versions).slice(0, 5); } + private async getState( + environmentId: string, + taskIdentifier: string | undefined, + fingerprint: string + ): Promise { + const row = await this.replica.errorGroupState.findFirst({ + where: { + environmentId, + ...(taskIdentifier ? { taskIdentifier } : {}), + errorFingerprint: fingerprint, + }, + select: { + status: true, + resolvedAt: true, + resolvedInVersion: true, + resolvedBy: true, + ignoredAt: true, + ignoredUntil: true, + ignoredReason: true, + ignoredByUserId: true, + ignoredUntilOccurrenceRate: true, + ignoredUntilTotalOccurrences: true, + ignoredAtOccurrenceCount: true, + }, + }); + + if (!row) { + return null; + } + + let ignoredByUserDisplayName: string | null = null; + if (row.ignoredByUserId) { + const user = await this.replica.user.findFirst({ + where: { id: row.ignoredByUserId }, + select: { displayName: true, name: true, email: true }, + }); + if (user) { + ignoredByUserDisplayName = user.displayName ?? user.name ?? user.email; + } + } + + return { + status: row.status, + resolvedAt: row.resolvedAt, + resolvedInVersion: row.resolvedInVersion, + resolvedBy: row.resolvedBy, + ignoredAt: row.ignoredAt, + ignoredUntil: row.ignoredUntil, + ignoredReason: row.ignoredReason, + ignoredByUserId: row.ignoredByUserId, + ignoredByUserDisplayName, + ignoredUntilOccurrenceRate: row.ignoredUntilOccurrenceRate, + ignoredUntilTotalOccurrences: row.ignoredUntilTotalOccurrences, + ignoredAtOccurrenceCount: row.ignoredAtOccurrenceCount + ? Number(row.ignoredAtOccurrenceCount) + : null, + }; + } + private async getRunList( organizationId: string, environmentId: string, @@ -275,6 +402,7 @@ export class ErrorGroupPresenter extends BasePresenter { userId?: string; projectId: string; fingerprint: string; + versions?: string[]; pageSize: number; from?: number; to?: number; @@ -289,6 +417,7 @@ export class ErrorGroupPresenter extends BasePresenter { projectId: options.projectId, rootOnly: false, errorId: ErrorId.toFriendlyId(options.fingerprint), + versions: options.versions, pageSize: options.pageSize, from: options.from, to: options.to, diff --git a/apps/webapp/app/presenters/v3/ErrorsListPresenter.server.ts b/apps/webapp/app/presenters/v3/ErrorsListPresenter.server.ts index 89832b28340..13da4ff91f8 100644 --- a/apps/webapp/app/presenters/v3/ErrorsListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ErrorsListPresenter.server.ts @@ -9,7 +9,7 @@ const errorsListGranularity = new TimeGranularity([ { max: "3 months", granularity: "1w" }, { max: "Infinity", granularity: "30d" }, ]); -import { type PrismaClientOrTransaction } from "@trigger.dev/database"; +import { type ErrorGroupStatus, type PrismaClientOrTransaction } from "@trigger.dev/database"; import { type Direction } from "~/components/ListPagination"; import { timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server"; @@ -22,6 +22,8 @@ export type ErrorsListOptions = { projectId: string; // filters tasks?: string[]; + versions?: string[]; + statuses?: ErrorGroupStatus[]; period?: string; from?: number; to?: number; @@ -39,6 +41,8 @@ export const ErrorsListOptionsSchema = z.object({ userId: z.string().optional(), projectId: z.string(), tasks: z.array(z.string()).optional(), + versions: z.array(z.string()).optional(), + statuses: z.array(z.enum(["UNRESOLVED", "RESOLVED", "IGNORED"])).optional(), period: z.string().optional(), from: z.number().int().nonnegative().optional(), to: z.number().int().nonnegative().optional(), @@ -88,7 +92,11 @@ function decodeCursor(cursor: string): ErrorGroupCursor | null { } } -function cursorFromRow(row: { occurrence_count: number; error_fingerprint: string; task_identifier: string }): string { +function cursorFromRow(row: { + occurrence_count: number; + error_fingerprint: string; + task_identifier: string; +}): string { return encodeCursor({ occurrenceCount: row.occurrence_count, fingerprint: row.error_fingerprint, @@ -123,6 +131,8 @@ export class ErrorsListPresenter extends BasePresenter { userId, projectId, tasks, + versions, + statuses, period, search, from, @@ -156,20 +166,49 @@ export class ErrorsListPresenter extends BasePresenter { const hasFilters = (tasks !== undefined && tasks.length > 0) || + (versions !== undefined && versions.length > 0) || (search !== undefined && search !== "") || - !time.isDefault; + (statuses !== undefined && statuses.length > 0); const possibleTasksAsync = getAllTaskIdentifiers(this.replica, environmentId); - const [possibleTasks, displayableEnvironment] = await Promise.all([ + // Pre-filter by status: since status lives in Postgres (ErrorGroupState) and the error + // list comes from ClickHouse, we resolve inclusion/exclusion sets upfront so that + // ClickHouse pagination operates on the correctly filtered dataset. + const statusFilterAsync = this.resolveStatusFilter(environmentId, statuses); + + const [possibleTasks, displayableEnvironment, statusFilter] = await Promise.all([ possibleTasksAsync, findDisplayableEnvironment(environmentId, userId), + statusFilterAsync, ]); if (!displayableEnvironment) { throw new ServiceValidationError("No environment found"); } + if (statusFilter.empty) { + return { + errorGroups: [], + pagination: { + next: undefined, + previous: undefined, + }, + filters: { + tasks, + versions, + statuses, + search, + period: time, + from: effectiveFrom, + to: effectiveTo, + hasFilters, + possibleTasks, + wasClampedByRetention, + }, + }; + } + // Query the per-minute error_occurrences_v1 table for time-scoped counts const queryBuilder = this.clickhouse.errors.occurrencesListQueryBuilder(); @@ -189,6 +228,23 @@ export class ErrorsListPresenter extends BasePresenter { queryBuilder.where("task_identifier IN {tasks: Array(String)}", { tasks }); } + if (versions && versions.length > 0) { + queryBuilder.where("task_version IN {versions: Array(String)}", { versions }); + } + + if (statusFilter.includeKeys) { + queryBuilder.where( + "concat(task_identifier, '::', error_fingerprint) IN {statusIncludeKeys: Array(String)}", + { statusIncludeKeys: statusFilter.includeKeys } + ); + } + if (statusFilter.excludeKeys) { + queryBuilder.where( + "concat(task_identifier, '::', error_fingerprint) NOT IN {statusExcludeKeys: Array(String)}", + { statusExcludeKeys: statusFilter.excludeKeys } + ); + } + queryBuilder.groupBy("error_fingerprint, task_identifier"); // Text search via HAVING (operates on aggregated values) @@ -254,15 +310,14 @@ export class ErrorsListPresenter extends BasePresenter { // Fetch global first_seen / last_seen from the errors_v1 summary table const fingerprints = errorGroups.map((e) => e.error_fingerprint); - const globalSummaryMap = await this.getGlobalSummary( - organizationId, - projectId, - environmentId, - fingerprints - ); + const [globalSummaryMap, stateMap] = await Promise.all([ + this.getGlobalSummary(organizationId, projectId, environmentId, fingerprints), + this.getErrorGroupStates(environmentId, errorGroups), + ]); - const transformedErrorGroups = errorGroups.map((error) => { + let transformedErrorGroups = errorGroups.map((error) => { const global = globalSummaryMap.get(error.error_fingerprint); + const state = stateMap.get(`${error.task_identifier}:${error.error_fingerprint}`); return { errorType: error.error_type, errorMessage: error.error_message, @@ -271,6 +326,9 @@ export class ErrorsListPresenter extends BasePresenter { firstSeen: global?.firstSeen ?? new Date(), lastSeen: global?.lastSeen ?? new Date(), count: error.occurrence_count, + status: state?.status ?? "UNRESOLVED", + resolvedAt: state?.resolvedAt ?? null, + ignoredUntil: state?.ignoredUntil ?? null, }; }); @@ -282,6 +340,8 @@ export class ErrorsListPresenter extends BasePresenter { }, filters: { tasks, + versions, + statuses, search, period: time, from: effectiveFrom, @@ -367,6 +427,106 @@ export class ErrorsListPresenter extends BasePresenter { return { data }; } + /** + * Determines which (task, fingerprint) pairs to include or exclude from the ClickHouse + * query based on the requested status filter. Since status lives in Postgres and errors + * live in ClickHouse, we resolve the filter set here so ClickHouse pagination is correct. + * + * - UNRESOLVED is the default (no ErrorGroupState row), so filtering FOR it means + * excluding groups with non-matching explicit statuses. + * - RESOLVED/IGNORED are explicit, so filtering for them means including only matching groups. + */ + private async resolveStatusFilter( + environmentId: string, + statuses?: ErrorGroupStatus[] + ): Promise<{ + includeKeys?: string[]; + excludeKeys?: string[]; + empty: boolean; + }> { + if (!statuses || statuses.length === 0) { + return { empty: false }; + } + + const allStatuses: ErrorGroupStatus[] = ["UNRESOLVED", "RESOLVED", "IGNORED"]; + const excludedStatuses = allStatuses.filter((s) => !statuses.includes(s)); + + if (excludedStatuses.length === 0) { + return { empty: false }; + } + + if (statuses.includes("UNRESOLVED")) { + const excluded = await this.replica.errorGroupState.findMany({ + where: { environmentId, status: { in: excludedStatuses } }, + select: { taskIdentifier: true, errorFingerprint: true }, + }); + if (excluded.length === 0) { + return { empty: false }; + } + return { + excludeKeys: excluded.map((g) => `${g.taskIdentifier}::${g.errorFingerprint}`), + empty: false, + }; + } + + const included = await this.replica.errorGroupState.findMany({ + where: { environmentId, status: { in: statuses } }, + select: { taskIdentifier: true, errorFingerprint: true }, + }); + if (included.length === 0) { + return { empty: true }; + } + return { + includeKeys: included.map((g) => `${g.taskIdentifier}::${g.errorFingerprint}`), + empty: false, + }; + } + + /** + * Batch-fetch ErrorGroupState rows from Postgres for the given ClickHouse error groups. + * Returns a map keyed by `${taskIdentifier}:${errorFingerprint}`. + */ + private async getErrorGroupStates( + environmentId: string, + errorGroups: Array<{ task_identifier: string; error_fingerprint: string }> + ) { + type StateValue = { + status: ErrorGroupStatus; + resolvedAt: Date | null; + ignoredUntil: Date | null; + }; + + const result = new Map(); + if (errorGroups.length === 0) return result; + + const states = await this.replica.errorGroupState.findMany({ + where: { + environmentId, + OR: errorGroups.map((e) => ({ + taskIdentifier: e.task_identifier, + errorFingerprint: e.error_fingerprint, + })), + }, + select: { + taskIdentifier: true, + errorFingerprint: true, + status: true, + resolvedAt: true, + ignoredUntil: true, + }, + }); + + for (const state of states) { + result.set(`${state.taskIdentifier}:${state.errorFingerprint}`, { + status: state.status, + resolvedAt: state.resolvedAt, + ignoredUntil: state.ignoredUntil, + }); + } + + return result; + } + /** * Fetches global first_seen / last_seen for a set of fingerprints from errors_v1. */ diff --git a/apps/webapp/app/presenters/v3/NewAlertChannelPresenter.server.ts b/apps/webapp/app/presenters/v3/NewAlertChannelPresenter.server.ts index 08bccc66ef7..bde51bda91f 100644 --- a/apps/webapp/app/presenters/v3/NewAlertChannelPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/NewAlertChannelPresenter.server.ts @@ -20,6 +20,7 @@ export class NewAlertChannelPresenter extends BasePresenter { where: { service: "SLACK", organizationId: project.organizationId, + deletedAt: null, }, orderBy: { createdAt: "desc", diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new.connect-to-slack.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new.connect-to-slack.ts index 6800ab2ed88..ddd1bf646b7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new.connect-to-slack.ts +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new.connect-to-slack.ts @@ -28,6 +28,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { where: { service: "SLACK", organizationId: project.organizationId, + deletedAt: null, }, }); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx index 1bedd30d0f9..9b888a43624 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx @@ -63,6 +63,7 @@ import { v3NewProjectAlertPath, v3ProjectAlertsPath, } from "~/utils/pathBuilder"; +import { alertsWorker } from "~/v3/alertsWorker.server"; export const meta: MetaFunction = () => { return [ @@ -156,6 +157,17 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { data: { enabled: true }, }); + if (alertChannel.alertTypes.includes("ERROR_GROUP")) { + await alertsWorker.enqueue({ + id: `evaluateErrorAlerts:${project.id}`, + job: "v3.evaluateErrorAlerts", + payload: { + projectId: project.id, + scheduledAt: Date.now(), + }, + }); + } + return redirectWithSuccessMessage( v3ProjectAlertsPath({ slug: organizationSlug }, { slug: projectParam }, { slug: envParam }), request, @@ -555,8 +567,10 @@ export function alertTypeTitle(alertType: ProjectAlertType): string { return "Deployment failure"; case "DEPLOYMENT_SUCCESS": return "Deployment success"; + case "ERROR_GROUP": + return "Error group"; default: { - assertNever(alertType); + throw new Error(`Unknown alertType: ${alertType}`); } } } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx index 245f117ffdb..051ea7a8a28 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx @@ -5,7 +5,6 @@ import { Form, useNavigation } from "@remix-run/react"; import { IconChartHistogram, IconEdit, IconTypography } from "@tabler/icons-react"; import { useCallback, useEffect, useState } from "react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; -import { toast } from "sonner"; import { z } from "zod"; import { defaultChartConfig } from "~/components/code/ChartConfigPanel"; import { Feedback } from "~/components/Feedback"; @@ -33,7 +32,7 @@ import { PopoverVerticalEllipseTrigger, } from "~/components/primitives/Popover"; import { Sheet, SheetContent } from "~/components/primitives/SheetV3"; -import { ToastUI } from "~/components/primitives/Toast"; +import { useToast } from "~/components/primitives/Toast"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; import { QueryEditor, type QueryEditorSaveData } from "~/components/query/QueryEditor"; import { $replica, prisma } from "~/db.server"; @@ -206,7 +205,8 @@ export default function Page() { const widgetActionUrl = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/dashboards/${friendlyId}/widgets`; const layoutActionUrl = widgetActionUrl; - // Handle sync errors by showing a toast + const toast = useToast(); + const handleSyncError = useCallback((error: Error, action: string) => { const actionMessages: Record = { add: "Failed to add widget", @@ -218,15 +218,8 @@ export default function Page() { const message = actionMessages[action] || "Failed to save changes"; - toast.custom((t) => ( - - )); - }, []); + toast.error(`${message}. Your changes may not be saved.`, { title: "Sync Error" }); + }, [toast]); // Add title dialog state const [showAddTitleDialog, setShowAddTitleDialog] = useState(false); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index 0ff8594fa36..f42c73b5ea3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -1,8 +1,13 @@ -import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { type MetaFunction } from "@remix-run/react"; +import { type LoaderFunctionArgs, type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { type MetaFunction, useFetcher, useRevalidator } from "@remix-run/react"; +import { BellAlertIcon } from "@heroicons/react/20/solid"; +import { IconAlarmSnooze as IconAlarmSnoozeBase, IconCircleDotted } from "@tabler/icons-react"; +import { parse } from "@conform-to/zod"; +import { z } from "zod"; +import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; -import { requireUser } from "~/services/session.server"; +import { requireUser, requireUserId } from "~/services/session.server"; import { EnvironmentParamSchema, v3CreateBulkActionPath, @@ -14,38 +19,69 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { ErrorGroupPresenter, type ErrorGroupActivity, + type ErrorGroupActivityVersions, type ErrorGroupOccurrences, type ErrorGroupSummary, + type ErrorGroupState, } from "~/presenters/v3/ErrorGroupPresenter.server"; import { type NextRunList } from "~/presenters/v3/NextRunListPresenter.server"; import { $replica } from "~/db.server"; import { logsClickhouseClient, clickhouseClient } from "~/services/clickhouseInstance.server"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { PageBody } from "~/components/layout/AppLayout"; -import { Suspense, useMemo } from "react"; +import { + ResizableHandle, + ResizablePanel, + ResizablePanelGroup, +} from "~/components/primitives/Resizable"; +import { AnimatePresence, motion } from "framer-motion"; +import { Suspense, useEffect, useMemo, useRef, useState } from "react"; import { Spinner } from "~/components/primitives/Spinner"; import { Paragraph } from "~/components/primitives/Paragraph"; import { Callout } from "~/components/primitives/Callout"; -import { Header1, Header2, Header3 } from "~/components/primitives/Headers"; -import { formatDistanceToNow } from "date-fns"; -import { formatNumberCompact } from "~/utils/numberFormatter"; +import { Header2, Header3 } from "~/components/primitives/Headers"; + +import { formatDistanceToNow, isPast } from "date-fns"; + import * as Property from "~/components/primitives/PropertyTable"; import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; import { DateTime, RelativeDateTime } from "~/components/primitives/DateTime"; import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; -import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; +import { + Bar, + BarChart, + CartesianGrid, + ResponsiveContainer, + Tooltip, + type TooltipProps, + XAxis, + YAxis, +} from "recharts"; +import TooltipPortal from "~/components/primitives/TooltipPortal"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; import { DirectionSchema, ListPagination } from "~/components/ListPagination"; -import { LinkButton } from "~/components/primitives/Buttons"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import { useEnvironment } from "~/hooks/useEnvironment"; import { RunsIcon } from "~/assets/icons/RunsIcon"; -import { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; +import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { CopyableText } from "~/components/primitives/CopyableText"; +import { cn } from "~/utils/cn"; +import { LogsVersionFilter } from "~/components/logs/LogsVersionFilter"; +import { CodeBlock } from "~/components/code/CodeBlock"; + +import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover"; +import { ErrorGroupActions } from "~/v3/services/errorGroupActions.server"; +import { + ErrorStatusMenuItems, + CustomIgnoreDialog, + statusActionToastMessage, +} from "~/components/errors/ErrorStatusMenu"; +import { useToast } from "~/components/primitives/Toast"; export const meta: MetaFunction = ({ data }) => { return [ @@ -55,6 +91,119 @@ export const meta: MetaFunction = ({ data }) => { ]; }; +const emptyStringToUndefined = z.preprocess( + (v) => (v === "" ? undefined : v), + z.coerce.number().positive().optional() +); + +const actionSchema = z.discriminatedUnion("action", [ + z.object({ + action: z.literal("resolve"), + taskIdentifier: z.string().min(1), + resolvedInVersion: z.string().optional(), + }), + z.object({ + action: z.literal("ignore"), + taskIdentifier: z.string().min(1), + duration: emptyStringToUndefined, + occurrenceRate: emptyStringToUndefined, + totalOccurrences: emptyStringToUndefined, + reason: z.preprocess((v) => (v === "" ? undefined : v), z.string().optional()), + }), + z.object({ + action: z.literal("unresolve"), + taskIdentifier: z.string().min(1), + }), +]); + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const fingerprint = params.fingerprint; + + if (!fingerprint) { + return json({ error: "Fingerprint parameter is required" }, { status: 400 }); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + const formData = await request.formData(); + const submission = parse(formData, { schema: actionSchema }); + + if (!submission.value) { + return json(submission); + } + + const actions = new ErrorGroupActions(); + const identifier = { + organizationId: project.organizationId, + projectId: project.id, + environmentId: environment.id, + taskIdentifier: submission.value.taskIdentifier, + errorFingerprint: fingerprint, + }; + + switch (submission.value.action) { + case "resolve": { + await actions.resolveError(identifier, { + userId, + resolvedInVersion: submission.value.resolvedInVersion, + }); + return json({ ok: true }); + } + case "ignore": { + let occurrenceCountAtIgnoreTime: number | undefined; + + if (submission.value.totalOccurrences) { + const qb = clickhouseClient.errors.listQueryBuilder(); + qb.where("organization_id = {organizationId: String}", { + organizationId: project.organizationId, + }); + qb.where("project_id = {projectId: String}", { projectId: project.id }); + qb.where("environment_id = {environmentId: String}", { + environmentId: environment.id, + }); + qb.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + qb.where("task_identifier = {taskIdentifier: String}", { + taskIdentifier: submission.value.taskIdentifier, + }); + qb.groupBy("error_fingerprint, task_identifier"); + + const [err, results] = await qb.execute(); + if (err || !results || results.length === 0) { + return json( + { error: "Failed to fetch current occurrence count. Please try again." }, + { status: 500 } + ); + } + occurrenceCountAtIgnoreTime = results[0].occurrence_count; + } + + await actions.ignoreError(identifier, { + userId, + duration: submission.value.duration, + occurrenceRateThreshold: submission.value.occurrenceRate, + totalOccurrencesThreshold: submission.value.totalOccurrences, + occurrenceCountAtIgnoreTime, + reason: submission.value.reason, + }); + return json({ ok: true }); + } + case "unresolve": { + await actions.unresolveError(identifier); + return json({ ok: true }); + } + } +}; + export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const userId = user.id; @@ -82,6 +231,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const toStr = url.searchParams.get("to"); const from = fromStr ? parseInt(fromStr, 10) : undefined; const to = toStr ? parseInt(toStr, 10) : undefined; + const versions = url.searchParams.getAll("versions").filter((v) => v.length > 0); const cursor = url.searchParams.get("cursor") ?? undefined; const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; @@ -93,6 +243,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { userId, projectId: project.id, fingerprint, + versions: versions.length > 0 ? versions : undefined, period, from, to, @@ -115,9 +266,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { environment.id, fingerprint, time.from, - time.to + time.to, + versions.length > 0 ? versions : undefined ) - .catch(() => ({ data: [] as ErrorGroupActivity })); + .catch(() => ({ data: [] as ErrorGroupActivity, versions: [] as string[] })); return typeddefer({ data: detailPromise, @@ -149,10 +301,19 @@ export default function Page() { if (period) carry.set("period", period); if (from) carry.set("from", from); if (to) carry.set("to", to); + for (const v of searchParams.getAll("versions")) { + if (v) carry.append("versions", v); + } const qs = carry.toString(); return qs ? `${base}?${qs}` : base; }, [organizationSlug, projectParam, envParam, searchParams.toString()]); + const alertsHref = useMemo(() => { + const params = new URLSearchParams(location.search); + params.set("alerts", "true"); + return `?${params.toString()}`; + }, [location.search]); + return ( <> @@ -205,6 +366,7 @@ export default function Page() { projectParam={projectParam} envParam={envParam} fingerprint={fingerprint} + alertsHref={alertsHref} /> ); }} @@ -223,6 +385,7 @@ function ErrorGroupDetail({ projectParam, envParam, fingerprint, + alertsHref, }: { errorGroup: ErrorGroupSummary | undefined; runList: NextRunList | undefined; @@ -231,8 +394,9 @@ function ErrorGroupDetail({ projectParam: string; envParam: string; fingerprint: string; + alertsHref: string; }) { - const { value } = useSearchParams(); + const { value, values } = useSearchParams(); const organization = useOrganization(); const project = useProject(); const environment = useEnvironment(); @@ -252,26 +416,181 @@ function ErrorGroupDetail({ const fromValue = value("from") ?? undefined; const toValue = value("to") ?? undefined; + const selectedVersions = values("versions").filter((v) => v !== ""); const filters: TaskRunListSearchFilters = { period: value("period") ?? undefined, from: fromValue ? parseInt(fromValue, 10) : undefined, to: toValue ? parseInt(toValue, 10) : undefined, + versions: selectedVersions.length > 0 ? selectedVersions : undefined, rootOnly: false, errorId: ErrorId.toFriendlyId(fingerprint), }; return ( -
    - {/* Error Summary */} -
    -
    - {errorGroup.errorMessage} - {formatNumberCompact(errorGroup.count)} total occurrences + + {/* Main content: chart + runs */} + +
    + {/* Activity chart */} +
    +
    + + +
    + + }> + }> + {(result) => { + if (result.data.length > 0 && result.versions.length > 0) { + return ; + } + return ; + }} + + +
    + + {/* Runs Table */} +
    +
    + Runs + {runList && ( +
    + + View all runs + + + Bulk replay… + + +
    + )} +
    + {runList ? ( + 0} + filters={{ + tasks: [], + versions: selectedVersions, + statuses: [], + from: undefined, + to: undefined, + }} + runs={runList.runs} + isLoading={false} + variant="dimmed" + additionalTableState={{ errorId: ErrorId.toFriendlyId(fingerprint) }} + /> + ) : ( + + No runs found for this error. + + )} +
    +     + + {/* Right-hand detail sidebar */} + + + + +     + ); +} -
    +function ErrorDetailSidebar({ + errorGroup, + fingerprint, + alertsHref, +}: { + errorGroup: ErrorGroupSummary; + fingerprint: string; + alertsHref: string; +}) { + return ( +
    +
    + Details + + Configure alerts + +
    +
    +
    + {/* Status */} + + Error status + +
    + + +
    + + + {errorGroup.state.status === "IGNORED" && ( + + + + )} + +     +     + + {/* Error message */} + + Error + + + + ID @@ -284,9 +603,12 @@ function ErrorGroupDetail({ -     - - + + Occurrences + + {errorGroup.count.toLocaleString()} + + First seen @@ -299,14 +621,11 @@ function ErrorGroupDetail({ - - - {errorGroup.affectedVersions.length > 0 && ( - Affected versions + Versions - + {errorGroup.affectedVersions.join(", ")} @@ -315,91 +634,170 @@ function ErrorGroupDetail({
    +
    + ); +} - {/* Activity chart */} -
    -
    - -
    +function IgnoredDetails({ + state, + totalOccurrences, + className, +}: { + state: ErrorGroupState; + totalOccurrences: number; + className?: string; +}) { + if (state.status !== "IGNORED") { + return null; + } - }> - }> - {(result) => - result.data.length > 0 ? ( - - ) : ( - - ) - } - - -
    + const hasConditions = + state.ignoredUntil || state.ignoredUntilOccurrenceRate || state.ignoredUntilTotalOccurrences; - {/* Runs Table */} -
    -
    - Runs - {runList && ( -
    - - View all runs - - - Bulk replay… - - -
    - )} + const ignoredForever = !hasConditions; + + const occurrencesSinceIgnore = + state.ignoredUntilTotalOccurrences && state.ignoredAtOccurrenceCount !== null + ? totalOccurrences - state.ignoredAtOccurrenceCount + : null; + + return ( +
    +
    +
    + + + {ignoredForever ? "Ignored permanently" : "Ignored with conditions"} +
    - {runList ? ( - - ) : ( - - No runs found for this error. + {(state.ignoredByUserDisplayName || state.ignoredAt) && ( + + {state.ignoredByUserDisplayName && <>Configured by {state.ignoredByUserDisplayName}} + {state.ignoredByUserDisplayName && state.ignoredAt && " "} + {state.ignoredAt && } )}
    + + {state.ignoredReason && ( + Reason: {state.ignoredReason} + )} + + {hasConditions && ( +
    + {state.ignoredUntil && ( + + Will revert to "Unresolved" at:{" "} + + + + {isPast(state.ignoredUntil) && (expired)} + + )} + {state.ignoredUntilOccurrenceRate !== null && state.ignoredUntilOccurrenceRate > 0 && ( + + Will revert to "Unresolved" when: Occurrence rate exceeds{" "} + + {state.ignoredUntilOccurrenceRate}/min + + + )} + {state.ignoredUntilTotalOccurrences !== null && + state.ignoredUntilTotalOccurrences > 0 && ( + + Will revert to "Unresolved" when: Total occurrences exceed{" "} + + {state.ignoredUntilTotalOccurrences.toLocaleString()} + + {occurrencesSinceIgnore !== null && ( + + ({occurrencesSinceIgnore.toLocaleString()} since ignored) + + )} + + )} +
    + )}
    ); } -const activityChartConfig: ChartConfig = { - count: { - label: "Occurrences", - color: "#6366F1", - }, -}; +function ErrorStatusDropdown({ + state, + taskIdentifier, +}: { + state: ErrorGroupState; + taskIdentifier: string; +}) { + const fetcher = useFetcher<{ ok?: boolean }>(); + const revalidator = useRevalidator(); + const [popoverOpen, setPopoverOpen] = useState(false); + const [customIgnoreOpen, setCustomIgnoreOpen] = useState(false); + const isSubmitting = fetcher.state !== "idle"; + const toast = useToast(); + const pendingToast = useRef(); + + useEffect(() => { + if (fetcher.state === "idle" && fetcher.data?.ok && pendingToast.current) { + toast.success(pendingToast.current); + pendingToast.current = undefined; + revalidator.revalidate(); + } + }, [fetcher.state, fetcher.data, toast, revalidator]); + + const act = (data: Record) => { + setPopoverOpen(false); + pendingToast.current = statusActionToastMessage(data); + fetcher.submit(data, { method: "post" }); + }; + + return ( + <> + + + + Mark error as… + + + { + setPopoverOpen(false); + setCustomIgnoreOpen(true); + }} + /> + + + + + + ); +} + +function ActivityChart({ + activity, + versions, +}: { + activity: ErrorGroupActivity; + versions: ErrorGroupActivityVersions; +}) { + const ERROR_CHART_COLORS = ["#6c5ce7", "#ec4899"]; + const colors = useMemo( + () => versions.map((_, i) => ERROR_CHART_COLORS[i % ERROR_CHART_COLORS.length]), + [versions] + ); -function ActivityChart({ activity }: { activity: ErrorGroupActivity }) { const data = useMemo( () => activity.map((d) => ({ @@ -433,48 +831,91 @@ function ActivityChart({ activity }: { activity: ErrorGroupActivity }) { }; }, []); - const tooltipLabelFormatter = useMemo(() => { - return (_label: string, payload: Array<{ payload?: Record }>) => { - const timestamp = payload[0]?.payload?.__timestamp as number | undefined; - if (timestamp) { - const date = new Date(timestamp); - return date.toLocaleString("en-US", { - month: "short", - day: "numeric", - year: "numeric", - hour: "2-digit", - minute: "2-digit", - hour12: false, - }); - } - return _label; - }; - }, []); - return ( - - - + + + + + dataMax * 1.15]} + /> + } + allowEscapeViewBox={{ x: true, y: true }} + wrapperStyle={{ zIndex: 1000 }} + animationDuration={0} + /> + {versions.map((version, i) => ( + + ))} + + ); } +const ActivityTooltip = ({ + active, + payload, + versions, + colors, +}: TooltipProps & { versions: string[]; colors: string[] }) => { + if (!active || !payload?.length) return null; + + const timestamp = payload[0]?.payload?.__timestamp as number | undefined; + if (!timestamp) return null; + + const date = new Date(timestamp); + const formattedDate = date.toLocaleString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + hour: "2-digit", + minute: "2-digit", + hour12: false, + }); + + return ( + +
    + {formattedDate} +
    + {payload.map((entry, i) => { + const value = (entry.value as number) ?? 0; + return ( +
    +
    + {entry.dataKey} + {value} +
    + ); + })} +
    +
    + + ); +}; + function ActivityChartBlankState() { return (
    diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx index 2459a067902..e92b5b34644 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx @@ -1,8 +1,11 @@ -import { XMarkIcon } from "@heroicons/react/20/solid"; -import { Form, type MetaFunction } from "@remix-run/react"; +import * as Ariakit from "@ariakit/react"; +import { BellAlertIcon, XMarkIcon } from "@heroicons/react/20/solid"; +import { Form, useFetcher, useRevalidator, type MetaFunction } from "@remix-run/react"; import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { IconBugFilled } from "@tabler/icons-react"; import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; -import { Suspense, useMemo } from "react"; +import { type ErrorGroupStatus } from "@trigger.dev/database"; +import { Suspense, useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { Bar, BarChart, @@ -13,30 +16,51 @@ import { type TooltipProps, } from "recharts"; import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; +import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge"; import { PageBody } from "~/components/layout/AppLayout"; -import { SearchInput } from "~/components/primitives/SearchInput"; +import { ListPagination } from "~/components/ListPagination"; import { LogsTaskFilter } from "~/components/logs/LogsTaskFilter"; -import { Button } from "~/components/primitives/Buttons"; +import { LogsVersionFilter } from "~/components/logs/LogsVersionFilter"; +import { AppliedFilter } from "~/components/primitives/AppliedFilter"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { formatDateTime, RelativeDateTime } from "~/components/primitives/DateTime"; import { Header3 } from "~/components/primitives/Headers"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; +import { SearchInput } from "~/components/primitives/SearchInput"; +import { + ComboBox, + SelectItem, + SelectList, + SelectPopover, + SelectProvider, + SelectTrigger, +} from "~/components/primitives/Select"; import { Spinner } from "~/components/primitives/Spinner"; import { CopyableTableCell, Table, TableBody, TableCell, - TableCellChevron, + TableCellMenu, TableHeader, TableHeaderCell, TableRow, } from "~/components/primitives/Table"; +import { PopoverSectionHeader } from "~/components/primitives/Popover"; +import { + ErrorStatusMenuItems, + CustomIgnoreDialog, + statusActionToastMessage, +} from "~/components/errors/ErrorStatusMenu"; +import { useToast } from "~/components/primitives/Toast"; import TooltipPortal from "~/components/primitives/TooltipPortal"; -import { TimeFilter } from "~/components/runs/v3/SharedFilters"; +import { appliedSummary, FilterMenuProvider, TimeFilter } from "~/components/runs/v3/SharedFilters"; import { $replica } from "~/db.server"; +import { useInterval } from "~/hooks/useInterval"; import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; +import { useSearchParams } from "~/hooks/useSearchParam"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { @@ -49,7 +73,6 @@ import { import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; import { requireUser } from "~/services/session.server"; -import { ListPagination } from "~/components/ListPagination"; import { formatNumberCompact } from "~/utils/numberFormatter"; import { EnvironmentParamSchema, v3ErrorPath } from "~/utils/pathBuilder"; import { ServiceValidationError } from "~/v3/services/baseService.server"; @@ -80,6 +103,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const url = new URL(request.url); const tasks = url.searchParams.getAll("tasks").filter((t) => t.length > 0); + const versions = url.searchParams.getAll("versions").filter((v) => v.length > 0); + const statuses = url.searchParams + .getAll("status") + .filter( + (s): s is ErrorGroupStatus => s === "UNRESOLVED" || s === "RESOLVED" || s === "IGNORED" + ); const search = url.searchParams.get("search") ?? undefined; const period = url.searchParams.get("period") ?? undefined; const fromStr = url.searchParams.get("from"); @@ -101,6 +130,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { userId, projectId: project.id, tasks: tasks.length > 0 ? tasks : undefined, + versions: versions.length > 0 ? versions : undefined, + statuses: statuses.length > 0 ? statuses : undefined, search, period, from, @@ -153,6 +184,24 @@ export default function Page() { envParam, } = useTypedLoaderData(); + const revalidator = useRevalidator(); + useInterval({ + interval: 60_000, + onLoad: false, + callback: useCallback(() => { + if (revalidator.state === "idle") { + revalidator.revalidate(); + } + }, [revalidator]), + }); + + const location = useOptimisticLocation(); + const alertsHref = useMemo(() => { + const params = new URLSearchParams(location.search); + params.set("alerts", "true"); + return `?${params.toString()}`; + }, [location.search]); + return ( <> @@ -177,7 +226,11 @@ export default function Page() { resolve={data} errorElement={
    - +
    Unable to load errors. Please refresh the page or try again in a moment. @@ -193,6 +246,7 @@ export default function Page() {
    @@ -208,6 +262,7 @@ export default function Page() { list={result} defaultPeriod={defaultPeriod} retentionLimitDays={retentionLimitDays} + alertsHref={alertsHref} /> ; +const statusShortcut = { key: "s" }; + +function StatusFilter() { + const { values, del } = useSearchParams(); + const selectedStatuses = values("status"); + + if (selectedStatuses.length === 0 || selectedStatuses.every((v) => v === "")) { + return ( + + {(search, setSearch) => ( + + Status + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + /> + )} + + ); + } + + return ( + + {(search, setSearch) => ( + }> + { + const opt = errorStatusOptions.find((o) => o.value === s); + return opt ? opt.label : s; + }) + )} + onRemove={() => del(["status", "cursor", "direction"])} + variant="secondary/small" + /> + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + /> + )} + + ); +} + +function ErrorStatusDropdown({ + trigger, + clearSearchValue, + onClose, +}: { + trigger: ReactNode; + clearSearchValue: () => void; + searchValue: string; + onClose?: () => void; +}) { + const { values, replace } = useSearchParams(); + + const handleChange = (values: string[]) => { + clearSearchValue(); + replace({ + status: values.length > 0 ? values : undefined, + cursor: undefined, + direction: undefined, + }); + }; + + return ( + + {trigger} + { + if (onClose) { + onClose(); + return false; + } + return true; + }} + > + + {errorStatusOptions.map((item) => ( + + + + ))} + + + + ); +} + function FiltersBar({ list, defaultPeriod, retentionLimitDays, + alertsHref, }: { list?: ErrorsListData; defaultPeriod?: string; retentionLimitDays: number; + alertsHref: string; }) { const location = useOptimisticLocation(); const searchParams = new URLSearchParams(location.search); const hasFilters = + searchParams.has("status") || searchParams.has("tasks") || + searchParams.has("versions") || searchParams.has("search") || searchParams.has("period") || searchParams.has("from") || @@ -246,10 +415,12 @@ function FiltersBar({ return (
    -
    +
    {list ? ( <> + + ) : ( <> + + {hasFilters && ( @@ -283,7 +456,17 @@ function FiltersBar({ )}
    - {list && } +
    + + Configure alerts + + {list && } +
    ); } @@ -303,22 +486,21 @@ function ErrorsList({ }) { if (errorGroups.length === 0) { return ( -
    -
    - No errors found - - No errors have been recorded in the selected time period. - -
    +
    + + + No errors found for this time period. +
    ); } return ( - +
    ID + Status Task Error Occurrences @@ -330,7 +512,7 @@ function ErrorsList({ {errorGroups.map((errorGroup) => ( {errorGroup.fingerprint.slice(-8)} + + + {errorGroup.taskIdentifier} - {errorMessage} + {errorMessage.length > 128 ? `${errorMessage.slice(0, 128)}…` : errorMessage} - {errorGroup.count.toLocaleString()} + + {errorGroup.count.toLocaleString()} + }> }> @@ -403,33 +593,112 @@ function ErrorGroupRow({ - + - + + ); } +function ErrorActionsCell({ + errorGroup, + organizationSlug, + projectParam, + envParam, +}: { + errorGroup: ErrorGroup; + organizationSlug: string; + projectParam: string; + envParam: string; +}) { + const fetcher = useFetcher<{ ok?: boolean }>(); + const revalidator = useRevalidator(); + const [customIgnoreOpen, setCustomIgnoreOpen] = useState(false); + const toast = useToast(); + const pendingToast = useRef(); + + useEffect(() => { + if (fetcher.state === "idle" && fetcher.data?.ok && pendingToast.current) { + toast.success(pendingToast.current); + pendingToast.current = undefined; + revalidator.revalidate(); + } + }, [fetcher.state, fetcher.data, toast, revalidator]); + + const actionUrl = v3ErrorPath( + { slug: organizationSlug }, + { slug: projectParam }, + { slug: envParam }, + { fingerprint: errorGroup.fingerprint } + ); + + return ( + <> + ( + <> + +
    + { + close(); + pendingToast.current = statusActionToastMessage(data); + fetcher.submit(data, { method: "post", action: actionUrl }); + }} + onCustomIgnore={() => { + close(); + setCustomIgnoreOpen(true); + }} + /> +
    + + )} + /> + + + ); +} + function ErrorActivityGraph({ activity }: { activity: ErrorOccurrenceActivity }) { const maxCount = Math.max(...activity.map((d) => d.count)); return (
    -
    +
    } allowEscapeViewBox={{ x: true, y: true }} wrapperStyle={{ zIndex: 1000 }} animationDuration={0} /> - + {maxCount > 0 && ( @@ -470,7 +739,7 @@ const ErrorActivityTooltip = ({ active, payload }: TooltipProps) function ErrorActivityBlankState() { return ( -
    +
    {[...Array(24)].map((_, i) => (
    ))} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.connect-to-slack.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.connect-to-slack.ts new file mode 100644 index 00000000000..b8bed6b631d --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.connect-to-slack.ts @@ -0,0 +1,48 @@ +import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { prisma } from "~/db.server"; +import { redirectWithSuccessMessage } from "~/models/message.server"; +import { OrgIntegrationRepository } from "~/models/orgIntegration.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { requireUserId } from "~/services/session.server"; +import { + EnvironmentParamSchema, + v3ErrorsPath, + v3ErrorsConnectToSlackPath, +} from "~/utils/pathBuilder"; + +export async function loader({ request, params }: LoaderFunctionArgs) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + + const url = new URL(request.url); + const shouldReinstall = url.searchParams.get("reinstall") === "true"; + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const integration = await prisma.organizationIntegration.findFirst({ + where: { + service: "SLACK", + organizationId: project.organizationId, + deletedAt: null, + }, + }); + + if (integration && !shouldReinstall) { + return redirectWithSuccessMessage( + `${v3ErrorsPath({ slug: organizationSlug }, project, { slug: envParam })}?alerts`, + request, + "Successfully connected your Slack workspace" + ); + } + + return await OrgIntegrationRepository.redirectToAuthService( + "SLACK", + project.organizationId, + request, + v3ErrorsConnectToSlackPath({ slug: organizationSlug }, project, { slug: envParam }) + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors/route.tsx index f6723ddebaa..dd9a5f6d593 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors/route.tsx @@ -1,10 +1,207 @@ -import { Outlet } from "@remix-run/react"; +import { parse } from "@conform-to/zod"; +import { Outlet, useNavigate } from "@remix-run/react"; +import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { useCallback } from "react"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { PageContainer } from "~/components/layout/AppLayout"; +import { + ConfigureErrorAlerts, + ErrorAlertsFormSchema, +} from "~/components/errors/ConfigureErrorAlerts"; +import { Sheet, SheetContent } from "~/components/primitives/SheetV3"; +import { prisma } from "~/db.server"; +import { ErrorAlertChannelPresenter } from "~/presenters/v3/ErrorAlertChannelPresenter.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { requireUserId } from "~/services/session.server"; +import { env } from "~/env.server"; +import { + EnvironmentParamSchema, + v3ErrorsConnectToSlackPath, + v3ErrorsPath, +} from "~/utils/pathBuilder"; +import { + type CreateAlertChannelOptions, + CreateAlertChannelService, +} from "~/v3/services/alerts/createAlertChannel.server"; +import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; +import { useSearchParams } from "~/hooks/useSearchParam"; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response("Environment not found", { status: 404 }); + } + + const presenter = new ErrorAlertChannelPresenter(); + const alertData = await presenter.call(project.id, environment.type); + + const connectToSlackHref = v3ErrorsConnectToSlackPath({ slug: organizationSlug }, project, { + slug: envParam, + }); + + const errorsPath = v3ErrorsPath({ slug: organizationSlug }, project, { slug: envParam }); + + return typedjson({ + alertData, + projectRef: project.externalRef, + projectId: project.id, + environmentType: environment.type, + connectToSlackHref, + errorsPath, + }); +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); + + if (request.method.toUpperCase() !== "POST") { + return json({ status: 405, error: "Method Not Allowed" }, { status: 405 }); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + const formData = await request.formData(); + const submission = parse(formData, { schema: ErrorAlertsFormSchema }); + + if (!submission.value) { + return json(submission); + } + + const { emails, webhooks, slackChannel, slackIntegrationId } = submission.value; + + const emailEnabled = env.ALERT_FROM_EMAIL !== undefined && env.ALERT_RESEND_API_KEY !== undefined; + const slackEnabled = !!slackIntegrationId; + + const existingChannels = await prisma.projectAlertChannel.findMany({ + where: { + projectId: project.id, + alertTypes: { has: "ERROR_GROUP" }, + environmentTypes: { has: environment.type }, + }, + }); + + const service = new CreateAlertChannelService(); + const environmentTypes = [environment.type]; + const processedChannelIds = new Set(); + + if (emailEnabled) { + for (const email of emails) { + const options: CreateAlertChannelOptions = { + name: `Error alert to ${email}`, + alertTypes: ["ERROR_GROUP"], + environmentTypes, + deduplicationKey: `error-email:${email}:${environment.type}`, + channel: { type: "EMAIL", email }, + }; + const channel = await service.call(project.externalRef, userId, options); + processedChannelIds.add(channel.id); + } + } + + if (slackEnabled && slackChannel) { + const [channelId, channelName] = slackChannel.split("/"); + if (channelId && channelName) { + const options: CreateAlertChannelOptions = { + name: `Error alert to #${channelName}`, + alertTypes: ["ERROR_GROUP"], + environmentTypes, + deduplicationKey: `error-slack:${environment.type}`, + channel: { + type: "SLACK", + channelId, + channelName, + integrationId: slackIntegrationId, + }, + }; + const channel = await service.call(project.externalRef, userId, options); + processedChannelIds.add(channel.id); + } + } + + for (const url of webhooks) { + const options: CreateAlertChannelOptions = { + name: `Error alert to ${new URL(url).hostname}`, + alertTypes: ["ERROR_GROUP"], + environmentTypes, + deduplicationKey: `error-webhook:${url}:${environment.type}`, + channel: { type: "WEBHOOK", url }, + }; + const channel = await service.call(project.externalRef, userId, options); + processedChannelIds.add(channel.id); + } + + const editableTypes = new Set(["WEBHOOK"]); + if (emailEnabled) { + editableTypes.add("EMAIL"); + } + if (slackEnabled) { + editableTypes.add("SLACK"); + } + + const channelsToDelete = existingChannels.filter( + (ch) => + !processedChannelIds.has(ch.id) && + editableTypes.has(ch.type) && + ch.alertTypes.length === 1 && + ch.alertTypes[0] === "ERROR_GROUP" + ); + + for (const ch of channelsToDelete) { + await prisma.projectAlertChannel.delete({ where: { id: ch.id } }); + } + + return json({ ok: true }); +}; export default function Page() { + const { alertData, connectToSlackHref, errorsPath } = useTypedLoaderData(); + const { has } = useSearchParams(); + const showAlerts = has("alerts") ?? false; + const navigate = useNavigate(); + const location = useOptimisticLocation(); + + const closeAlerts = useCallback(() => { + const params = new URLSearchParams(location.search); + params.delete("alerts"); + const qs = params.toString(); + navigate(qs ? `?${qs}` : location.pathname, { replace: true }); + }, [location.search, location.pathname, navigate]); + return ( + + !open && closeAlerts()}> + e.preventDefault()} + > + + + ); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.slack.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.slack.tsx index c954a6fe697..ba11cf8f8a1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.slack.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.slack.tsx @@ -1,13 +1,14 @@ -import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/node"; -import { json, redirect } from "@remix-run/node"; +import { type ActionFunctionArgs, type LoaderFunctionArgs, json, redirect } from "@remix-run/node"; import { fromPromise } from "neverthrow"; import { Form, useActionData, useNavigation } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { DialogClose } from "@radix-ui/react-dialog"; -import { SlackIcon } from "@trigger.dev/companyicons"; import { TrashIcon } from "@heroicons/react/20/solid"; +import { IconBugFilled } from "@tabler/icons-react"; +import { SlackMonoIcon } from "~/assets/icons/SlackMonoIcon"; import { Button } from "~/components/primitives/Buttons"; +import { DateTime } from "~/components/primitives/DateTime"; import { Dialog, DialogContent, @@ -17,8 +18,14 @@ import { DialogTrigger, } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; -import { Header1 } from "~/components/primitives/Headers"; -import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { Header2, Header3 } from "~/components/primitives/Headers"; +import { Hint } from "~/components/primitives/Hint"; +import { + MainHorizontallyCenteredContainer, + PageBody, + PageContainer, +} from "~/components/layout/AppLayout"; +import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; import { Table, @@ -31,21 +38,9 @@ import { import { EnabledStatus } from "~/components/runs/v3/EnabledStatus"; import { $transaction, prisma } from "~/db.server"; import { requireOrganization } from "~/services/org.server"; -import { OrganizationParamsSchema, organizationSettingsPath } from "~/utils/pathBuilder"; +import { OrganizationParamsSchema, organizationSlackIntegrationPath } from "~/utils/pathBuilder"; import { logger } from "~/services/logger.server"; -function formatDate(date: Date): string { - return new Intl.DateTimeFormat("en-US", { - month: "short", - day: "numeric", - year: "numeric", - hour: "numeric", - minute: "2-digit", - second: "2-digit", - hour12: true, - }).format(date); -} - export const loader = async ({ request, params }: LoaderFunctionArgs) => { const { organizationSlug } = OrganizationParamsSchema.parse(params); const { organization } = await requireOrganization(request, organizationSlug); @@ -183,12 +178,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { integrationId: slackIntegration.id, }); - return redirect(organizationSettingsPath({ slug: organizationSlug })); + return redirect(organizationSlackIntegrationPath({ slug: organizationSlug })); }; export default function SlackIntegrationPage() { - const { slackIntegration, alertChannels, teamName } = - useTypedLoaderData(); + const { slackIntegration, alertChannels, teamName } = useTypedLoaderData(); const actionData = useActionData(); const navigation = useNavigation(); const isUninstalling = @@ -197,12 +191,18 @@ export default function SlackIntegrationPage() { if (!slackIntegration) { return ( + + + -
    - No Slack Integration Found - - This organization doesn't have a Slack integration configured. You can connect Slack - when setting up alert channels in your project settings. +
    + + No Slack integration found + + Your organization doesn't have a Slack integration configured. You can connect Slack + when setting up alerts from the{" "} + + Errors page.
    @@ -212,114 +212,131 @@ export default function SlackIntegrationPage() { return ( + + + -
    - Slack Integration - - Manage your organization's Slack integration and connected alert channels. - -
    - - {/* Integration Info Section */} -
    -
    + +
    -

    Integration Details

    -
    +
    + Integration details +
    +
    {teamName && ( -
    - Slack Workspace: {teamName} -
    + + Workspace:{" "} + {teamName} + )} -
    - Installed:{" "} - {formatDate(new Date(slackIntegration.createdAt))} -
    + + Installed:{" "} + + + +
    -
    - - - - - - - Remove Slack Integration - - - This will remove the Slack integration and disable all connected alert channels. - This action cannot be undone. - - - + +
    + + Connected alert channels + ({alertChannels.length}) + + {alertChannels.length === 0 ? ( + + No alert channels are currently connected to this Slack integration. + + ) : ( +
    + + + Channel + Project + Status + Created + + + + {alertChannels.map((channel) => ( + + {channel.name} + {channel.project.name} + + + + + + + + ))} + +
    + )} +
    + +
    + Danger zone +
    + Remove integration + + This will remove the Slack integration and disable all connected alert channels. + This action cannot be undone. + + {actionData?.error && ( + + {actionData.error} + + )} + + - - } - cancelButton={ - - - - } - /> - - - {actionData?.error && ( - - {actionData.error} - - )} + + + + Remove Slack integration + + + This will remove the Slack integration and disable all connected alert + channels. This action cannot be undone. + + + + + + } + cancelButton={ + + + + } + /> + + + } + /> +
    -
    - - {/* Connected Alert Channels Section */} -
    -

    - Connected Alert Channels ({alertChannels.length}) -

    - - {alertChannels.length === 0 ? ( -
    - - No alert channels are currently connected to this Slack integration. - -
    - ) : ( - - - - Channel Name - Project - Status - Created - - - - {alertChannels.map((channel) => ( - - {channel.name} - {channel.project.name} - - - - {formatDate(new Date(channel.createdAt))} - - ))} - -
    - )} -
    + ); diff --git a/apps/webapp/app/routes/storybook.unordered-list/route.tsx b/apps/webapp/app/routes/storybook.unordered-list/route.tsx new file mode 100644 index 00000000000..b17bb2dda11 --- /dev/null +++ b/apps/webapp/app/routes/storybook.unordered-list/route.tsx @@ -0,0 +1,67 @@ +import { Header2 } from "~/components/primitives/Headers"; +import { Paragraph, type ParagraphVariant } from "~/components/primitives/Paragraph"; +import { UnorderedList } from "~/components/primitives/UnorderedList"; + +const sampleItems = [ + "A new issue is seen for the first time", + "A resolved issue re-occurs", + "An ignored issue re-occurs depending on the settings you configured", +]; + +const variantGroups: { label: string; variants: ParagraphVariant[] }[] = [ + { + label: "Base", + variants: ["base", "base/bright"], + }, + { + label: "Small", + variants: ["small", "small/bright", "small/dimmed"], + }, + { + label: "Extra small", + variants: [ + "extra-small", + "extra-small/bright", + "extra-small/dimmed", + "extra-small/mono", + "extra-small/bright/mono", + "extra-small/dimmed/mono", + "extra-small/caps", + "extra-small/bright/caps", + ], + }, + { + label: "Extra extra small", + variants: [ + "extra-extra-small", + "extra-extra-small/bright", + "extra-extra-small/caps", + "extra-extra-small/bright/caps", + "extra-extra-small/dimmed/caps", + ], + }, +]; + +export default function Story() { + return ( +
    + {variantGroups.map((group) => ( +
    + {group.label} + {group.variants.map((variant) => ( +
    + {variant} + This is a paragraph before the list. + + {sampleItems.map((item) => ( +
  • {item}
    • + ))} +     + This is a paragraph after the list. +
    + ))} +
    + ))} +
    + ); +} diff --git a/apps/webapp/app/routes/storybook/route.tsx b/apps/webapp/app/routes/storybook/route.tsx index 83d455c2a55..bcaee62d6b0 100644 --- a/apps/webapp/app/routes/storybook/route.tsx +++ b/apps/webapp/app/routes/storybook/route.tsx @@ -136,6 +136,10 @@ const stories: Story[] = [ name: "Typography", slug: "typography", }, + { + name: "Unordered list", + slug: "unordered-list", + }, { name: "Usage", slug: "usage", diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index f73f4139a01..7a151053f5a 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -584,6 +584,14 @@ export function v3ErrorsPath( return `${v3EnvironmentPath(organization, project, environment)}/errors`; } +export function v3ErrorsConnectToSlackPath( + organization: OrgForPath, + project: ProjectForPath, + environment: EnvironmentForPath +) { + return `${v3ErrorsPath(organization, project, environment)}/connect-to-slack`; +} + export function v3ErrorPath( organization: OrgForPath, project: ProjectForPath, diff --git a/apps/webapp/app/v3/alertsWorker.server.ts b/apps/webapp/app/v3/alertsWorker.server.ts index 46670887a75..693b16b738a 100644 --- a/apps/webapp/app/v3/alertsWorker.server.ts +++ b/apps/webapp/app/v3/alertsWorker.server.ts @@ -1,10 +1,12 @@ import { Logger } from "@trigger.dev/core/logger"; -import { Worker as RedisWorker } from "@trigger.dev/redis-worker"; +import { CronSchema, Worker as RedisWorker } from "@trigger.dev/redis-worker"; import { z } from "zod"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; import { DeliverAlertService } from "./services/alerts/deliverAlert.server"; +import { DeliverErrorGroupAlertService } from "./services/alerts/deliverErrorGroupAlert.server"; +import { ErrorAlertEvaluator } from "./services/alerts/errorAlertEvaluator.server"; import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server"; import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; @@ -55,6 +57,42 @@ function initializeWorker() { }, logErrors: false, }, + "v3.evaluateErrorAlerts": { + schema: z.object({ + projectId: z.string(), + scheduledAt: z.number(), + }), + visibilityTimeoutMs: 60_000 * 5, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, + "v3.deliverErrorGroupAlert": { + schema: z.object({ + channelId: z.string(), + projectId: z.string(), + classification: z.enum(["new_issue", "regression", "unignored"]), + error: z.object({ + fingerprint: z.string(), + environmentId: z.string(), + environmentSlug: z.string(), + environmentName: z.string(), + taskIdentifier: z.string(), + errorType: z.string(), + errorMessage: z.string(), + sampleStackTrace: z.string(), + firstSeen: z.string(), + lastSeen: z.string(), + occurrenceCount: z.number(), + }), + }), + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, }, concurrency: { workers: env.ALERTS_WORKER_CONCURRENCY_WORKERS, @@ -80,6 +118,14 @@ function initializeWorker() { const service = new PerformTaskRunAlertsService(); await service.call(payload.runId); }, + "v3.evaluateErrorAlerts": async ({ payload }) => { + const evaluator = new ErrorAlertEvaluator(); + await evaluator.evaluate(payload.projectId, payload.scheduledAt); + }, + "v3.deliverErrorGroupAlert": async ({ payload }) => { + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + }, }, }); diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 5fe2624557d..7505693e3ab 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -1194,4 +1194,4 @@ function initializeOTLPExporter() { ? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10) : 8192 ); -} +} \ No newline at end of file diff --git a/apps/webapp/app/v3/services/alerts/createAlertChannel.server.ts b/apps/webapp/app/v3/services/alerts/createAlertChannel.server.ts index b2bbb423983..3b0a3a13360 100644 --- a/apps/webapp/app/v3/services/alerts/createAlertChannel.server.ts +++ b/apps/webapp/app/v3/services/alerts/createAlertChannel.server.ts @@ -1,12 +1,13 @@ import { - ProjectAlertChannel, - ProjectAlertType, - RuntimeEnvironmentType, + type ProjectAlertChannel, + type ProjectAlertType, + type RuntimeEnvironmentType, } from "@trigger.dev/database"; import { nanoid } from "nanoid"; import { env } from "~/env.server"; import { findProjectByRef } from "~/models/project.server"; import { encryptSecret } from "~/services/secrets/secretStore.server"; +import { alertsWorker } from "~/v3/alertsWorker.server"; import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; import { BaseService, ServiceValidationError } from "../baseService.server"; @@ -60,7 +61,7 @@ export class CreateAlertChannelService extends BaseService { : undefined; if (existingAlertChannel) { - return await this._prisma.projectAlertChannel.update({ + const updated = await this._prisma.projectAlertChannel.update({ where: { id: existingAlertChannel.id }, data: { name: options.name, @@ -68,8 +69,15 @@ export class CreateAlertChannelService extends BaseService { type: options.channel.type, properties: await this.#createProperties(options.channel), environmentTypes, + enabled: true, }, }); + + if (options.alertTypes.includes("ERROR_GROUP")) { + await this.#scheduleErrorAlertEvaluation(project.id); + } + + return updated; } const alertChannel = await this._prisma.projectAlertChannel.create({ @@ -87,9 +95,24 @@ export class CreateAlertChannelService extends BaseService { }, }); + if (options.alertTypes.includes("ERROR_GROUP")) { + await this.#scheduleErrorAlertEvaluation(project.id); + } + return alertChannel; } + async #scheduleErrorAlertEvaluation(projectId: string): Promise { + await alertsWorker.enqueue({ + id: `evaluateErrorAlerts:${projectId}`, + job: "v3.evaluateErrorAlerts", + payload: { + projectId, + scheduledAt: Date.now(), + }, + }); + } + async #createProperties(channel: CreateAlertChannelOptions["channel"]) { switch (channel.type) { case "EMAIL": diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index 8b922f91e9f..5ab99bf8046 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -319,6 +319,9 @@ export class DeliverAlertService extends BaseService { break; } + case "ERROR_GROUP": { + break; + } default: { assertNever(alert.type); } @@ -657,6 +660,9 @@ export class DeliverAlertService extends BaseService { break; } + case "ERROR_GROUP": { + break; + } default: { assertNever(alert.type); } @@ -913,6 +919,9 @@ export class DeliverAlertService extends BaseService { return; } } + case "ERROR_GROUP": { + break; + } default: { assertNever(alert.type); } diff --git a/apps/webapp/app/v3/services/alerts/deliverErrorGroupAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverErrorGroupAlert.server.ts new file mode 100644 index 00000000000..422811cdee3 --- /dev/null +++ b/apps/webapp/app/v3/services/alerts/deliverErrorGroupAlert.server.ts @@ -0,0 +1,404 @@ +import { + type ChatPostMessageArguments, + ErrorCode, + type WebAPIPlatformError, + type WebAPIRateLimitedError, +} from "@slack/web-api"; +import { type ProjectAlertChannelType } from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { v3ErrorPath } from "~/utils/pathBuilder"; +import { + isIntegrationForService, + type OrganizationIntegrationForService, + OrgIntegrationRepository, +} from "~/models/orgIntegration.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, + ProjectAlertWebhookProperties, +} from "~/models/projectAlert.server"; +import { sendAlertEmail } from "~/services/email.server"; +import { logger } from "~/services/logger.server"; +import { decryptSecret } from "~/services/secrets/secretStore.server"; +import { subtle } from "crypto"; +import { generateErrorGroupWebhookPayload } from "./errorGroupWebhook.server"; + +type ErrorAlertClassification = "new_issue" | "regression" | "unignored"; + +interface ErrorAlertPayload { + channelId: string; + projectId: string; + classification: ErrorAlertClassification; + error: { + fingerprint: string; + environmentId: string; + environmentSlug: string; + environmentName: string; + taskIdentifier: string; + errorType: string; + errorMessage: string; + sampleStackTrace: string; + firstSeen: string; + lastSeen: string; + occurrenceCount: number; + }; +} + +class SkipRetryError extends Error {} + +export class DeliverErrorGroupAlertService { + async call(payload: ErrorAlertPayload): Promise { + const channel = await prisma.projectAlertChannel.findFirst({ + where: { id: payload.channelId, enabled: true }, + include: { + project: { + include: { + organization: true, + }, + }, + }, + }); + + if (!channel) { + logger.warn("[DeliverErrorGroupAlert] Channel not found or disabled", { + channelId: payload.channelId, + }); + return; + } + + const errorLink = this.#buildErrorLink(channel.project.organization, channel.project, payload.error); + + try { + switch (channel.type) { + case "EMAIL": + await this.#sendEmail(channel, payload, errorLink); + break; + case "SLACK": + await this.#sendSlack(channel, payload, errorLink); + break; + case "WEBHOOK": + await this.#sendWebhook(channel, payload, errorLink); + break; + default: + assertNever(channel.type); + } + } catch (error) { + if (error instanceof SkipRetryError) { + logger.warn("[DeliverErrorGroupAlert] Skipping retry", { reason: (error as Error).message }); + return; + } + throw error; + } + } + + #buildErrorLink( + organization: { slug: string }, + project: { slug: string }, + error: ErrorAlertPayload["error"] + ): string { + return `${env.APP_ORIGIN}${v3ErrorPath(organization, project, { slug: error.environmentSlug }, { fingerprint: error.fingerprint })}`; + } + + #classificationLabel(classification: ErrorAlertClassification): string { + switch (classification) { + case "new_issue": + return "New error"; + case "regression": + return "Regression"; + case "unignored": + return "Error resurfaced"; + } + } + + async #sendEmail( + channel: { type: ProjectAlertChannelType; properties: unknown; project: { name: string; organization: { title: string } } }, + payload: ErrorAlertPayload, + errorLink: string + ): Promise { + const emailProperties = ProjectAlertEmailProperties.safeParse(channel.properties); + if (!emailProperties.success) { + logger.error("[DeliverErrorGroupAlert] Failed to parse email properties", { + issues: emailProperties.error.issues, + }); + return; + } + + await sendAlertEmail({ + email: "alert-error-group", + to: emailProperties.data.email, + classification: payload.classification, + taskIdentifier: payload.error.taskIdentifier, + environment: payload.error.environmentName, + error: { + message: payload.error.errorMessage, + type: payload.error.errorType, + stackTrace: payload.error.sampleStackTrace || undefined, + }, + occurrenceCount: payload.error.occurrenceCount, + errorLink, + organization: channel.project.organization.title, + project: channel.project.name, + }); + } + + async #sendSlack( + channel: { + type: ProjectAlertChannelType; + properties: unknown; + project: { organizationId: string; name: string; organization: { title: string } }; + }, + payload: ErrorAlertPayload, + errorLink: string + ): Promise { + const slackProperties = ProjectAlertSlackProperties.safeParse(channel.properties); + if (!slackProperties.success) { + logger.error("[DeliverErrorGroupAlert] Failed to parse slack properties", { + issues: slackProperties.error.issues, + }); + return; + } + + const integration = slackProperties.data.integrationId + ? await prisma.organizationIntegration.findFirst({ + where: { + id: slackProperties.data.integrationId, + organizationId: channel.project.organizationId, + }, + include: { tokenReference: true }, + }) + : await prisma.organizationIntegration.findFirst({ + where: { + service: "SLACK", + organizationId: channel.project.organizationId, + }, + orderBy: { createdAt: "desc" }, + include: { tokenReference: true }, + }); + + if (!integration || !isIntegrationForService(integration, "SLACK")) { + logger.error("[DeliverErrorGroupAlert] Slack integration not found"); + return; + } + + const message = this.#buildErrorGroupSlackMessage( + payload, + errorLink, + channel.project.name + ); + + await this.#postSlackMessage(integration, { + channel: slackProperties.data.channelId, + ...message, + } as ChatPostMessageArguments); + } + + async #sendWebhook( + channel: { + type: ProjectAlertChannelType; + properties: unknown; + project: { id: string; externalRef: string; slug: string; name: string; organizationId: string; organization: { slug: string; title: string } }; + }, + payload: ErrorAlertPayload, + errorLink: string + ): Promise { + const webhookProperties = ProjectAlertWebhookProperties.safeParse(channel.properties); + if (!webhookProperties.success) { + logger.error("[DeliverErrorGroupAlert] Failed to parse webhook properties", { + issues: webhookProperties.error.issues, + }); + return; + } + + const webhookPayload = generateErrorGroupWebhookPayload({ + classification: payload.classification, + error: payload.error, + organization: { + id: channel.project.organizationId, + slug: channel.project.organization.slug, + name: channel.project.organization.title, + }, + project: { + id: channel.project.id, + externalRef: channel.project.externalRef, + slug: channel.project.slug, + name: channel.project.name, + }, + dashboardUrl: errorLink, + }); + + const rawPayload = JSON.stringify(webhookPayload); + const hashPayload = Buffer.from(rawPayload, "utf-8"); + const secret = await decryptSecret(env.ENCRYPTION_KEY, webhookProperties.data.secret); + const hmacSecret = Buffer.from(secret, "utf-8"); + const key = await subtle.importKey( + "raw", + hmacSecret, + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + const signature = await subtle.sign("HMAC", key, hashPayload); + const signatureHex = Buffer.from(signature).toString("hex"); + + const response = await fetch(webhookProperties.data.url, { + method: "POST", + headers: { + "content-type": "application/json", + "x-trigger-signature-hmacsha256": signatureHex, + }, + body: rawPayload, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + logger.info("[DeliverErrorGroupAlert] Failed to send webhook", { + status: response.status, + statusText: response.statusText, + url: webhookProperties.data.url, + }); + throw new Error(`Failed to send error group alert webhook to ${webhookProperties.data.url}`); + } + } + + async #postSlackMessage( + integration: OrganizationIntegrationForService<"SLACK">, + message: ChatPostMessageArguments + ) { + const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration( + integration, + { forceBotToken: true } + ); + + try { + return await client.chat.postMessage({ + ...message, + unfurl_links: false, + unfurl_media: false, + }); + } catch (error) { + if (isWebAPIRateLimitedError(error)) { + throw new Error("Slack rate limited"); + } + if (isWebAPIPlatformError(error)) { + if ( + (error as WebAPIPlatformError).data.error === "invalid_blocks" || + (error as WebAPIPlatformError).data.error === "account_inactive" + ) { + throw new SkipRetryError(`Slack: ${(error as WebAPIPlatformError).data.error}`); + } + throw new Error("Slack platform error"); + } + throw error; + } + } + + #buildErrorGroupSlackMessage( + payload: ErrorAlertPayload, + errorLink: string, + projectName: string + ): { text: string; blocks: object[]; attachments: object[] } { + const label = this.#classificationLabel(payload.classification); + const errorType = payload.error.errorType || "Error"; + const task = payload.error.taskIdentifier; + const envName = payload.error.environmentName; + + return { + text: `${label}: ${errorType} in ${task} [${envName}]`, + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: `*${label} in ${task} [${envName}]*`, + }, + }, + ], + attachments: [ + { + color: "danger", + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: this.#wrapInCodeBlock( + payload.error.sampleStackTrace || payload.error.errorMessage + ), + }, + }, + { + type: "section", + fields: [ + { + type: "mrkdwn", + text: `*Task:*\n${task}`, + }, + { + type: "mrkdwn", + text: `*Environment:*\n${envName}`, + }, + { + type: "mrkdwn", + text: `*Project:*\n${projectName}`, + }, + { + type: "mrkdwn", + text: `*Occurrences:*\n${payload.error.occurrenceCount}`, + }, + { + type: "mrkdwn", + text: `*Last seen:*\n${this.#formatTimestamp(new Date(Number(payload.error.lastSeen)))}`, + }, + ], + }, + { + type: "actions", + elements: [ + { + type: "button", + text: { type: "plain_text", text: "Investigate" }, + url: errorLink, + style: "primary", + }, + ], + }, + ], + }, + ], + }; + } + + #wrapInCodeBlock(text: string, maxLength = 3000) { + const wrapperLength = 6; // ``` prefix + ``` suffix + const truncationSuffix = "\n\n...truncated — check dashboard for full error"; + const innerMax = maxLength - wrapperLength; + + const truncated = + text.length > innerMax + ? text.slice(0, innerMax - truncationSuffix.length) + truncationSuffix + : text; + return `\`\`\`${truncated}\`\`\``; + } + + #formatTimestamp(date: Date): string { + return new Intl.DateTimeFormat("en-US", { + month: "short", + day: "numeric", + year: "numeric", + hour: "numeric", + minute: "2-digit", + second: "2-digit", + hour12: true, + }).format(date); + } +} + +function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError { + return (error as WebAPIPlatformError).code === ErrorCode.PlatformError; +} + +function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError { + return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError; +} diff --git a/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts b/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts new file mode 100644 index 00000000000..e935a8a6911 --- /dev/null +++ b/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts @@ -0,0 +1,484 @@ +import { type ActiveErrorsSinceQueryResult, type ClickHouse } from "@internal/clickhouse"; +import { + type ErrorGroupState, + type PrismaClientOrTransaction, + type ProjectAlertChannel, + type RuntimeEnvironmentType, +} from "@trigger.dev/database"; +import { $replica, prisma } from "~/db.server"; +import { ErrorAlertConfig } from "~/models/projectAlert.server"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { logger } from "~/services/logger.server"; +import { alertsWorker } from "~/v3/alertsWorker.server"; + +type ErrorClassification = "new_issue" | "regression" | "unignored"; + +interface AlertableError { + classification: ErrorClassification; + error: ActiveErrorsSinceQueryResult; + environmentSlug: string; + environmentName: string; +} + +interface ResolvedEnvironment { + id: string; + slug: string; + type: RuntimeEnvironmentType; + displayName: string; +} + +const DEFAULT_INTERVAL_MS = 300_000; + +/** + * For a project evalutes whether to send error alerts + * + * Alerts are sent if an error is + * 1. A new issue + * 2. A regression (was resolved and now back) + * 3. Unignored (was ignored and is no longer) + * + * Unignored happens in 3 situations + * 1. It was ignored with a future date, and that's now in the past + * 2. It was ignored until reaching an error rate (e.g. 10/minute) and that has been exceeded + * 3. It was ignored until reaching a total occurrence count (e.g. 1,000) and that has been exceeded + */ +export class ErrorAlertEvaluator { + constructor( + protected readonly _prisma: PrismaClientOrTransaction = prisma, + protected readonly _replica: PrismaClientOrTransaction = $replica, + protected readonly _clickhouse: ClickHouse = clickhouseClient + ) {} + + async evaluate(projectId: string, scheduledAt: number): Promise { + const nextScheduledAt = Date.now(); + + const channels = await this.resolveChannels(projectId); + if (channels.length === 0) { + logger.info("[ErrorAlertEvaluator] No active ERROR_GROUP channels, self-terminating", { + projectId, + }); + return; + } + + const minIntervalMs = this.computeMinInterval(channels); + const windowMs = nextScheduledAt - scheduledAt; + + if (windowMs > minIntervalMs * 2) { + logger.info("[ErrorAlertEvaluator] Large evaluation window (gap detected)", { + projectId, + scheduledAt, + nextScheduledAt, + windowMs, + minIntervalMs, + }); + } + + const allEnvTypes = this.collectEnvironmentTypes(channels); + + try { + const [project, environments] = await Promise.all([ + this._replica.project.findFirst({ + where: { id: projectId }, + select: { organizationId: true }, + }), + this.resolveEnvironments(projectId, allEnvTypes), + ]); + + if (!project) { + logger.error("[ErrorAlertEvaluator] Project not found", { projectId }); + return; + } + + if (environments.length === 0) { + return; + } + + const envIds = environments.map((e) => e.id); + const envMap = new Map(environments.map((e) => [e.id, e])); + const channelsByEnvId = this.buildChannelsByEnvId(channels, environments); + + const activeErrors = await this.getActiveErrors( + project.organizationId, + projectId, + envIds, + scheduledAt + ); + + if (activeErrors.length === 0) { + return; + } + + const states = await this.getErrorGroupStates(activeErrors); + const stateMap = this.buildStateMap(states); + + const occurrenceCounts = await this.getOccurrenceCountsSince( + project.organizationId, + projectId, + envIds, + scheduledAt + ); + const occurrenceMap = this.buildOccurrenceMap(occurrenceCounts); + + const alertableErrors: AlertableError[] = []; + + for (const error of activeErrors) { + const key = `${error.environment_id}:${error.task_identifier}:${error.error_fingerprint}`; + const state = stateMap.get(key); + const env = envMap.get(error.environment_id); + const firstSeenMs = Number(error.first_seen); + + const classification = this.classifyError(error, state, firstSeenMs, scheduledAt, { + occurrencesSince: occurrenceMap.get(key) ?? 0, + windowMs, + totalOccurrenceCount: error.occurrence_count, + }); + + if (classification) { + alertableErrors.push({ + classification, + error, + environmentSlug: env?.slug ?? "", + environmentName: env?.displayName ?? error.environment_id, + }); + } + } + + for (const alertable of alertableErrors) { + const envChannels = channelsByEnvId.get(alertable.error.environment_id) ?? []; + for (const channel of envChannels) { + await alertsWorker.enqueue({ + id: `deliverErrorGroupAlert:${channel.id}:${alertable.error.error_fingerprint}:${scheduledAt}`, + job: "v3.deliverErrorGroupAlert", + payload: { + channelId: channel.id, + projectId, + classification: alertable.classification, + error: { + fingerprint: alertable.error.error_fingerprint, + environmentId: alertable.error.environment_id, + environmentSlug: alertable.environmentSlug, + environmentName: alertable.environmentName, + taskIdentifier: alertable.error.task_identifier, + errorType: alertable.error.error_type, + errorMessage: alertable.error.error_message, + sampleStackTrace: alertable.error.sample_stack_trace, + firstSeen: alertable.error.first_seen, + lastSeen: alertable.error.last_seen, + occurrenceCount: alertable.error.occurrence_count, + }, + }, + }); + } + } + + await this.updateErrorGroupStates( + alertableErrors, + stateMap, + project.organizationId, + projectId + ); + + logger.info("[ErrorAlertEvaluator] Evaluation complete", { + projectId, + activeErrors: activeErrors.length, + alertableErrors: alertableErrors.length, + deliveryJobsEnqueued: alertableErrors.reduce( + (sum, a) => sum + (channelsByEnvId.get(a.error.environment_id)?.length ?? 0), + 0 + ), + }); + } catch (error) { + logger.error("[ErrorAlertEvaluator] Evaluation failed, will retry on next cycle", { + projectId, + error, + }); + } finally { + await this.selfChain(projectId, nextScheduledAt, minIntervalMs); + } + } + + private classifyError( + error: ActiveErrorsSinceQueryResult, + state: ErrorGroupState | undefined, + firstSeenMs: number, + scheduledAt: number, + thresholdContext: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number } + ): ErrorClassification | null { + if (!state) { + return firstSeenMs > scheduledAt ? "new_issue" : null; + } + + switch (state.status) { + case "UNRESOLVED": + return null; + + case "RESOLVED": { + if (!state.resolvedAt) return null; + const lastSeenMs = Number(error.last_seen); + return lastSeenMs > state.resolvedAt.getTime() ? "regression" : null; + } + + case "IGNORED": + return this.isIgnoreBreached(state, thresholdContext) ? "unignored" : null; + + default: + return null; + } + } + + private isIgnoreBreached( + state: ErrorGroupState, + context: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number } + ): boolean { + if (state.ignoredUntil && state.ignoredUntil.getTime() <= Date.now()) { + return true; + } + + if ( + state.ignoredUntilOccurrenceRate !== null && + state.ignoredUntilOccurrenceRate !== undefined + ) { + const windowMinutes = Math.max(context.windowMs / 60_000, 1); + const rate = context.occurrencesSince / windowMinutes; + if (rate > state.ignoredUntilOccurrenceRate) { + return true; + } + } + + if ( + state.ignoredUntilTotalOccurrences != null && + state.ignoredAtOccurrenceCount != null + ) { + const occurrencesSinceIgnored = + context.totalOccurrenceCount - Number(state.ignoredAtOccurrenceCount); + if (occurrencesSinceIgnored >= state.ignoredUntilTotalOccurrences) { + return true; + } + } + + return false; + } + + private async resolveChannels(projectId: string): Promise { + return this._replica.projectAlertChannel.findMany({ + where: { + projectId, + alertTypes: { has: "ERROR_GROUP" }, + enabled: true, + }, + }); + } + + private computeMinInterval(channels: ProjectAlertChannel[]): number { + let min = DEFAULT_INTERVAL_MS; + for (const ch of channels) { + const config = ErrorAlertConfig.safeParse(ch.errorAlertConfig); + if (config.success) { + min = Math.min(min, config.data.evaluationIntervalMs); + } + } + return min; + } + + private collectEnvironmentTypes(channels: ProjectAlertChannel[]): RuntimeEnvironmentType[] { + const types = new Set(); + for (const ch of channels) { + for (const t of ch.environmentTypes) { + types.add(t); + } + } + return Array.from(types); + } + + private async resolveEnvironments( + projectId: string, + types: RuntimeEnvironmentType[] + ): Promise { + const envs = await this._replica.runtimeEnvironment.findMany({ + where: { + projectId, + type: { in: types }, + }, + select: { + id: true, + type: true, + slug: true, + branchName: true, + }, + }); + + return envs.map((e) => ({ + id: e.id, + slug: e.slug, + type: e.type, + displayName: e.branchName ?? e.slug, + })); + } + + private buildChannelsByEnvId( + channels: ProjectAlertChannel[], + environments: ResolvedEnvironment[] + ): Map { + const result = new Map(); + for (const env of environments) { + const matching = channels.filter((ch) => ch.environmentTypes.includes(env.type)); + if (matching.length > 0) { + result.set(env.id, matching); + } + } + return result; + } + + private async getActiveErrors( + organizationId: string, + projectId: string, + envIds: string[], + scheduledAt: number + ): Promise { + const qb = this._clickhouse.errors.activeErrorsSinceQueryBuilder(); + qb.where("organization_id = {organizationId: String}", { organizationId }); + qb.where("project_id = {projectId: String}", { projectId }); + qb.where("environment_id IN {envIds: Array(String)}", { envIds }); + qb.groupBy("environment_id, task_identifier, error_fingerprint"); + qb.having("toInt64(last_seen) > {scheduledAt: Int64}", { + scheduledAt, + }); + + const [err, results] = await qb.execute(); + if (err) { + logger.error("[ErrorAlertEvaluator] Failed to query active errors", { error: err }); + return []; + } + return results ?? []; + } + + private async getErrorGroupStates( + activeErrors: ActiveErrorsSinceQueryResult[] + ): Promise { + if (activeErrors.length === 0) return []; + + return this._replica.errorGroupState.findMany({ + where: { + OR: activeErrors.map((e) => ({ + environmentId: e.environment_id, + taskIdentifier: e.task_identifier, + errorFingerprint: e.error_fingerprint, + })), + }, + }); + } + + private buildStateMap(states: ErrorGroupState[]): Map { + const map = new Map(); + for (const s of states) { + map.set(`${s.environmentId}:${s.taskIdentifier}:${s.errorFingerprint}`, s); + } + return map; + } + + private async getOccurrenceCountsSince( + organizationId: string, + projectId: string, + envIds: string[], + scheduledAt: number + ): Promise< + Array<{ + environment_id: string; + task_identifier: string; + error_fingerprint: string; + occurrences_since: number; + }> + > { + const qb = this._clickhouse.errors.occurrenceCountsSinceQueryBuilder(); + qb.where("organization_id = {organizationId: String}", { organizationId }); + qb.where("project_id = {projectId: String}", { projectId }); + qb.where("environment_id IN {envIds: Array(String)}", { envIds }); + qb.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({scheduledAt: Int64}))", { + scheduledAt, + }); + qb.groupBy("environment_id, task_identifier, error_fingerprint"); + + const [err, results] = await qb.execute(); + if (err) { + logger.error("[ErrorAlertEvaluator] Failed to query occurrence counts", { error: err }); + return []; + } + return results ?? []; + } + + private buildOccurrenceMap( + counts: Array<{ + environment_id: string; + task_identifier: string; + error_fingerprint: string; + occurrences_since: number; + }> + ): Map { + const map = new Map(); + for (const c of counts) { + map.set( + `${c.environment_id}:${c.task_identifier}:${c.error_fingerprint}`, + c.occurrences_since + ); + } + return map; + } + + private async updateErrorGroupStates( + alertableErrors: AlertableError[], + stateMap: Map, + organizationId: string, + projectId: string + ): Promise { + for (const alertable of alertableErrors) { + const key = `${alertable.error.environment_id}:${alertable.error.task_identifier}:${alertable.error.error_fingerprint}`; + const state = stateMap.get(key); + + if (state) { + await this._prisma.errorGroupState.update({ + where: { id: state.id }, + data: { + status: "UNRESOLVED", + ignoredUntil: null, + ignoredUntilOccurrenceRate: null, + ignoredUntilTotalOccurrences: null, + ignoredAtOccurrenceCount: null, + ignoredAt: null, + ignoredReason: null, + ignoredByUserId: null, + resolvedAt: null, + resolvedInVersion: null, + resolvedBy: null, + }, + }); + } else if (alertable.classification === "new_issue") { + await this._prisma.errorGroupState.create({ + data: { + organizationId, + projectId, + environmentId: alertable.error.environment_id, + taskIdentifier: alertable.error.task_identifier, + errorFingerprint: alertable.error.error_fingerprint, + status: "UNRESOLVED", + }, + }); + } + } + } + + private async selfChain( + projectId: string, + nextScheduledAt: number, + intervalMs: number + ): Promise { + await alertsWorker.enqueue({ + id: `evaluateErrorAlerts:${projectId}`, + job: "v3.evaluateErrorAlerts", + payload: { + projectId, + scheduledAt: nextScheduledAt, + }, + availableAt: new Date(nextScheduledAt + intervalMs), + }); + } +} diff --git a/apps/webapp/app/v3/services/alerts/errorGroupWebhook.server.ts b/apps/webapp/app/v3/services/alerts/errorGroupWebhook.server.ts new file mode 100644 index 00000000000..1c0f939862c --- /dev/null +++ b/apps/webapp/app/v3/services/alerts/errorGroupWebhook.server.ts @@ -0,0 +1,74 @@ +import { nanoid } from "nanoid"; +import type { ErrorWebhook } from "@trigger.dev/core/v3/schemas"; + +export type ErrorAlertClassification = "new_issue" | "regression" | "unignored"; + +export type ErrorGroupAlertData = { + classification: ErrorAlertClassification; + error: { + fingerprint: string; + environmentId: string; + environmentName: string; + taskIdentifier: string; + errorType: string; + errorMessage: string; + sampleStackTrace: string; + firstSeen: string; + lastSeen: string; + occurrenceCount: number; + }; + organization: { + id: string; + slug: string; + name: string; + }; + project: { + id: string; + externalRef: string; + slug: string; + name: string; + }; + dashboardUrl: string; +}; + +/** + * Generates a webhook payload for an error group alert that conforms to the + * ErrorWebhook schema from @trigger.dev/core/v3/schemas + */ +export function generateErrorGroupWebhookPayload(data: ErrorGroupAlertData): ErrorWebhook { + return { + id: nanoid(), + created: new Date(), + webhookVersion: "2025-01-01", + type: "alert.error" as const, + object: { + classification: data.classification, + error: { + fingerprint: data.error.fingerprint, + type: data.error.errorType, + message: data.error.errorMessage, + stackTrace: data.error.sampleStackTrace || undefined, + firstSeen: new Date(Number(data.error.firstSeen)), + lastSeen: new Date(Number(data.error.lastSeen)), + occurrenceCount: data.error.occurrenceCount, + taskIdentifier: data.error.taskIdentifier, + }, + environment: { + id: data.error.environmentId, + name: data.error.environmentName, + }, + organization: { + id: data.organization.id, + slug: data.organization.slug, + name: data.organization.name, + }, + project: { + id: data.project.id, + ref: data.project.externalRef, + slug: data.project.slug, + name: data.project.name, + }, + dashboardUrl: data.dashboardUrl, + }, + }; +} diff --git a/apps/webapp/app/v3/services/errorGroupActions.server.ts b/apps/webapp/app/v3/services/errorGroupActions.server.ts new file mode 100644 index 00000000000..c026efe2aba --- /dev/null +++ b/apps/webapp/app/v3/services/errorGroupActions.server.ts @@ -0,0 +1,144 @@ +import { type PrismaClientOrTransaction, prisma } from "~/db.server"; + +type ErrorGroupIdentifier = { + organizationId: string; + projectId: string; + environmentId: string; + taskIdentifier: string; + errorFingerprint: string; +}; + +export class ErrorGroupActions { + constructor(private readonly _prisma: PrismaClientOrTransaction = prisma) {} + + async resolveError( + identifier: ErrorGroupIdentifier, + params: { + userId: string; + resolvedInVersion?: string; + } + ) { + const where = { + environmentId_taskIdentifier_errorFingerprint: { + environmentId: identifier.environmentId, + taskIdentifier: identifier.taskIdentifier, + errorFingerprint: identifier.errorFingerprint, + }, + }; + + const now = new Date(); + + return this._prisma.errorGroupState.upsert({ + where, + update: { + status: "RESOLVED", + resolvedAt: now, + resolvedInVersion: params.resolvedInVersion ?? null, + resolvedBy: params.userId, + ignoredUntil: null, + ignoredUntilOccurrenceRate: null, + ignoredUntilTotalOccurrences: null, + ignoredAtOccurrenceCount: null, + ignoredAt: null, + ignoredReason: null, + ignoredByUserId: null, + }, + create: { + organizationId: identifier.organizationId, + projectId: identifier.projectId, + environmentId: identifier.environmentId, + taskIdentifier: identifier.taskIdentifier, + errorFingerprint: identifier.errorFingerprint, + status: "RESOLVED", + resolvedAt: now, + resolvedInVersion: params.resolvedInVersion ?? null, + resolvedBy: params.userId, + }, + }); + } + + async ignoreError( + identifier: ErrorGroupIdentifier, + params: { + userId: string; + duration?: number; + occurrenceRateThreshold?: number; + totalOccurrencesThreshold?: number; + occurrenceCountAtIgnoreTime?: number; + reason?: string; + } + ) { + const where = { + environmentId_taskIdentifier_errorFingerprint: { + environmentId: identifier.environmentId, + taskIdentifier: identifier.taskIdentifier, + errorFingerprint: identifier.errorFingerprint, + }, + }; + + const now = new Date(); + const ignoredUntil = params.duration ? new Date(now.getTime() + params.duration) : null; + + const data = { + status: "IGNORED" as const, + ignoredAt: now, + ignoredUntil, + ignoredUntilOccurrenceRate: params.occurrenceRateThreshold ?? null, + ignoredUntilTotalOccurrences: params.totalOccurrencesThreshold ?? null, + ignoredAtOccurrenceCount: params.occurrenceCountAtIgnoreTime ?? null, + ignoredReason: params.reason ?? null, + ignoredByUserId: params.userId, + resolvedAt: null, + resolvedInVersion: null, + resolvedBy: null, + }; + + return this._prisma.errorGroupState.upsert({ + where, + update: data, + create: { + organizationId: identifier.organizationId, + projectId: identifier.projectId, + environmentId: identifier.environmentId, + taskIdentifier: identifier.taskIdentifier, + errorFingerprint: identifier.errorFingerprint, + ...data, + }, + }); + } + + async unresolveError(identifier: ErrorGroupIdentifier) { + const where = { + environmentId_taskIdentifier_errorFingerprint: { + environmentId: identifier.environmentId, + taskIdentifier: identifier.taskIdentifier, + errorFingerprint: identifier.errorFingerprint, + }, + }; + + return this._prisma.errorGroupState.upsert({ + where, + update: { + status: "UNRESOLVED", + resolvedAt: null, + resolvedInVersion: null, + resolvedBy: null, + ignoredUntil: null, + ignoredUntilOccurrenceRate: null, + ignoredUntilTotalOccurrences: null, + ignoredAtOccurrenceCount: null, + ignoredAt: null, + ignoredReason: null, + ignoredByUserId: null, + }, + create: { + organizationId: identifier.organizationId, + projectId: identifier.projectId, + environmentId: identifier.environmentId, + taskIdentifier: identifier.taskIdentifier, + errorFingerprint: identifier.errorFingerprint, + status: "UNRESOLVED", + }, + }); + } +} diff --git a/apps/webapp/tailwind.config.js b/apps/webapp/tailwind.config.js index dd053b0ac7f..d598ae83d20 100644 --- a/apps/webapp/tailwind.config.js +++ b/apps/webapp/tailwind.config.js @@ -177,6 +177,7 @@ const docs = colors.blue[500]; const bulkActions = colors.emerald[500]; const aiPrompts = colors.blue[500]; const aiMetrics = colors.green[500]; +const errors = colors.amber[500]; /** Other variables */ const radius = "0.5rem"; @@ -262,6 +263,7 @@ module.exports = { customDashboards, aiPrompts, aiMetrics, + errors, }, focusStyles: { outline: "1px solid", diff --git a/apps/webapp/test/errorGroupWebhook.test.ts b/apps/webapp/test/errorGroupWebhook.test.ts new file mode 100644 index 00000000000..a7e797685ae --- /dev/null +++ b/apps/webapp/test/errorGroupWebhook.test.ts @@ -0,0 +1,248 @@ +import { describe, test, expect } from "vitest"; +import { Webhook } from "@trigger.dev/core/v3/schemas"; +import { + generateErrorGroupWebhookPayload, + type ErrorGroupAlertData, +} from "~/v3/services/alerts/errorGroupWebhook.server"; + +function createMockAlertData(overrides: Partial = {}): ErrorGroupAlertData { + const now = Date.now(); + const earlier = now - 3600000; // 1 hour ago + + return { + classification: "new_issue", + error: { + fingerprint: "fp_test_12345", + environmentId: "env_abc123", + environmentName: "Production", + taskIdentifier: "process-payment", + errorType: "TypeError", + errorMessage: "Cannot read property 'id' of undefined", + sampleStackTrace: `TypeError: Cannot read property 'id' of undefined + at processPayment (src/tasks/payment.ts:42:15) + at Object.run (src/tasks/payment.ts:15:20)`, + firstSeen: String(earlier), + lastSeen: String(now), + occurrenceCount: 5, + }, + organization: { + id: "org_xyz789", + slug: "acme-corp", + name: "Acme Corp", + }, + project: { + id: "proj_123", + externalRef: "proj_abc", + slug: "my-project", + name: "My Project", + }, + dashboardUrl: + "https://cloud.trigger.dev/orgs/acme-corp/projects/my-project/errors/fp_test_12345", + ...overrides, + }; +} + +describe("generateErrorGroupWebhookPayload", () => { + test("generates a valid webhook payload", () => { + const alertData = createMockAlertData(); + const payload = generateErrorGroupWebhookPayload(alertData); + + expect(payload).toMatchObject({ + type: "alert.error", + object: { + classification: "new_issue", + error: { + fingerprint: "fp_test_12345", + type: "TypeError", + message: "Cannot read property 'id' of undefined", + taskIdentifier: "process-payment", + occurrenceCount: 5, + }, + environment: { + id: "env_abc123", + name: "Production", + }, + organization: { + id: "org_xyz789", + slug: "acme-corp", + name: "Acme Corp", + }, + project: { + id: "proj_123", + ref: "proj_abc", + slug: "my-project", + name: "My Project", + }, + dashboardUrl: + "https://cloud.trigger.dev/orgs/acme-corp/projects/my-project/errors/fp_test_12345", + }, + }); + + expect(payload.id).toBeDefined(); + expect(payload.created).toBeInstanceOf(Date); + expect(payload.webhookVersion).toBe("2025-01-01"); + }); + + test("payload is valid according to Webhook schema", () => { + const alertData = createMockAlertData(); + const payload = generateErrorGroupWebhookPayload(alertData); + + const parsed = Webhook.parse(payload); + expect(parsed.type).toBe("alert.error"); + }); + + test("payload can be serialized and deserialized", () => { + const alertData = createMockAlertData(); + const payload = generateErrorGroupWebhookPayload(alertData); + + // Serialize to JSON (simulating sending over HTTP) + const serialized = JSON.stringify(payload); + const deserialized = JSON.parse(serialized); + + // Verify it can still be parsed by the schema + const parsed = Webhook.parse(deserialized); + expect(parsed.type).toBe("alert.error"); + + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe("new_issue"); + expect(parsed.object.error.fingerprint).toBe("fp_test_12345"); + } + }); + + test("handles new_issue classification", () => { + const alertData = createMockAlertData({ classification: "new_issue" }); + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe("new_issue"); + } + }); + + test("handles regression classification", () => { + const alertData = createMockAlertData({ classification: "regression" }); + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe("regression"); + } + }); + + test("handles unignored classification", () => { + const alertData = createMockAlertData({ classification: "unignored" }); + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe("unignored"); + } + }); + + test("handles empty stack trace", () => { + const alertData = createMockAlertData({ + error: { + ...createMockAlertData().error, + sampleStackTrace: "", + }, + }); + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.stackTrace).toBeUndefined(); + } + }); + + test("includes stack trace when present", () => { + const stackTrace = "Error at line 42"; + const alertData = createMockAlertData({ + error: { + ...createMockAlertData().error, + sampleStackTrace: stackTrace, + }, + }); + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.stackTrace).toBe(stackTrace); + } + }); + + test("preserves date fields correctly", () => { + const firstSeen = new Date("2024-01-01T00:00:00Z"); + const lastSeen = new Date("2024-01-02T12:00:00Z"); + + const alertData = createMockAlertData({ + error: { + ...createMockAlertData().error, + firstSeen: String(firstSeen.getTime()), + lastSeen: String(lastSeen.getTime()), + }, + }); + + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.firstSeen).toEqual(firstSeen); + expect(parsed.object.error.lastSeen).toEqual(lastSeen); + } + }); + + test("handles special characters in error messages", () => { + const alertData = createMockAlertData({ + error: { + ...createMockAlertData().error, + errorMessage: "Unexpected token `<` in JSON at position 0", + sampleStackTrace: `SyntaxError: Unexpected token \`<\` in JSON + at JSON.parse () + at fetch("https://api.example.com/data?query=test&limit=10")`, + }, + }); + + const payload = generateErrorGroupWebhookPayload(alertData); + const serialized = JSON.stringify(payload); + const deserialized = JSON.parse(serialized); + const parsed = Webhook.parse(deserialized); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.message).toBe("Unexpected token `<` in JSON at position 0"); + } + }); + + test("handles unicode and emoji in error messages", () => { + const alertData = createMockAlertData({ + error: { + ...createMockAlertData().error, + errorMessage: "Failed to process emoji 🔥 in message: Hello 世界", + }, + }); + + const payload = generateErrorGroupWebhookPayload(alertData); + const serialized = JSON.stringify(payload); + const deserialized = JSON.parse(serialized); + const parsed = Webhook.parse(deserialized); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.message).toBe("Failed to process emoji 🔥 in message: Hello 世界"); + } + }); + + test("handles large occurrence counts", () => { + const alertData = createMockAlertData({ + error: { + ...createMockAlertData().error, + occurrenceCount: 999999, + }, + }); + + const payload = generateErrorGroupWebhookPayload(alertData); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.occurrenceCount).toBe(999999); + } + }); +}); diff --git a/apps/webapp/test/slackErrorAlerts.test.ts b/apps/webapp/test/slackErrorAlerts.test.ts new file mode 100644 index 00000000000..b86856adc4c --- /dev/null +++ b/apps/webapp/test/slackErrorAlerts.test.ts @@ -0,0 +1,403 @@ +import { describe, test, expect, beforeAll, afterAll } from "vitest"; +import type { PrismaClient } from "@trigger.dev/database"; + +let DeliverErrorGroupAlertService: typeof import("../app/v3/services/alerts/deliverErrorGroupAlert.server.js").DeliverErrorGroupAlertService; +let prisma: PrismaClient; +let getSecretStore: typeof import("../app/services/secrets/secretStore.server.js").getSecretStore; + +type ErrorAlertPayload = { + channelId: string; + projectId: string; + classification: "new_issue" | "regression" | "unignored"; + error: { + fingerprint: string; + environmentId: string; + environmentSlug: string; + environmentName: string; + taskIdentifier: string; + errorType: string; + errorMessage: string; + sampleStackTrace: string; + firstSeen: string; + lastSeen: string; + occurrenceCount: number; + }; +}; + +let testChannelId: string; +let testProjectId: string; +let testOrganizationId: string; +let testSecretKey: string; +let testSecretReferenceId: string; + +// Helper to create mock error payloads +function createMockErrorPayload( + overrides: Partial> & { + error?: Partial; + } = {} +): ErrorAlertPayload { + const { error: errorOverrides, ...payloadOverrides } = overrides; + + const defaultError: ErrorAlertPayload["error"] = { + fingerprint: "fp_test_" + Date.now(), + environmentId: "env_test_dev", + environmentSlug: "dev", + environmentName: "Development", + taskIdentifier: "process-payment", + errorType: "TypeError", + errorMessage: "Cannot read property 'id' of undefined", + sampleStackTrace: `TypeError: Cannot read property 'id' of undefined + at processPayment (src/tasks/payment.ts:42:15) + at Object.run (src/tasks/payment.ts:15:20) + at TaskExecutor.execute (node_modules/@trigger.dev/core/dist/index.js:234:18)`, + firstSeen: Date.now().toString(), + lastSeen: Date.now().toString(), + occurrenceCount: 42, + ...errorOverrides, + }; + + return { + channelId: testChannelId, + projectId: testProjectId, + classification: "new_issue", + ...payloadOverrides, + error: defaultError, + }; +} + +// Skip tests if Slack credentials not configured +const hasSlackCredentials = + !!process.env.TEST_SLACK_CHANNEL_ID && + !!process.env.TEST_SLACK_BOT_TOKEN; + +describe.skipIf(!hasSlackCredentials)("Slack Error Alert Visual Tests", () => { + beforeAll(async () => { + const dbModule = await import("../app/db.server.js"); + prisma = dbModule.prisma; + const secretModule = await import("../app/services/secrets/secretStore.server.js"); + getSecretStore = secretModule.getSecretStore; + const alertModule = await import( + "../app/v3/services/alerts/deliverErrorGroupAlert.server.js" + ); + DeliverErrorGroupAlertService = alertModule.DeliverErrorGroupAlertService; + + const organization = await prisma.organization.create({ + data: { + title: "Slack Test Org", + slug: "slack-test-org-" + Date.now(), + }, + }); + testOrganizationId = organization.id; + + // Create test project + const project = await prisma.project.create({ + data: { + name: "Slack Test Project", + slug: "slack-test-project-" + Date.now(), + externalRef: "proj_slack_test_" + Date.now(), + organizationId: organization.id, + }, + }); + testProjectId = project.id; + + const secretStore = getSecretStore("DATABASE"); + testSecretKey = `slack-test-token-${Date.now()}`; + + await secretStore.setSecret(testSecretKey, { + botAccessToken: process.env.TEST_SLACK_BOT_TOKEN!, + }); + + const secretReference = await prisma.secretReference.create({ + data: { + key: testSecretKey, + provider: "DATABASE", + }, + }); + testSecretReferenceId = secretReference.id; + + // Create Slack organization integration + const integration = await prisma.organizationIntegration.create({ + data: { + friendlyId: "integration_test_" + Date.now(), + organizationId: organization.id, + service: "SLACK", + integrationData: {}, + tokenReferenceId: secretReference.id, + }, + }); + + // Create alert channel + const channel = await prisma.projectAlertChannel.create({ + data: { + friendlyId: "channel_test_" + Date.now(), + name: "Test Slack Channel", + type: "SLACK", + enabled: true, + projectId: project.id, + integrationId: integration.id, + properties: { + channelId: process.env.TEST_SLACK_CHANNEL_ID!, + channelName: "test-slack-alerts", + integrationId: integration.id, + }, + }, + }); + testChannelId = channel.id; + }); + + afterAll(async () => { + if (testChannelId) { + await prisma.projectAlertChannel.deleteMany({ + where: { id: testChannelId }, + }); + } + if (testOrganizationId) { + await prisma.organizationIntegration.deleteMany({ + where: { organizationId: testOrganizationId }, + }); + } + if (testSecretReferenceId) { + await prisma.secretReference.deleteMany({ + where: { id: testSecretReferenceId }, + }); + } + if (testSecretKey) { + const secretStore = getSecretStore("DATABASE"); + await secretStore.deleteSecret(testSecretKey); + } + if (testProjectId) { + await prisma.project.deleteMany({ + where: { id: testProjectId }, + }); + } + if (testOrganizationId) { + await prisma.organization.deleteMany({ + where: { id: testOrganizationId }, + }); + } + }); + + test("new_issue classification", async () => { + const payload = createMockErrorPayload({ + classification: "new_issue", + error: { + taskIdentifier: "process-order", + errorMessage: "Failed to process order due to invalid payment method", + errorType: "PaymentError", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + // Message sent - check Slack channel visually + expect(true).toBe(true); + }); + + test("regression classification", async () => { + const payload = createMockErrorPayload({ + classification: "regression", + error: { + taskIdentifier: "send-email", + errorMessage: "SMTP connection timeout after 30 seconds", + errorType: "TimeoutError", + occurrenceCount: 156, + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("unignored (resurfaced) classification", async () => { + const payload = createMockErrorPayload({ + classification: "unignored", + error: { + taskIdentifier: "sync-database", + errorMessage: "Connection pool exhausted", + errorType: "DatabaseError", + occurrenceCount: 99, + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("short error message", async () => { + const payload = createMockErrorPayload({ + error: { + errorMessage: "Not found", + errorType: "NotFoundError", + sampleStackTrace: "NotFoundError: Not found\n at findUser (src/db.ts:10:5)", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("long stack trace", async () => { + const longStackTrace = `ReferenceError: processData is not defined + at handler (src/tasks/data-processor.ts:125:15) + at async TaskRunner.execute (node_modules/@trigger.dev/sdk/dist/runner.js:89:12) + at async WorkerThread.processTask (node_modules/@trigger.dev/sdk/dist/worker.js:234:18) + at async WorkerPool.run (src/lib/worker-pool.ts:56:10) + at async TaskQueue.dequeue (src/lib/queue.ts:142:8) + at async Orchestrator.processNextTask (src/orchestrator.ts:98:5) + at async Orchestrator.start (src/orchestrator.ts:45:7) + at async main (src/index.ts:12:3) + at Object. (src/index.ts:20:1) + at Module._compile (node:internal/modules/cjs/loader:1376:14) + at Module._extensions..js (node:internal/modules/cjs/loader:1435:10) + at Module.load (node:internal/modules/cjs/loader:1207:32) + at Module._load (node:internal/modules/cjs/loader:1023:12) + at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:135:12) + at node:internal/main/run_main_module:28:49`; + + const payload = createMockErrorPayload({ + error: { + errorType: "ReferenceError", + errorMessage: "processData is not defined", + sampleStackTrace: longStackTrace, + taskIdentifier: "data-processor", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("very long error message (triggers truncation)", async () => { + // Create a message that's over 3000 characters + const longMessage = "x".repeat(3500); + const longStackTrace = `Error: ${longMessage} + at verylongfunctionname (src/tasks/long-task.ts:1:1) + ${" at stackframe (file.ts:1:1)\n".repeat(100)}`; + + const payload = createMockErrorPayload({ + error: { + errorMessage: longMessage, + sampleStackTrace: longStackTrace, + taskIdentifier: "long-error-task", + errorType: "Error", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + // Should see truncation message in Slack + expect(true).toBe(true); + }); + + test("special characters in error", async () => { + const payload = createMockErrorPayload({ + error: { + errorMessage: "Unexpected token `<` in JSON at position 0", + errorType: "SyntaxError", + sampleStackTrace: `SyntaxError: Unexpected token \`<\` in JSON at position 0 + at JSON.parse () + at parseResponse (src/api/client.ts:42:15) + at fetch("https://api.example.com/data?query=test&limit=10")`, + taskIdentifier: "api-fetch-task", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("unicode and emoji in error", async () => { + const payload = createMockErrorPayload({ + error: { + errorMessage: "Failed to process emoji 🔥 in message: Hello 世界", + errorType: "EncodingError", + sampleStackTrace: `EncodingError: Failed to process emoji 🔥 in message: Hello 世界 + at encodeMessage (src/utils/encoding.ts:15:10) + at sendMessage (src/tasks/messaging.ts:42:8)`, + taskIdentifier: "messaging-task", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("different error types - TypeError", async () => { + const payload = createMockErrorPayload({ + error: { + errorType: "TypeError", + errorMessage: "Cannot call method 'map' on undefined", + sampleStackTrace: `TypeError: Cannot call method 'map' on undefined + at transformData (src/transformers/data.ts:18:25)`, + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("different error types - ReferenceError", async () => { + const payload = createMockErrorPayload({ + error: { + errorType: "ReferenceError", + errorMessage: "userConfig is not defined", + sampleStackTrace: `ReferenceError: userConfig is not defined + at initializeApp (src/app.ts:32:10)`, + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("different error types - Custom Error", async () => { + const payload = createMockErrorPayload({ + error: { + errorType: "InvalidConfigurationError", + errorMessage: "API key is missing or invalid", + sampleStackTrace: `InvalidConfigurationError: API key is missing or invalid + at validateConfig (src/config/validator.ts:45:11) + at loadConfig (src/config/loader.ts:23:5)`, + taskIdentifier: "config-loader", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); + + test("error with no stack trace", async () => { + const payload = createMockErrorPayload({ + error: { + errorMessage: "An unknown error occurred", + errorType: "Error", + sampleStackTrace: "", + }, + }); + + const service = new DeliverErrorGroupAlertService(); + await service.call(payload); + + expect(true).toBe(true); + }); +}); diff --git a/apps/webapp/test/webhookErrorAlerts.test.ts b/apps/webapp/test/webhookErrorAlerts.test.ts new file mode 100644 index 00000000000..d0e3e9e1a89 --- /dev/null +++ b/apps/webapp/test/webhookErrorAlerts.test.ts @@ -0,0 +1,128 @@ +import { describe, test, expect } from "vitest"; +import { Webhook } from "@trigger.dev/core/v3/schemas"; +import { generateErrorGroupWebhookPayload } from "~/v3/services/alerts/errorGroupWebhook.server"; + +type ErrorData = { + fingerprint: string; + environmentId: string; + environmentName: string; + taskIdentifier: string; + errorType: string; + errorMessage: string; + sampleStackTrace: string; + firstSeen: string; + lastSeen: string; + occurrenceCount: number; +}; + +const TEST_ORG = { id: "org_test_123", slug: "webhook-test-org", name: "Webhook Test Org" }; +const TEST_PROJECT = { + id: "proj_test_456", + externalRef: "proj_webhook_test", + slug: "webhook-test-project", + name: "Webhook Test Project", +}; +const DASHBOARD_URL = "https://cloud.trigger.dev/test"; + +function createMockError(overrides: Partial = {}): ErrorData { + return { + fingerprint: "fp_test_default", + environmentId: "env_test_dev", + environmentName: "Development", + taskIdentifier: "process-payment", + errorType: "TypeError", + errorMessage: "Cannot read property 'id' of undefined", + sampleStackTrace: `TypeError: Cannot read property 'id' of undefined + at processPayment (src/tasks/payment.ts:42:15) + at Object.run (src/tasks/payment.ts:15:20) + at TaskExecutor.execute (node_modules/@trigger.dev/core/dist/index.js:234:18)`, + firstSeen: Date.now().toString(), + lastSeen: Date.now().toString(), + occurrenceCount: 42, + ...overrides, + }; +} + +function buildPayload(classification: "new_issue" | "regression" | "unignored", error: ErrorData) { + return generateErrorGroupWebhookPayload({ + classification, + error, + organization: TEST_ORG, + project: TEST_PROJECT, + dashboardUrl: DASHBOARD_URL, + }); +} + +describe("Webhook Error Alert Payload", () => { + test("payload structure is valid and parseable", () => { + const payload = buildPayload("new_issue", createMockError()); + const parsed = Webhook.parse(payload); + + expect(parsed.type).toBe("alert.error"); + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe("new_issue"); + expect(parsed.object.error.type).toBe("TypeError"); + expect(parsed.object.organization.slug).toBe("webhook-test-org"); + expect(parsed.object.project.ref).toBe("proj_webhook_test"); + } + }); + + test("payload survives JSON round-trip", () => { + const error = createMockError(); + const payload = buildPayload("regression", error); + + const deserialized = JSON.parse(JSON.stringify(payload)); + const parsed = Webhook.parse(deserialized); + + expect(parsed.type).toBe("alert.error"); + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe("regression"); + expect(parsed.object.error.fingerprint).toBe(error.fingerprint); + } + }); + + test("all classifications are valid", () => { + const classifications = ["new_issue", "regression", "unignored"] as const; + + for (const classification of classifications) { + const payload = buildPayload(classification, createMockError()); + const parsed = Webhook.parse(payload); + if (parsed.type === "alert.error") { + expect(parsed.object.classification).toBe(classification); + } + } + }); + + test("error details are preserved", () => { + const error = createMockError({ + fingerprint: "fp_custom_123", + errorType: "CustomError", + errorMessage: "Custom error message", + sampleStackTrace: "CustomError: at line 42", + taskIdentifier: "my-custom-task", + occurrenceCount: 999, + }); + + const payload = buildPayload("new_issue", error); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.fingerprint).toBe("fp_custom_123"); + expect(parsed.object.error.type).toBe("CustomError"); + expect(parsed.object.error.message).toBe("Custom error message"); + expect(parsed.object.error.stackTrace).toBe("CustomError: at line 42"); + expect(parsed.object.error.taskIdentifier).toBe("my-custom-task"); + expect(parsed.object.error.occurrenceCount).toBe(999); + } + }); + + test("empty stack trace becomes undefined", () => { + const error = createMockError({ sampleStackTrace: "" }); + const payload = buildPayload("new_issue", error); + const parsed = Webhook.parse(payload); + + if (parsed.type === "alert.error") { + expect(parsed.object.error.stackTrace).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/clickhouse/src/errors.ts b/internal-packages/clickhouse/src/errors.ts index c93efbcaf1f..4b13ce18c80 100644 --- a/internal-packages/clickhouse/src/errors.ts +++ b/internal-packages/clickhouse/src/errors.ts @@ -94,8 +94,8 @@ export function getErrorGroups(ch: ClickhouseReader, settings?: ClickHouseSettin AND project_id = {projectId: String} AND environment_id = {environmentId: String} GROUP BY error_fingerprint, task_identifier - HAVING max(last_seen) >= now() - INTERVAL {days: Int64} DAY - ORDER BY last_seen DESC + HAVING toInt64(last_seen) >= toInt64(toUnixTimestamp(now() - INTERVAL {days: Int64} DAY)) * 1000 + ORDER BY toInt64(last_seen) DESC LIMIT {limit: Int64} OFFSET {offset: Int64} `, @@ -314,3 +314,148 @@ export function createErrorOccurrencesQueryBuilder( settings ); } + +export const ErrorOccurrencesByVersionQueryResult = z.object({ + error_fingerprint: z.string(), + task_version: z.string(), + bucket_epoch: z.number(), + count: z.number(), +}); + +export type ErrorOccurrencesByVersionQueryResult = z.infer< + typeof ErrorOccurrencesByVersionQueryResult +>; + +/** + * Creates a query builder for bucketed error occurrence counts grouped by task_version. + * Used for stacked-by-version activity charts on the error detail page. + */ +export function createErrorOccurrencesByVersionQueryBuilder( + ch: ClickhouseReader, + intervalExpr: string, + settings?: ClickHouseSettings +): ClickhouseQueryBuilder { + return new ClickhouseQueryBuilder( + "getErrorOccurrencesByVersion", + ` + SELECT + error_fingerprint, + task_version, + toUnixTimestamp(toStartOfInterval(minute, ${intervalExpr})) as bucket_epoch, + sum(count) as count + FROM trigger_dev.error_occurrences_v1 + `, + ch, + ErrorOccurrencesByVersionQueryResult, + settings + ); +} + +// --------------------------------------------------------------------------- +// Alert evaluator – active errors since a timestamp +// --------------------------------------------------------------------------- + +export const ActiveErrorsSinceQueryResult = z.object({ + environment_id: z.string(), + task_identifier: z.string(), + error_fingerprint: z.string(), + error_type: z.string(), + error_message: z.string(), + sample_stack_trace: z.string(), + first_seen: z.string(), + last_seen: z.string(), + occurrence_count: z.number(), +}); + +export type ActiveErrorsSinceQueryResult = z.infer; + +/** + * Query builder for fetching all errors active since a given timestamp. + * Returns errors with last_seen > scheduledAt, grouped by env/task/fingerprint. + * Used by the error alert evaluator to find new issues, regressions, and un-ignored errors. + */ +export function getActiveErrorsSinceQueryBuilder( + ch: ClickhouseReader, + settings?: ClickHouseSettings +) { + return ch.queryBuilder({ + name: "getActiveErrorsSince", + baseQuery: ` + SELECT + environment_id, + task_identifier, + error_fingerprint, + any(error_type) as error_type, + any(error_message) as error_message, + any(sample_stack_trace) as sample_stack_trace, + toString(toUnixTimestamp64Milli(min(first_seen))) as first_seen, + toString(toUnixTimestamp64Milli(max(last_seen))) as last_seen, + toUInt64(sumMerge(occurrence_count)) as occurrence_count + FROM trigger_dev.errors_v1 + `, + schema: ActiveErrorsSinceQueryResult, + settings, + }); +} + +export const OccurrenceCountsSinceQueryResult = z.object({ + environment_id: z.string(), + task_identifier: z.string(), + error_fingerprint: z.string(), + occurrences_since: z.number(), +}); + +export type OccurrenceCountsSinceQueryResult = z.infer; + +/** + * Query builder for occurrence counts since a given timestamp, grouped by error. + * Used by the alert evaluator to check ignore thresholds. + */ +export function getOccurrenceCountsSinceQueryBuilder( + ch: ClickhouseReader, + settings?: ClickHouseSettings +) { + return ch.queryBuilder({ + name: "getOccurrenceCountsSince", + baseQuery: ` + SELECT + environment_id, + task_identifier, + error_fingerprint, + sum(count) as occurrences_since + FROM trigger_dev.error_occurrences_v1 + `, + schema: OccurrenceCountsSinceQueryResult, + settings, + }); +} + +// --------------------------------------------------------------------------- +// Alert evaluator helpers – occurrence rate & count since timestamp +// --------------------------------------------------------------------------- + +export const ErrorOccurrenceTotalCountResult = z.object({ + total_count: z.number(), +}); + +export type ErrorOccurrenceTotalCountResult = z.infer; + +/** + * Query builder for summing occurrences since a given timestamp. + * Used by the alert evaluator to check total-count-based ignore thresholds. + */ +export function getOccurrenceCountSinceQueryBuilder( + ch: ClickhouseReader, + settings?: ClickHouseSettings +) { + return ch.queryBuilder({ + name: "getOccurrenceCountSince", + baseQuery: ` + SELECT + sum(count) as total_count + FROM trigger_dev.error_occurrences_v1 + `, + schema: ErrorOccurrenceTotalCountResult, + settings, + }); +} diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 99d22a5a18e..c6b8858fa9c 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -40,7 +40,11 @@ import { getErrorHourlyOccurrences, getErrorOccurrencesListQueryBuilder, createErrorOccurrencesQueryBuilder, + createErrorOccurrencesByVersionQueryBuilder, getErrorAffectedVersionsQueryBuilder, + getOccurrenceCountSinceQueryBuilder, + getActiveErrorsSinceQueryBuilder, + getOccurrenceCountsSinceQueryBuilder, } from "./errors.js"; export { msToClickHouseInterval } from "./intervals.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; @@ -273,6 +277,11 @@ export class ClickHouse { occurrencesListQueryBuilder: getErrorOccurrencesListQueryBuilder(this.reader), createOccurrencesQueryBuilder: (intervalExpr: string) => createErrorOccurrencesQueryBuilder(this.reader, intervalExpr), + createOccurrencesByVersionQueryBuilder: (intervalExpr: string) => + createErrorOccurrencesByVersionQueryBuilder(this.reader, intervalExpr), + occurrenceCountSinceQueryBuilder: getOccurrenceCountSinceQueryBuilder(this.reader), + activeErrorsSinceQueryBuilder: getActiveErrorsSinceQueryBuilder(this.reader), + occurrenceCountsSinceQueryBuilder: getOccurrenceCountsSinceQueryBuilder(this.reader), }; } } diff --git a/internal-packages/database/prisma/migrations/20260306102053_error_group_state/migration.sql b/internal-packages/database/prisma/migrations/20260306102053_error_group_state/migration.sql new file mode 100644 index 00000000000..0510505b6ae --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260306102053_error_group_state/migration.sql @@ -0,0 +1,53 @@ +-- CreateEnum +CREATE TYPE "public"."ErrorGroupStatus" AS ENUM ('UNRESOLVED', 'RESOLVED', 'IGNORED'); + +-- AlterEnum +ALTER TYPE "public"."ProjectAlertType" ADD VALUE IF NOT EXISTS 'ERROR_GROUP'; + +-- CreateTable +CREATE TABLE + "public"."ErrorGroupState" ( + "id" TEXT NOT NULL, + "organizationId" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "environmentId" TEXT, + "taskIdentifier" TEXT NOT NULL, + "errorFingerprint" TEXT NOT NULL, + "status" "public"."ErrorGroupStatus" NOT NULL DEFAULT 'UNRESOLVED', + "ignoredUntil" TIMESTAMP(3), + "ignoredUntilOccurrenceRate" INTEGER, + "ignoredUntilTotalOccurrences" INTEGER, + "ignoredAtOccurrenceCount" BIGINT, + "ignoredAt" TIMESTAMP(3), + "ignoredReason" TEXT, + "ignoredByUserId" TEXT, + "resolvedAt" TIMESTAMP(3), + "resolvedInVersion" TEXT, + "resolvedBy" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + CONSTRAINT "ErrorGroupState_pkey" PRIMARY KEY ("id") + ); + +-- CreateIndex +CREATE UNIQUE INDEX "ErrorGroupState_environmentId_taskIdentifier_errorFingerpri_key" ON "public"."ErrorGroupState" ( + "environmentId", + "taskIdentifier", + "errorFingerprint" +); + +-- CreateIndex +CREATE INDEX "ErrorGroupState_environmentId_status_idx" ON "public"."ErrorGroupState" ("environmentId", "status"); + +-- AddForeignKey +ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "public"."Organization" ("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."Project" ("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "public"."RuntimeEnvironment" ("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AlterTable +ALTER TABLE "public"."ProjectAlertChannel" +ADD COLUMN "errorAlertConfig" JSONB; \ No newline at end of file diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index a0ff9aee690..1de0aaf1ddf 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -61,11 +61,10 @@ model User { backupCodes MfaBackupCode[] bulkActions BulkActionGroup[] - impersonationsPerformed ImpersonationAuditLog[] @relation("ImpersonationAdmin") - impersonationsReceived ImpersonationAuditLog[] @relation("ImpersonationTarget") - customerQueries CustomerQuery[] - metricsDashboards MetricsDashboard[] - + impersonationsPerformed ImpersonationAuditLog[] @relation("ImpersonationAdmin") + impersonationsReceived ImpersonationAuditLog[] @relation("ImpersonationTarget") + customerQueries CustomerQuery[] + metricsDashboards MetricsDashboard[] platformNotifications PlatformNotification[] platformNotificationInteractions PlatformNotificationInteraction[] } @@ -233,7 +232,8 @@ model Organization { metricsDashboards MetricsDashboard[] prompts Prompt[] - platformNotifications PlatformNotification[] + platformNotifications PlatformNotification[] + errorGroupStates ErrorGroupState[] } model OrgMember { @@ -353,6 +353,7 @@ model RuntimeEnvironment { BulkActionGroup BulkActionGroup[] customerQueries CustomerQuery[] prompts Prompt[] + errorGroupStates ErrorGroupState[] @@unique([projectId, slug, orgMemberId]) @@unique([projectId, shortcode]) @@ -426,8 +427,8 @@ model Project { metricsDashboards MetricsDashboard[] llmModels LlmModel[] prompts Prompt[] - platformNotifications PlatformNotification[] + errorGroupStates ErrorGroupState[] } enum ProjectVersion { @@ -2116,6 +2117,8 @@ model ProjectAlertChannel { alertTypes ProjectAlertType[] environmentTypes RuntimeEnvironmentType[] @default([STAGING, PRODUCTION]) + errorAlertConfig Json? + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String @@ -2170,6 +2173,7 @@ enum ProjectAlertType { TASK_RUN_ATTEMPT DEPLOYMENT_FAILURE DEPLOYMENT_SUCCESS + ERROR_GROUP } enum ProjectAlertStatus { @@ -2839,3 +2843,83 @@ model PlatformNotificationInteraction { @@unique([notificationId, userId]) } + +enum ErrorGroupStatus { + UNRESOLVED + RESOLVED + IGNORED +} + +/** + * Error group state is used to track when a user has interacted with an error (ignored/resolved) + * The actual error data is in ClickHouse. + */ +model ErrorGroupState { + id String @id @default(cuid()) + + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade, onUpdate: Cascade) + organizationId String + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + /** + * You can ignore/resolve an error across all environments, or specific ones + */ + environment RuntimeEnvironment? @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + environmentId String? + + taskIdentifier String + errorFingerprint String + + status ErrorGroupStatus @default(UNRESOLVED) + + /** + * Error is ignored until this date + */ + ignoredUntil DateTime? + /** + * Error is ignored until this occurrence rate + */ + ignoredUntilOccurrenceRate Int? + /** + * Error is ignored until this total occurrences + */ + ignoredUntilTotalOccurrences Int? + + /// Total occurrence count at the time the error was ignored (from ClickHouse). + /// Used with ignoredUntilTotalOccurrences to compute occurrences since ignoring. + ignoredAtOccurrenceCount BigInt? + + /** + * Error was ignored at this date + */ + ignoredAt DateTime? + /** + * Reason for ignoring the error + */ + ignoredReason String? + /** + * User who ignored the error + */ + ignoredByUserId String? + + /** + * Error was resolved at this date + */ + resolvedAt DateTime? + /** + * Error was resolved in this version + */ + resolvedInVersion String? + /** + * User who resolved the error + */ + resolvedBy String? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([environmentId, taskIdentifier, errorFingerprint]) + @@index([environmentId, status]) +} diff --git a/internal-packages/emails/emails/alert-error-group.tsx b/internal-packages/emails/emails/alert-error-group.tsx new file mode 100644 index 00000000000..f584f06edba --- /dev/null +++ b/internal-packages/emails/emails/alert-error-group.tsx @@ -0,0 +1,114 @@ +import { + Body, + CodeBlock, + Container, + Head, + Html, + Link, + Preview, + Text, + dracula, +} from "@react-email/components"; +import { z } from "zod"; +import { Footer } from "./components/Footer"; +import { Image } from "./components/Image"; +import { anchor, container, h1, main, paragraphLight, paragraphTight } from "./components/styles"; +import React from "react"; + +export const AlertErrorGroupEmailSchema = z.object({ + email: z.literal("alert-error-group"), + classification: z.enum(["new_issue", "regression", "unignored"]), + taskIdentifier: z.string(), + environment: z.string(), + error: z.object({ + message: z.string(), + type: z.string().optional(), + stackTrace: z.string().optional(), + }), + occurrenceCount: z.number(), + errorLink: z.string().url(), + organization: z.string(), + project: z.string(), +}); + +type AlertErrorGroupEmailProps = z.infer; + +const classificationLabels: Record = { + new_issue: "New error", + regression: "Regression", + unignored: "Error resurfaced", +}; + +const previewDefaults: AlertErrorGroupEmailProps = { + email: "alert-error-group", + classification: "new_issue", + taskIdentifier: "my-task", + environment: "Production", + error: { + message: "Cannot read property 'foo' of undefined", + type: "TypeError", + stackTrace: "TypeError: Cannot read property 'foo' of undefined\n at Object.", + }, + occurrenceCount: 42, + errorLink: "https://trigger.dev", + organization: "my-organization", + project: "my-project", +}; + +export default function Email(props: AlertErrorGroupEmailProps) { + const { + classification, + taskIdentifier, + environment, + error, + occurrenceCount, + errorLink, + organization, + project, + } = { + ...previewDefaults, + ...props, + }; + + const label = classificationLabels[classification] ?? "Error alert"; + + return ( + + + + {`${organization}: [${label}] ${error.type ?? "Error"} in ${taskIdentifier} (${environment})`} + + + + + {label}: {error.type ?? "Error"} in {taskIdentifier} + + Organization: {organization} + Project: {project} + Task: {taskIdentifier} + Environment: {environment} + Occurrences: {occurrenceCount} + + {error.message} + {error.stackTrace && ( + + )} + + Investigate this error + + + Trigger.dev +