diff --git a/.changeset/config.json b/.changeset/config.json index 115f54fefe..f135929f2c 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -14,10 +14,8 @@ "updateInternalDependencies": "patch", "ignore": [ "webapp", - "coordinator", - "docker-provider", - "kubernetes-provider", - "supervisor" + "supervisor", + "@trigger.dev/plugins" ], "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": { "onlyUpdatePeerDependentsWhenOutOfRange": true diff --git a/.changeset/fair-queue-concurrency-slot-leak.md b/.changeset/fair-queue-concurrency-slot-leak.md new file mode 100644 index 0000000000..41f41e7e3d --- /dev/null +++ b/.changeset/fair-queue-concurrency-slot-leak.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/redis-worker": patch +--- + +Fair queue tenants can no longer get permanently stuck behind leaked concurrency slots. Slots are now freed on every path that finishes a message, a failed release no longer causes a message to run twice or lose its retry, and a background sweep frees any slot that does leak, so a tenant's queues recover on their own instead of needing manual cleanup. diff --git a/.changeset/fluffy-crews-rhyme.md b/.changeset/fluffy-crews-rhyme.md deleted file mode 100644 index ba2a2e8bf7..0000000000 --- a/.changeset/fluffy-crews-rhyme.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/sdk": patch ---- - -feat: Support for new batch trigger system diff --git a/.changeset/lucky-pillows-invite.md b/.changeset/lucky-pillows-invite.md new file mode 100644 index 0000000000..2f43f51ff9 --- /dev/null +++ b/.changeset/lucky-pillows-invite.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Task metrics no longer go missing for projects that configure their own `metricExporters` or `metricReaders`, and the flush error that came with it is gone. diff --git a/.changeset/ninety-cows-lay.md b/.changeset/ninety-cows-lay.md deleted file mode 100644 index 67e588ec94..0000000000 --- a/.changeset/ninety-cows-lay.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/sdk": patch ---- - -feat(sdk): Support debouncing runs when triggering with new debounce options diff --git a/.changeset/node-24-project-default.md b/.changeset/node-24-project-default.md new file mode 100644 index 0000000000..551a7d5a23 --- /dev/null +++ b/.changeset/node-24-project-default.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +New projects created with `trigger init` use Node.js 24 by default. Deployments without explicit `runtime` now use their project's configured default runtime. diff --git a/.changeset/prebuilt-base-images.md b/.changeset/prebuilt-base-images.md new file mode 100644 index 0000000000..4f3f0be094 --- /dev/null +++ b/.changeset/prebuilt-base-images.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Deployment builds now use custom base layer images and no longer install system packages during every build. This improves layer caching resulting in both faster deployments and faster image pulls on the worker cluster side. diff --git a/.changeset/quick-plums-tan.md b/.changeset/quick-plums-tan.md deleted file mode 100644 index bacd44fb90..0000000000 --- a/.changeset/quick-plums-tan.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@trigger.dev/sdk": patch -"@trigger.dev/core": patch ---- - -Added support for idempotency reset diff --git a/.changeset/smooth-schedule-windows.md b/.changeset/smooth-schedule-windows.md new file mode 100644 index 0000000000..6e7bba2eb7 --- /dev/null +++ b/.changeset/smooth-schedule-windows.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Define stable execution windows on declarative scheduled tasks. Schedule API responses now expose both the nominal CRON time and its assigned time, while the dashboard shows configured windows and upcoming assignments. diff --git a/.claude/REVIEW.md b/.claude/REVIEW.md new file mode 100644 index 0000000000..b9438c08af --- /dev/null +++ b/.claude/REVIEW.md @@ -0,0 +1,82 @@ +# REVIEW.md โ€” Trigger.dev OSS + +Repo-specific signal for anyone (human or agent) reviewing a PR in this codebase. Calibrates what counts as critical, what to always check, and what to skip. + +## What makes a ๐Ÿ”ด Important finding here + +Reserve ๐Ÿ”ด for things that would page someone or block a rollback. In this codebase, that means: + +- **Rolling-deploy breakage.** Old and new versions of the webapp/supervisor run side-by-side during deploys. A change is broken if: + - A Lua script's behavior changes for a given key set without versioning (rename the script with a behavior-descriptive suffix like `Tracked` rather than `V2` โ€” both versions must coexist safely). + - A Redis data shape used by both versions changes in place. New shapes need a new key namespace. + - A migration is not backward-compatible with the prior image. +- **Schema / migration safety.** Prisma migrations must be backward-compatible with the prior deploy. Adding NOT NULL without a default, dropping a column an old image still reads, renaming a column โ€” all ๐Ÿ”ด. +- **ClickHouse migration ordering + idempotency.** Goose runs in strict mode in the deploy pipeline and refuses to apply a missing version below the current version โ€” slotting a new file in below the latest already-applied version blocks the deploy. New ClickHouse migration files MUST use the next available number (`max(files in internal-packages/clickhouse/schema/) + 1`); if main has added migrations while you've been on a branch, renumber yours. DDL must also be idempotent (`ADD COLUMN IF NOT EXISTS`, `DROP COLUMN IF EXISTS`, `CREATE TABLE IF NOT EXISTS`, `ADD INDEX IF NOT EXISTS`) so a partial / `--allow-missing` apply elsewhere doesn't fail on retry. Either fault is ๐Ÿ”ด โ€” both break test/prod deploys. Rules live in `internal-packages/clickhouse/CLAUDE.md`. +- **Queue / concurrency correctness.** RunQueue, MarQS (V1, legacy), redis-worker โ€” any change to enqueue / dequeue / locking semantics. Re-derive the invariant on paper before flagging or accepting. +- **Missing index on a hot table.** New Prisma queries against `TaskRun`, `TaskRunExecutionSnapshot`, `JobRun`, `Project`, etc. must use an existing index. Check `internal-packages/database/prisma/schema.prisma` for the relevant `@@index` lines โ€” don't guess and don't propose `EXPLAIN`. +- **Recovery-path queries.** Any `TaskRun.findFirst` / `findMany` added to a schedule, run-recovery, or restart loop. Recovery fan-outs (Redis crash, restart storms) turn "rare indexed query" into a DB incident. ๐Ÿ”ด even if indexed. +- **Aggregations on hot tables.** No `COUNT` / `GROUP BY` on `TaskRun` or other tables that can reach billions of rows. Use Redis or ClickHouse for counts. +- **Prod Redis blast-radius.** New code paths that `SCAN` with broad patterns (`*foo*`) on prod-shaped Redis, or `EVAL` Lua with `SCAN` loops inside. Both are ๐Ÿ”ด. +- **`@trigger.dev/core` direct import** from anywhere outside the SDK package. Always import from `@trigger.dev/sdk`. Core direct imports are ๐Ÿ”ด โ€” they break the public API contract. +- **Heavy execute-deps imported into request-handler bundles.** Specifically `chat.handover` and similar split-bundle entry points must not transitively import the agent task's execute path. Watch for new imports added at module top-level of route files. +- **V1 engine code modified in a "V2 only" PR.** The `apps/webapp/app/v3/` directory contains both. If the PR description says V2-only but it touches `triggerTaskV1`, `cancelTaskRunV1`, `MarQS`, etc. โ€” ๐Ÿ”ด. + +## Performance (always review) + +Every PR gets a performance pass โ€” not just the ones that look perf-sensitive. For each new query or unit of work, weigh three things: (a) the size of the table it hits, (b) whether it sits on a hot path, (c) whether the data it walks can be deep or wide (run trees, batches). The ๐Ÿ”ด bullets above on indexes, recovery-path queries, aggregations, and Redis `SCAN` are part of this pass โ€” the rest below extends it. + +**Treat these tables as large โ€” no scans, no `COUNT` / `GROUP BY`, no unbounded fetch:** + +- **Postgres โ€” the `TaskRun` family:** `TaskRun`, `TaskRunExecutionSnapshot`, `Waitpoint`, `BatchTaskRun` and their join tables. Assume billions of rows. +- **ClickHouse โ€” `task_events_v1` / `task_events_v2`.** Partitioned by `toDate(inserted_at)`; `ORDER BY (environment_id, toUnixTimestamp(start_time), trace_id)`. Note `span_id` / `parent_span_id` are NOT in the sort key โ€” span-id lookups can't skip granules, only `environment_id` + a `start_time` window can. + +**Hot paths โ€” extra scrutiny on any added query or work:** + +- **Trigger + batch trigger** (`triggerTask.server.ts`, `batchTriggerV3.server.ts`) โ€” see `apps/webapp/CLAUDE.md`; do not add DB queries to these. +- **Dequeue / RunQueue** (`dequeueSystem.ts`, run-queue read/lock paths) โ€” runs on every execution. +- **Execution-snapshot creation in the run engine** โ€” any engine function that writes a `TaskRunExecutionSnapshot` runs per state transition; a new query there multiplies by run volume. +- **OTEL ingestion** (`otel.v1.traces.ts`, `otel.v1.logs.ts`) โ€” write volume scales with customer span counts. +- **Trace + run-list reads** (trace view, run list, span detail) โ€” read paths over the large tables above. + +**Deep / wide shapes โ€” one run can explode into a huge tree or batch; code that walks them is the trap:** + +- Trace span subtrees (deeply nested child runs โ†’ deep span trees). +- Batch + parent/child fan-out (one run triggers thousands of children). +- Waitpoint / run-dependency chains. +- Tag / attribute many-to-many joins against the run/event tables. + +**Anti-patterns (severity):** + +- **Per-level fan-out that re-scans a large table once per tree depth** โ†’ ๐Ÿ”ด. A BFS issuing one query per level (e.g. `parent_span_id IN {thisLevel}`) re-reads the same granules D times for a depth-D tree. Prefer one windowed query + an in-memory tree build. +- **Dropping the partition-pruning predicate** โ€” `inserted_at` for ClickHouse, the `createdAt` window for partitioned Postgres โ€” to "widen" a lookup โ†’ ๐Ÿ”ด. Without it the query scans every partition. Keep a bounded window even for ancestor / backfill lookups. +- **Unbounded `IN (...)` built from a result set** (a BFS frontier, a batch's child ids) โ†’ ๐ŸŸก. It can reach the row cap (`MAXIMUM_TRACE_SUMMARY_VIEW_COUNT` defaults to 25k). Cap or chunk to โ‰ค1โ€“2k ids per query. +- **Sequential per-level round-trips** where one recursive or windowed query would do โ†’ ๐ŸŸก. N levels = N round-trip latencies stacked. +- **Replacing a single bounded query with a multi-query walk for _every_ call** (not just a rare fallback) โ†’ ๐Ÿ”ด on a hot read path, ๐ŸŸก elsewhere. Keep the cheap single-query path; branch into the expensive walk only when the cheap one comes up short. + +## Always check + +- **Tests use testcontainers, not mocks.** Vitest with `redisTest` / `postgresTest` / `containerTest` from `@internal/testcontainers`. Any new `vi.mock(...)` on Redis, Postgres, BullMQ, or other infra is wrong here โ€” ๐Ÿ”ด if added in production-path tests, ๐ŸŸก if isolated unit test. +- **User-facing public-package changes have a changeset.** `pnpm run changeset:add` produces `.changeset/*.md`. Changesets are user-facing release notes, not a catalog of every change: required when a `packages/*` or `integrations/*` change is something a user would notice or act on, skipped for internal-only changes, refactors, chores, and packages not consumed independently (e.g. `@trigger.dev/redis-worker`). Missing on a user-facing change โ†’ ๐ŸŸก; missing on a breaking change โ†’ ๐Ÿ”ด. Do not flag a missing note when the change is not user-facing. +- **User-facing server-only changes have `.server-changes/*.md`.** Required for user-facing `apps/webapp/`, `apps/supervisor/` edits in a PR with no package or integration change that requires a changeset; skip internal-only or admin-only changes, refactors, and chores. Body should be 1-2 sentences (it has to fit as one bullet in a future changelog). Missing on a user-facing change โ†’ ๐ŸŸก. +- **Lua script naming.** Coexisting scripts use behavior-descriptive suffixes (`Tracked`), never `V2`. Old name must keep working until the next deploy clears it. +- **RunQueue payload shape.** V2 run-queue payload's `projectId` is consumed by `workerQueueResolver` for override matching. If a PR drops it from the payload, ๐Ÿ”ด. +- **`safeSend` scope.** Defensive IPC wrappers belong on loop / interval / handler contexts, not one-shot terminal sends. If the PR adds `safeSend` to a single terminal call for consistency, ๐ŸŸก with a "remove this" suggestion. +- **Zod version.** Pinned to `3.25.76` monorepo-wide. New package adding zod with a different version or range โ€” ๐Ÿ”ด. + +## Skip (do NOT flag) + +- Anything oxfmt / oxlint catches. CI enforces both via the `code-quality` check. +- TypeScript style preferences (`type` vs `interface`) โ€” already covered by repo standards. +- Test coverage exhortations as a generic suggestion. Only flag missing tests when a specific code path is genuinely untested and the path has prior incidents. +- `agentcrumbs` markers (`// @crumbs`, `// #region @crumbs`) and `agentcrumbs` imports โ€” these are temporary debug instrumentation stripped before merge. +- `// removed comments for removed code`, renamed `_unused` vars, re-exported types as "backwards compatibility shims" โ€” also covered by repo standards. +- Suggestions to "add error handling" without naming a specific scenario that breaks. +- Documentation prose nitpicks in `docs/*` MDX files unless factually wrong. + +## Things V1/legacy that should NOT block a PR + +The `apps/webapp/app/v3/` directory name is misleading โ€” most code there is V2. Only specific files are V1-only legacy: `MarQS` queue, `triggerTaskV1`, `cancelTaskRunV1`, and a handful of others (see `apps/webapp/CLAUDE.md` for the exact list). Don't flag "you should refactor this to use V2" on those โ€” they're frozen. + +## Confidence calibration for this repo + +The most common false-positive pattern: speculating about race conditions in code paths the agent doesn't have runtime visibility into. If the only evidence is "this *could* race", drop it. If you can point to a specific interleaving with file:line for each step, surface it. diff --git a/.claude/agents/code-reviewer.md b/.claude/agents/code-reviewer.md new file mode 100644 index 0000000000..3acb1362bf --- /dev/null +++ b/.claude/agents/code-reviewer.md @@ -0,0 +1,13 @@ +--- +name: code-reviewer +description: Adversarially verifies one landed packet against its requirement; read-only. +model: opus +--- + +You are an adversarial code reviewer for one landed packet. READ-ONLY: never modify code, never commit, never push, never post to GitHub. + +- Try to refute that the change answers its stated requirement; look for the failure scenario, not confirmation. +- Check the diff for unrelated drift, dead code, broken semantics of neighbors, and whether tests prove the actual invariant (would the test fail if the fix were subtly wrong?). +- Check the change landed in the correct PR/branch of the stack. +- Distinguish fact from inference; cite exact file:line evidence. +- Return: verdict (approve / needs-changes) with evidence per concern, and the exact minimal correction when needs-changes. diff --git a/.claude/agents/code-writer.md b/.claude/agents/code-writer.md new file mode 100644 index 0000000000..96c8219e4e --- /dev/null +++ b/.claude/agents/code-writer.md @@ -0,0 +1,16 @@ +--- +name: code-writer +description: Implements exactly one work packet โ€” minimal diff, targeted checks, own-paths-only commits. +model: opus +--- + +You are a code writer. Implement exactly the one work packet in your prompt. + +- Minimal diff; match surrounding style and idiom. +- Prefer no comment at all; comment only a non-obvious constraint, max 2 short lines. All texts (comments, commit messages) short, clear, simple. +- Verify the packet's own diagnosis against the code before applying; if it is wrong, STOP without committing and report why. +- Run only the targeted checks for your packet: the relevant vitest files, `pnpm run typecheck --filter ` when the change warrants it. Never full suites unless asked. +- `pnpm run format` on touched files before committing. +- Stage and commit ONLY your packet's files. Conventional commit message. NO Claude attribution, no Co-Authored-By. +- Push only if the packet explicitly says to. +- Return: what changed, evidence (test output), commit SHA, and anything contradicting the diagnosis. diff --git a/.claude/agents/software-architect.md b/.claude/agents/software-architect.md new file mode 100644 index 0000000000..a58a3b095a --- /dev/null +++ b/.claude/agents/software-architect.md @@ -0,0 +1,12 @@ +--- +name: software-architect +description: Resolves contested design questions against the specs; decision + rationale, never code. +model: opus +--- + +You are a software architect. Resolve exactly the contested design question in your prompt against the given specs/contracts. READ-ONLY. + +- Ground the decision in the actual code and the project's design contracts (GUIDEBOOK, Linear specs) โ€” not in generic best practice. +- Weigh stack boundaries: which PR owns the change, what merges independently. +- Prefer the smallest decision that unblocks the packet; flag speculative architecture rather than endorsing it. +- Return: the decision, its rationale, rejected alternatives (one line each), and exactly what the dependent packet should do. diff --git a/.claude/review-guides/chat-agent-sessions-row-agnostic.md b/.claude/review-guides/chat-agent-sessions-row-agnostic.md new file mode 100644 index 0000000000..7fb9851f30 --- /dev/null +++ b/.claude/review-guides/chat-agent-sessions-row-agnostic.md @@ -0,0 +1,287 @@ +# Review guide โ€” chat.agent on Sessions, row-agnostic addressing + +Scope: the 12 uncommitted files. **No new behaviour beyond the public surface +already on this branch** โ€” this is plumbing cleanup that: + +1. Eliminates the transport's session-creation step +2. Makes `chatId` the universal addressing string everywhere +3. Makes the server-side stream/append/wait routes row-agnostic + +## The two design moves + +**Move 1 โ€” agent owns session lifecycle.** `chat.agent` and +`chat.customAgent` upsert the backing `Session` row at bind, fire-and-forget, +keyed on `externalId = payload.chatId`. The transport, server-side +`AgentChat`, and `chat.createTriggerAction` no longer create sessions at all. +Browsers cannot mint sessions either (`POST /api/v1/sessions` is now +secret-key-only). One owner, one path. + +**Move 2 โ€” `chatId` is the only address.** The transport, server-side +`AgentChat`, JWT scopes, and S2 stream paths all use `chatId` directly. The +Session's friendlyId is informational. To make this safe, the three stream +routes (`.in/.out` PUT, GET, POST append, plus the run-engine `wait` +endpoint) became "row-optional" and derive a *canonical addressing key* +(`row.externalId ?? row.friendlyId`, fallback to the URL param when the row +hasn't been upserted yet). Same canonical key is used to build the S2 stream +path, the waitpoint cache key, and the JWT resource set โ€” so any caller +addressing by either form converges on the same physical stream. + +Together these remove an entire class of "did the row land yet?" races. The +transport can subscribe to `/sessions/{chatId}/out` before the agent boots, +the agent's `void sessions.create({externalId: chatId})` lands a moment +later, and any earlier reads/writes are already on the right S2 key. + +--- + +## Read in this order + +### 1. `apps/webapp/app/services/realtime/sessions.server.ts` (+34 lines) + +The new primitive. Two helpers: + +- `isSessionFriendlyIdForm(value)` โ€” `value.startsWith("session_")`. Used to + decide whether a missing row is a hard 404 (opaque friendlyId) or a soft + "row will land later" (externalId form). +- `canonicalSessionAddressingKey(row, paramSession)` โ€” `row.externalId ?? + row.friendlyId` if the row exists, else `paramSession`. **This is the load- + bearing function.** Read its docstring. + +**Question to ask:** can two callers addressing the "same" session ever get +different canonical keys? Only if the row exists for one and not the other, +*and* the URL forms differ โ€” but in that case the row-less caller used the +externalId form (friendlyId-form would have 404'd earlier), and the row-ful +caller computes `row.externalId ?? row.friendlyId`. If the row's externalId +matches the URL, they converge. If it doesn't, there's no row to find by +that string anyway. The interesting edge is "row exists with no externalId", +addressed via friendlyId โ€” both sides read `row.friendlyId`. โœ“ + +### 2. `apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts` (+47/-12) + +PUT initialize + GET subscribe (SSE). Both use the helper. The interesting +part is the loader's `findResource` + `authorization.resource`: + +```ts +findResource: async (params, auth) => { + const row = await resolveSessionByIdOrExternalId(...); + if (!row && isSessionFriendlyIdForm(params.session)) return undefined; // 404 + return { row, addressingKey: canonicalSessionAddressingKey(row, params.session) }; +}, +authorization: { + resource: ({ row, addressingKey }) => { + const ids = new Set([addressingKey]); + if (row) { + ids.add(row.friendlyId); + if (row.externalId) ids.add(row.externalId); + } + return { sessions: [...ids] }; + }, + superScopes: ["read:sessions", "read:all", "admin"], +}, +``` + +**Why three IDs in the resource set?** `checkAuthorization` is "any-match" +across the resource values. We want a JWT scoped to *either* form to +authorize *either* URL form. Smoke test verified the 4-cell matrix passes. + +**The PUT path** (action handler) is simpler โ€” it just resolves the row, +builds an addressing key, and hands it to `initializeSessionStream`. Worth +noting the `closedAt` check is now `maybeSession?.closedAt` โ€” no row means +no closedAt to enforce. + +### 3. `apps/webapp/app/routes/realtime.v1.sessions.$session.$io.append.ts` (+22/-13) + +POST append (browser writes a record to `.in` or server writes to `.out`). +Same row-optional pattern. Both the S2 append and the waitpoint drain use +`addressingKey`. + +**Question to ask:** what fires the waitpoint? An agent's +`session.in.wait()` registers a waitpoint keyed on `(addressingKey, io)` via +the wait endpoint (file 4). The append handler drains by the *same* key โ€” +even if the agent registered with externalId form and the transport +appended via friendlyId form, both compute the same canonical key, so they +converge. โœ“ + +### 4. `apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts` (+18/-13) + +The agent's `.in.wait()` endpoint. Run-engine creates the waitpoint, then +registers it in Redis under `(addressingKey, io)`. The race-check that runs +right after creation reads from S2 by the same key. Three call sites โ€” +`addSessionStreamWaitpoint`, `readSessionStreamRecords`, +`removeSessionStreamWaitpoint` โ€” all consistent. + +### 5. `apps/webapp/app/routes/api.v1.sessions.ts` (+4/-2) + +**Security tightening.** Removed `allowJWT: true` and `corsStrategy: "all"` +from the `POST /api/v1/sessions` action โ€” secret-key only now. + +**Question to ask:** was the JWT path actually used? Until this branch, the +transport called it via `ensureSession` (now deleted). After this branch, +nobody reaches it from the browser. `chat.createTriggerAction` (server +secret key) is the only browser-adjacent path. + +### 6. `packages/trigger-sdk/src/v3/ai.ts` (+62/-39) + +Two near-identical edits โ€” one in `chatAgent`, one in `chatCustomAgent`. +Both bind on `payload.chatId` and fire-and-forget the upsert: + +```ts +locals.set(chatSessionHandleKey, sessions.open(payload.chatId)); +void sessions + .create({ type: "chat.agent", externalId: payload.chatId }) + .catch(() => { /* best effort */ }); +``` + +**Question to ask:** why `void`-and-`catch`? Awaiting the upsert would gate +the agent's bind on a network round-trip that doesn't unblock anything +user-visible โ€” `.in/.out` routes are row-agnostic and the waitpoint cache +is keyed on the addressing string, not the row id. If the upsert genuinely +fails, the next bind retries the same idempotent call (`sessions.create` +upserts on `externalId`, so concurrent triggers on one chatId converge to +one row). The row matters for downstream metadata + listing, not for live +addressing. + +The PAT scope minting in `chatAgent` (two call sites โ€” preload and +sendMessage) now uses `payload.chatId` for the `sessions:` resource. That +matches what the transport/AgentChat use as the JWT resource and what the +JWT's resource set in the loader includes. Cross-form addressing works +either way (smoke-tested), but using `chatId` keeps the chain tight. + +`createChatTriggerAction` is the most visibly trimmed: no pre-create, no +threading `sessionId` into payload, scope mint uses `chatId`. Return type +no longer carries `sessionId` โ€” note `TriggerChatTaskResult.sessionId` was +already declared optional, so this isn't a public-API break. + +**Stale docstring to flag:** `chat.ts:59` and `chat.ts:112` still describe +PAT scopes as `read:sessions:{sessionId}` and +`write:sessions:{sessionId}`. Functionally either ID works (row lookup +canonicalises), but the doc text is now out of date โ€” it should say +`{chatId}`. Worth a tidy-up before merge but not blocking. + +### 7. `packages/trigger-sdk/src/v3/chat.ts` (+63/-117) + +**The biggest mechanical edit.** Net -54 lines from deleting `ensureSession` +and untangling its callers. + +What disappeared: +- `private async ensureSession(chatId)` โ€” gone +- The "lazy upsert from the browser if no triggerTask callback" branch in + `sendMessages` and `preload` โ€” gone +- The "throw if neither path surfaced a sessionId" guard โ€” gone +- All `state.sessionId` URL params replaced with `chatId` +- `subscribeToSessionStream`'s `chatId?` (optional) is now `chatId` (required) + +What stayed: +- `state.sessionId` in `ChatSessionState` โ€” optional, informational +- The `restore from external storage` branch in the constructor still + hydrates `sessionId` if persisted, just doesn't *require* it +- `notifySessionChange` still surfaces `sessionId` if known + +**Question to ask:** does the transport ever still need the friendlyId? The +only place is the `onSessionChange` callback's payload (so consumers +persisting state can save it for later display). The transport itself never +puts it in a URL or a waitpoint key. + +The `sendMessages` path is worth re-reading: when state.runId is set, it +appends to `.in/append` and subscribes to `.out`. If the append fails with +a non-auth error, it falls through to triggering a new run (legacy "run is +dead" detection โ€” unchanged from pre-Sessions, doesn't depend on +addressing). + +### 8. `packages/trigger-sdk/src/v3/chat-client.ts` (+34/-33) + +Server-side `AgentChat`. Mirrors the transport changes โ€” every URL uses +`this.chatId`. `triggerNewRun` no longer pre-creates a session. `ChatSession` +and internal `SessionState` types now have optional `sessionId`. + +The shape of the diff is identical to the transport: delete the upsert, +swap addressing identifiers, optionalise the friendlyId. If you've read +`chat.ts` carefully, this one is mostly mechanical confirmation that both +client surfaces (browser transport + server-side AgentChat) speak the same +addressing protocol. + +### 9. Test infrastructure โ€” `sessions.ts` (+18) + `mock-chat-agent.ts` (+25) + +`__setSessionCreateImplForTests` mirrors the existing +`__setSessionOpenImplForTests`. `mockChatAgent` installs a no-op create stub +returning a synthetic `CreatedSessionResponseBody` so the agent's bind-time +`void sessions.create(...)` doesn't try to hit a real API. Cleanup runs in +the same `.finally` as the open override. + +**Question to ask:** is the synthetic response shape correct? It mirrors +`CreatedSessionResponseBody` โ€” `id`, `externalId`, `type`, `tags`, +`metadata`, `closedAt`, `closedReason`, `expiresAt`, `createdAt`, +`updatedAt`, `isCached`. Tests don't currently assert on this object, so +the bar is "doesn't crash + matches the type". Met. + +### 10. `packages/trigger-sdk/src/v3/chat.test.ts` (+13/-12) + +Three classes of test edits, all consequences: + +- Stream URL assertion: `chat-1` (the chatId) instead of + `session_streamurl` (the friendlyId) +- `renewRunAccessToken` callback: `sessionId: undefined` (was + `DEFAULT_SESSION_ID` because the mocked trigger doesn't surface it) +- Token resolve count: `1` (was `2` โ€” second resolve was for `ensureSession`) +- One `onSessionChange` matchObject loses `sessionId` + +### 11. `apps/webapp/app/routes/_app.../playground/.../route.tsx` (1 line) + +`sessionId: string` โ†’ `sessionId?: string` in the playground sidebar prop +to track the transport type change. + +--- + +## Edge cases I checked, so you don't have to + +- **Cross-form JWT auth (curl matrix).** JWT scoped to externalId can call + externalId URL โœ“ and friendlyId URL โœ“. JWT scoped to friendlyId can call + externalId URL โœ“ and friendlyId URL โœ“. Smoke-tested. +- **Row materialises after subscribe.** Transport opens + `GET /sessions/{chatId}/out` before agent's bind upsert lands โ†’ 200 OK, + `addressingKey = chatId` (paramSession fallback). Once the row lands + with `externalId = chatId`, addressingKey resolves to the same value via + `row.externalId`. Same S2 key throughout. +- **Concurrent triggers on one chatId.** Two browser tabs trigger two runs + โ†’ two binds โ†’ two `sessions.create({externalId: chatId})` calls. Upsert + semantics: both return the same row. +- **Closed session enforcement.** Still enforced when a row exists. + `maybeSession?.closedAt` is null-safe; no row = no close-state to honour. +- **Agent run cancellation.** Frontend doesn't auto-detect โ€” unchanged from + pre-Sessions; messages sit in S2 until the next trigger (the existing + run-PAT auth-error path is the only reaper). Out of scope for this branch. +- **Idle timeout in dev.** Runs stay `EXECUTING_WITH_WAITPOINTS` past the + configured idle because dev runs don't snapshot/restore; the in-process + idle clock advances locally without touching the row. Expected, not a + regression. + +## Things explicitly **not** in this branch + +- Run-state subscription on the transport side (the "run died, re-trigger + silently" UX gap) +- Session auto-close on agent exit (still client-driven by design) +- Any change to `Session` schema, `sessions.create` semantics, or + `chatAccessTokenTTL` +- Docstring updates for `read:sessions:{sessionId}` / `write:sessions:{sessionId}` + in `chat.ts:59` and `chat.ts:112` (functional but textually stale โ€” + follow-up nit) + +--- + +## What I'd be ready to answer cold + +- Why fire-and-forget upsert (vs. `await`) in the agent's bind step +- Why the route's authorization resource set has three IDs (cross-form JWT + auth) +- Why `POST /api/v1/sessions` lost `allowJWT` (security tightening โ€” no + caller needs it after the transport's `ensureSession` is gone) +- What converges two callers using different URL forms onto the same S2 + stream (`canonicalSessionAddressingKey`, identical computation on both + sides for any given row) +- What makes `sessions.create` race-safe under concurrent triggers + (`externalId` upsert) +- Why `state.sessionId` stayed on `ChatSessionState` at all (pure + informational, surfaced via `onSessionChange` for consumer persistence; + zero addressing role) +- Why the chat-client (server-side AgentChat) and chat (transport) edits + look near-identical (they implement the same client protocol against the + same row-agnostic routes) diff --git a/.claude/rules/database-safety.md b/.claude/rules/database-safety.md new file mode 100644 index 0000000000..14a6523595 --- /dev/null +++ b/.claude/rules/database-safety.md @@ -0,0 +1,13 @@ +--- +paths: + - "internal-packages/database/**" +--- + +# Database Migration Safety + +- When adding indexes to **existing tables**, use `CREATE INDEX CONCURRENTLY IF NOT EXISTS` to avoid table locks. These must be in their own separate migration file (one index per file). +- Indexes on **newly created tables** (same migration as `CREATE TABLE`) do not need CONCURRENTLY. +- When indexing a **new column on an existing table**, split into two migrations: first `ADD COLUMN IF NOT EXISTS`, then `CREATE INDEX CONCURRENTLY IF NOT EXISTS` in a separate file. +- After generating a migration with Prisma, remove extraneous lines for: `_BackgroundWorkerToBackgroundWorkerFile`, `_BackgroundWorkerToTaskQueue`, `_TaskRunToTaskRunTag`, `_WaitpointRunConnections`, `_completedWaitpoints`, `SecretStore_key_idx`, and unrelated TaskRun indexes. +- Never drop columns or tables without explicit approval. +- New code should target `RunEngineVersion.V2` only. diff --git a/.claude/rules/docs-writing.md b/.claude/rules/docs-writing.md new file mode 100644 index 0000000000..bbfb471368 --- /dev/null +++ b/.claude/rules/docs-writing.md @@ -0,0 +1,14 @@ +--- +paths: + - "docs/**" +--- + +# Documentation Writing Rules + +- Use Mintlify MDX format. Frontmatter: `title`, `description`, `sidebarTitle` (optional). +- After creating a new page, add it to `docs.json` navigation under the correct group. +- Use Mintlify components: ``, ``, ``, ``, ``, ``, ``/``. +- Code examples should be complete and runnable where possible. +- Always import from `@trigger.dev/sdk`, never `@trigger.dev/sdk/v3`. +- Keep paragraphs short. Use headers to break up content. +- Link to related pages using relative paths (e.g., `[Tasks](/tasks/overview)`). diff --git a/.claude/rules/legacy-v3-code.md b/.claude/rules/legacy-v3-code.md new file mode 100644 index 0000000000..c5e0d46597 --- /dev/null +++ b/.claude/rules/legacy-v3-code.md @@ -0,0 +1,26 @@ +--- +paths: + - "apps/webapp/app/v3/**" +--- + +# v3 (engine V1) has been removed + +The v3 engine (RunEngineVersion `V1`: MarQS queue + Graphile worker) is end-of-life and its execution code has been removed from the webapp. The `app/v3/` directory name is historical: everything under it now serves the current V2 engine (`@internal/run-engine` + `@trigger.dev/redis-worker`). + +There is no `V1` execution path anymore. If you find a `RunEngineVersion` branch, the `V1` arm should only reject or finalize gracefully (for example, mark a historical run cancelled in the DB), never run V1 work. Do not reintroduce MarQS, the graphile worker, or the v3 socket.io namespaces. + +## The deprecation boundary (keep this) + +Requests from clients still on v3 (old SDK/CLI) or historical V1 runs must return a clean 4xx, never a 5xx. The boundary lives in: + +- `engineDeprecation.server.ts` - the `V3_TRIGGER_DEPRECATION_MESSAGE` / `V3_DEV_DEPRECATION_MESSAGE` / `V3_MIGRATION_URL` upgrade messages. +- `engineVersion.server.ts` - `determineEngineVersion()` still detects a V1 project/run so callers can reject it. +- `services/triggerTask.server.ts`, `services/cancelTaskRun.server.ts`, `services/rescheduleTaskRun.server.ts` - the `V1` arm rejects or finalizes gracefully instead of executing. +- `services/initializeDeployment.server.ts` - the `DEPRECATE_V3_CLI_DEPLOYS_ENABLED`-gated v3 CLI deploy rejection. +- `handleWebsockets.server.ts` - the legacy `trigger dev` websocket closes with the upgrade message. + +## V2 modern stack + +- **Run lifecycle**: `@internal/run-engine` (`runEngine.server.ts`, `runEngineHandlers.server.ts`) +- **Background jobs**: `@trigger.dev/redis-worker` (`commonWorker.server.ts`, `alertsWorker.server.ts`, `batchTriggerWorker.server.ts`; `legacyRunEngineWorker.server.ts` still hosts the live batch-completion jobs) +- **Queue operations**: RunQueue inside run-engine (`runQueue.server.ts`), not MarQS diff --git a/.claude/rules/package-installation.md b/.claude/rules/package-installation.md new file mode 100644 index 0000000000..310074823c --- /dev/null +++ b/.claude/rules/package-installation.md @@ -0,0 +1,22 @@ +--- +paths: + - "**/package.json" +--- + +# Installing Packages + +When adding a new dependency to any package.json in the monorepo: + +1. **Look up the latest version** on npm before adding: + ```bash + pnpm view version + ``` + If unsure which version to use (e.g. major version compatibility), confirm with the user. + +2. **Edit the package.json directly** โ€” do NOT use `pnpm add` as it can cause issues in the monorepo. Add the dependency with the correct version range (typically `^x.y.z`). + +3. **Run `pnpm i` from the repo root** after editing to install and update the lockfile: + ```bash + pnpm i + ``` + Always run from the repo root, not from the package directory. diff --git a/.claude/rules/sdk-packages.md b/.claude/rules/sdk-packages.md new file mode 100644 index 0000000000..343be2045f --- /dev/null +++ b/.claude/rules/sdk-packages.md @@ -0,0 +1,12 @@ +--- +paths: + - "packages/**" +--- + +# Public Package Rules + +- Changes to `packages/` are **customer-facing**. Always add a changeset: `pnpm run changeset:add` +- Default to **patch**. Get maintainer approval for minor. Never select major without explicit approval. +- `@trigger.dev/core`: **Never import the root**. Always use subpath imports (e.g., `@trigger.dev/core/v3`). +- Do NOT update `rules/` or `.claude/skills/trigger-dev-tasks/` unless explicitly asked. These are maintained in separate dedicated passes. +- Test changes using the `hello-world` project in the [`triggerdotdev/references`](https://github.com/triggerdotdev/references) repo. diff --git a/.claude/rules/server-apps.md b/.claude/rules/server-apps.md new file mode 100644 index 0000000000..cc3e6852a0 --- /dev/null +++ b/.claude/rules/server-apps.md @@ -0,0 +1,25 @@ +--- +paths: + - "apps/**" +--- + +# Server App Changes + +`.server-changes/` files are user-facing release notes, not a catalog of every change. When a user-facing server app change (webapp, supervisor, etc.) is in a PR with **no package or integration change that requires a changeset**, add a `.server-changes/` file instead of a changeset. Skip it for internal-only or admin-only changes, refactors, and chores: + +```bash +cat > .server-changes/descriptive-name.md << 'EOF' +--- +area: webapp +type: fix +--- + +Fix pages occasionally loading unstyled during deploys. The dashboard now recovers automatically. +EOF +``` + +- **area**: `webapp` | `supervisor` +- **type**: `feature` | `fix` | `improvement` | `breaking` +- If the PR also touches `packages/` or `integrations/` and that change needs a changeset, the changeset covers it (no `.server-changes/` needed). If the package or integration change is internal and needs no changeset, still add a `.server-changes/` file for the user-facing server change. + +The body ships **verbatim in user-facing release notes**. Keep it to 1โ€“2 short sentences, non-technical, written for a dashboard user: describe what changed for them, never the implementation (no header names, endpoints, middleware, storage mechanisms, internal tools). See `.server-changes/README.md` for full guidance. diff --git a/.claude/skills/drizzle/SKILL.md b/.claude/skills/drizzle/SKILL.md new file mode 100644 index 0000000000..e647343512 --- /dev/null +++ b/.claude/skills/drizzle/SKILL.md @@ -0,0 +1,191 @@ +--- +name: drizzle +description: Use this skill when writing or modifying Drizzle ORM schemas, queries, or migrations in this repo โ€” specifically the `@internal/dashboard-agent-db` package (the dashboard agent's conversation datastore). Covers pg-core schema definition, the postgres-js driver, drizzle-kit migrations, and this repo's conventions: a dedicated Postgres schema, foreign-key-free cross-database design, pooler-safe connections, and the access-pattern query layer. Drizzle is NOT the main database โ€” that's Prisma. +allowed-tools: Read, Write, Edit, Glob, Grep, Bash +--- + +# Drizzle ORM (this repo) + +Drizzle is used in exactly one place: **`internal-packages/dashboard-agent-db`** (`@internal/dashboard-agent-db`), the in-dashboard agent's conversation store. Everything else in the monorepo is **Prisma** (`@trigger.dev/database`). Keep them separate. + +Pinned versions: **`drizzle-orm` ^0.45**, **`drizzle-kit` ^0.31** (dev), **`postgres` ^3.4** (postgres.js driver). drizzle-orm and drizzle-kit are intentionally on different version lines โ€” 0.31.x is the correct companion for 0.45.x, there is no peer dependency between them. + +## Critical rules + +1. **Drizzle is only the agent's own datastore.** The agent (and its task bundle) must have **no access to the main Prisma database or ClickHouse**. Never import the Prisma client into the agent task or into `@internal/dashboard-agent-db`. Main data is reached via the API, not Drizzle. +2. **Foreign-key-free.** In cloud this DB is a *separate* PlanetScale database, so it can't FK into the main DB. Reference main entities (`organizationId`, `userId`, โ€ฆ) **by id only โ€” never `.references()`**. Joins happen in app code; tenant scoping is enforced in the query layer. +3. **One dedicated Postgres schema.** All tables live under `pgSchema("trigger_dashboard_agent")` so they're schema-qualified and isolated from Prisma's `public` schema (this is what makes the OSS single-database fallback safe). +4. **Pooler-safe connections.** Connections go through a transaction-mode pooler (PlanetScale / PgBouncer-style), so postgres.js must run with **`prepare: false`** โ€” prepared statements don't survive a connection being handed to another client between checkouts. +5. **Node16 module resolution.** Relative imports need explicit **`.js`** extensions (`import { chats } from "./schema.js"`), even though the source is `.ts`. +6. **Scope every user query.** All queries that touch user data go through `src/queries.ts` and are scoped by `organizationId` / `userId`, so callers can't forget the `where`. Don't write ad-hoc cross-tenant queries elsewhere. + +## Package layout + +```text +internal-packages/dashboard-agent-db/ + drizzle.config.ts # drizzle-kit config (schema path, out dir, schemaFilter) + drizzle/ # generated migrations (committed) + src/ + schema.ts # pgSchema + table definitions + client.ts # createDashboardAgentDb() โ€” postgres.js + drizzle + queries.ts # the access-pattern layer (org/user-scoped) + index.ts # barrel: re-exports schema, client, queries +``` + +`package.json` points `main`/`types` at `./src/index.ts` (consumed as source, no build step) โ€” same as other simple internal packages. + +## Schema (pg-core) + +Use `pgSchema(...).table(...)`, not the bare `pgTable`, so tables land in the dedicated schema. ([schemas](https://orm.drizzle.team/docs/schemas), [pg column types](https://orm.drizzle.team/docs/column-types/pg), [indexes](https://orm.drizzle.team/docs/indexes-constraints)) + +```ts +import { sql } from "drizzle-orm"; +import { index, jsonb, pgSchema, text, timestamp } from "drizzle-orm/pg-core"; + +export const dashboardAgentSchema = pgSchema("trigger_dashboard_agent"); + +export const chats = dashboardAgentSchema.table( + "chats", + { + id: text("id").primaryKey(), + organizationId: text("organization_id").notNull(), // FK-free: id only, no .references() + userId: text("user_id").notNull(), + title: text("title").notNull().default("New chat"), + // JSONB with a typed view; .default([]) / .default({}) emit '[]'::jsonb / '{}'::jsonb + messages: jsonb("messages").$type().notNull().default([]), + metadata: jsonb("metadata").$type>().notNull().default({}), + deletedAt: timestamp("deleted_at", { withTimezone: true }), // soft delete + lastMessageAt: timestamp("last_message_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + // Extra config returns an ARRAY in drizzle-orm 0.36+ (not an object). + (t) => [ + // Partial + ordered composite index. `.desc()` on the column, `.where(sql`...`)` for partial. + index("chats_org_user_last_msg_idx") + .on(t.organizationId, t.userId, t.lastMessageAt.desc()) + .where(sql`${t.deletedAt} is null`), + ] +); + +// Inferred row types for the query layer + consumers. +export type Chat = typeof chats.$inferSelect; +export type NewChat = typeof chats.$inferInsert; +``` + +Notes: +- `timestamp(..., { withTimezone: true })` โ†’ `timestamp with time zone`. Use `.defaultNow()` for `DEFAULT now()`. +- For a "newest first, nulls last" sort the partial index uses `.desc()`; the *query* uses raw `sql` for `NULLS LAST` (see below). +- Don't add `.references()` โ€” see critical rule 2. + +## Client (postgres.js + drizzle) + +([connect overview](https://orm.drizzle.team/docs/connect-overview)) One small pool, `prepare: false`. In the agent task create it once in `onBoot` (per-process); in the webapp wrap it in the `singleton(...)` helper. + +```ts +import { drizzle, type PostgresJsDatabase } from "drizzle-orm/postgres-js"; +import postgres, { type Sql } from "postgres"; +import * as schema from "./schema.js"; + +export type DashboardAgentDb = PostgresJsDatabase; + +export function createDashboardAgentDb(connectionString: string, opts: { max?: number } = {}) { + const sql: Sql = postgres(connectionString, { + max: opts.max ?? 5, // small โ€” the pooler does the real pooling + idle_timeout: 20, // release conns when an agent run suspends + prepare: false, // REQUIRED for transaction-mode poolers + }); + return { db: drizzle(sql, { schema }), sql, close: () => sql.end() }; +} +``` + +## Queries (the access-pattern layer) + +([select](https://orm.drizzle.team/docs/select), [insert](https://orm.drizzle.team/docs/insert), [operators](https://orm.drizzle.team/docs/operators), [transactions](https://orm.drizzle.team/docs/transactions), [joins](https://orm.drizzle.team/docs/joins)) + +```ts +import { and, desc, eq, isNull, sql } from "drizzle-orm"; + +// Select EXPLICIT columns for list views โ€” never select a large blob (messages) +// or a secret (tokens) you don't need. `NULLS LAST` needs raw sql in orderBy. +await db + .select({ id: chats.id, title: chats.title, lastMessageAt: chats.lastMessageAt }) + .from(chats) + .where(and(eq(chats.organizationId, orgId), eq(chats.userId, userId), isNull(chats.deletedAt))) + .orderBy(sql`${chats.pinnedAt} desc nulls last`, desc(chats.lastMessageAt)) + .limit(50); + +// Idempotent create (avoids a duplicate-key race between two writers). +await db.insert(chats).values({ id, organizationId: orgId, userId }).onConflictDoNothing(); + +// Upsert. +await db + .insert(chatSessions) + .values({ chatId, publicAccessToken }) + .onConflictDoUpdate({ target: chatSessions.chatId, set: { publicAccessToken, updatedAt: sql`now()` } }); + +// Owner-scope a join (this DB is FK-free, so enforce ownership in the query). +await db + .select({ /* session cols */ }) + .from(chatSessions) + .innerJoin(chats, eq(chats.id, chatSessions.chatId)) + .where(and(eq(chatSessions.chatId, chatId), eq(chats.userId, userId))); + +// Multi-write that must be consistent on the next read โ†’ one transaction. +await db.transaction(async (tx) => { + await tx.update(chats).set({ messages, updatedAt: sql`now()` }).where(eq(chats.id, chatId)); + await tx.insert(chatSessions).values({ /* ... */ }).onConflictDoUpdate({ /* ... */ }); +}); +``` + +Use `sql\`now()\`` for DB-side timestamps in updates. + +## Migrations (drizzle-kit) + +([kit overview](https://orm.drizzle.team/docs/kit-overview), [generate](https://orm.drizzle.team/docs/drizzle-kit-generate), [migrate](https://orm.drizzle.team/docs/drizzle-kit-migrate)) + +`drizzle.config.ts` must set **`schemaFilter`** so drizzle-kit only ever manages our schema โ€” never Prisma's `public` (critical in the OSS single-DB fallback): + +```ts +import { defineConfig } from "drizzle-kit"; +export default defineConfig({ + schema: "./src/schema.ts", + out: "./drizzle", + dialect: "postgresql", + schemaFilter: ["trigger_dashboard_agent"], + dbCredentials: { url: process.env.DASHBOARD_AGENT_DATABASE_URL ?? process.env.DATABASE_URL ?? "postgres://placeholder" }, +}); +``` + +Workflow: + +```bash +cd internal-packages/dashboard-agent-db +pnpm run db:generate # diff schema.ts โ†’ emit SQL into drizzle/. OFFLINE (no DB needed). +# review the generated drizzle/000N_*.sql before committing +pnpm run db:migrate # apply pending migrations. Needs a real DATABASE URL. +``` + +- `db:generate` is **offline** โ€” it only reads `schema.ts`, so you can verify a schema change compiles to valid DDL with no database. Use it as a fast check. +- drizzle-kit names migration files with a **random suffix** (`0000_magenta_lilandra.sql`). Don't regenerate a committed migration just to "refresh" it โ€” that churns the filename. After the first migration is committed, schema changes produce a **new** `000N_*.sql`; commit that. +- Generated DDL for a new schema is one `CREATE SCHEMA` + schema-qualified `CREATE TABLE`s + indexes, **no foreign keys** (by design here). + +## Common gotchas + +- **`prepare: false`** is not optional with a pooler โ€” without it you'll get prepared-statement errors under load. +- **Missing `.js` extension** on a relative import โ†’ TS2835 under Node16 resolution. +- **Extra-config callback returns an array** `(t) => [ ... ]` in drizzle-orm 0.36+. The old object form `(t) => ({ ... })` is deprecated. +- **`NULLS LAST` / `NULLS FIRST`** aren't on the `desc()` helper โ€” use raw `sql\`col desc nulls last\`` in `orderBy`. +- **Don't `SELECT *` into list views** โ€” explicitly pick columns so you never ship a megabyte `messages` blob or a session token to a list query. +- **Adding a dependency**: edit `package.json`, then `pnpm i` from the repo root (never `pnpm add`). Mind the repo's `minimumReleaseAge` (3 days) โ€” pin with a caret range and let pnpm resolve an old-enough version. + +## Reference (official docs) + +- Schema declaration โ€” https://orm.drizzle.team/docs/sql-schema-declaration +- PostgreSQL column types โ€” https://orm.drizzle.team/docs/column-types/pg +- Schemas (`pgSchema`) โ€” https://orm.drizzle.team/docs/schemas +- Indexes & constraints โ€” https://orm.drizzle.team/docs/indexes-constraints +- Connect (postgres-js) โ€” https://orm.drizzle.team/docs/connect-overview +- Select / Insert / Update / Delete โ€” https://orm.drizzle.team/docs/select ยท /insert ยท /update ยท /delete +- Joins / Operators โ€” https://orm.drizzle.team/docs/joins ยท /operators +- Transactions โ€” https://orm.drizzle.team/docs/transactions +- drizzle-kit (generate / migrate / push) โ€” https://orm.drizzle.team/docs/kit-overview diff --git a/.claude/skills/errors-api-e2e/SKILL.md b/.claude/skills/errors-api-e2e/SKILL.md new file mode 100644 index 0000000000..162526ee44 --- /dev/null +++ b/.claude/skills/errors-api-e2e/SKILL.md @@ -0,0 +1,199 @@ +--- +name: errors-api-e2e +description: End-to-end smoke test for the public Errors HTTP API (error groups). Seeds failed runs into ClickHouse so the error materialized views populate, then drives the real endpoints against the running webapp โ€” list (with filters + pagination), retrieve, resolve/ignore/unresolve, the `filter[error]` runs filter, user attribution via the `trigger.dev mint-token` -> JWT exchange, and the 401/403/404 negatives. Use for "smoke test the errors API", "test the errors API e2e", "prove the errors endpoints work", or to re-verify after changes. +allowed-tools: Read, Bash +--- + +# Errors API โ€” end-to-end smoke test + +Proves the public Errors API against the **running** webapp with real HTTP. No +mocks. The error data plane is ClickHouse (`errors_v1` + `error_occurrences_v1`, +both materialized-view-fed from `task_runs_v2`) plus Postgres `ErrorGroupState` +for lifecycle status; this skill seeds straight into `task_runs_v2` and lets the +MVs do the rest. + +Code under test: +- `apps/webapp/app/routes/api.v1.errors.ts` โ€” `GET /api/v1/errors` (list). +- `apps/webapp/app/routes/api.v1.errors.$errorId.ts` โ€” `GET /api/v1/errors/:errorId` (detail). +- `apps/webapp/app/routes/api.v1.errors.$errorId.{resolve,ignore,unresolve}.ts` โ€” state actions. +- `apps/webapp/app/presenters/v3/ApiErrorListPresenter.server.ts` / `ApiErrorGroupPresenter.server.ts`. +- `apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts` โ€” the `filter[error]` addition on `GET /api/v1/runs`. +- `apps/webapp/app/v3/services/errorGroupActions.server.ts` โ€” resolve/ignore/unresolve (nullable `userId`). +- Attribution: `api.v1.projects.$projectRef.$env.jwt.ts` stamps `act:{sub}` for PAT **and** UAT exchanges; `@trigger.dev/rbac` surfaces `act.sub` through bearer auth; the action handlers read `authentication.actor?.sub`. + +`errorId` is `error_` (round-trips via `ErrorId` in `@trigger.dev/core/v3/isomorphic`). + +## Prerequisites + +- Webapp running on http://localhost:3030 (`pnpm run dev --filter webapp`). Confirm `curl -s http://localhost:3030/healthcheck`. +- DB seeded (`pnpm run db:seed`), and a local ClickHouse reachable at `CLICKHOUSE_URL` (the `pnpm run docker` stack). +- The CLI built + logged in to localhost:3030 (`pnpm run build --filter trigger.dev`; profile `default` points at localhost:3030). Needed only for the attribution leg. + +> Important wiring facts the seed relies on (verified): +> - The MVs read the error type/message from `error.data.*`, so the seeded +> `error` JSON column **must** be wrapped: `{"data": {"type": ..., "message": ..., "stack": ...}}`. +> - The MVs only fire for failed statuses: `SYSTEM_FAILURE | CRASHED | INTERRUPTED | COMPLETED_WITH_ERRORS | TIMED_OUT`, and require a non-empty `error_fingerprint`. +> - `GET /api/v1/runs` lists run **ids** from ClickHouse but **hydrates from Postgres** `TaskRun`. So the error-list/detail/action legs work from a ClickHouse-only seed, but the `filter[error]` leg needs a **paired** Postgres `TaskRun` row whose `id` equals the ClickHouse `run_id`. + +Run everything from the repo root in one shell. Invoke the built CLI via a +function (a `CLI="node โ€ฆ"` variable won't word-split under zsh): +```bash +cli() { node packages/cli-v3/dist/esm/index.js "$@"; } +PROFILE=default +``` + +## Setup โ€” resolve a dev environment + connection strings + +```bash +cd apps/webapp +CHURL=$(grep -E "^CLICKHOUSE_URL=" .env | head -1 | cut -d= -f2- | tr -d '"') +DBURL=$(grep -E "^DATABASE_URL=" .env | head -1 | cut -d= -f2- | tr -d '"' | tr -d "'" | sed 's/?.*//') + +# Pick the seeded hello-world dev env (proj_rrkpdguyagvsoktglnod). Adjust the +# WHERE if you want a different project. +read ENV ORG PROJ REF < <(psql "$DBURL" -t -A -F' ' -c " + SELECT re.id, re.\"organizationId\", re.\"projectId\", p.\"externalRef\" + FROM \"RuntimeEnvironment\" re + JOIN \"Project\" p ON p.id = re.\"projectId\" + WHERE re.slug='dev' AND p.\"externalRef\"='proj_rrkpdguyagvsoktglnod' LIMIT 1;") +APIKEY=$(psql "$DBURL" -t -A -c "SELECT \"apiKey\" FROM \"RuntimeEnvironment\" WHERE id='$ENV';") +cd .. +H="Authorization: Bearer $APIKEY" +B="http://localhost:3030" +``` + +## Steps + +### 1. Seed two error groups (ClickHouse, MV-fed) + +```bash +RUN=$(node -e 'console.log(Date.now().toString(36))') +TASK="errors-api-e2e-$RUN"; FP_A="fpA${RUN}"; FP_B="fpB${RUN}" +ERRID_A="error_$FP_A"; ERRID_B="error_$FP_B" +NOW_CH=$(node -e 'console.log(new Date().toISOString().replace("T"," ").replace("Z","").slice(0,23))') +NOW_MS=$(node -e 'console.log(Date.now())') +Q=$(python3 -c "import urllib.parse;print(urllib.parse.quote('INSERT INTO trigger_dev.task_runs_v2 FORMAT JSONEachRow'))") + +mkrow() { # status fingerprint errorType message runId + echo "{\"environment_id\":\"$ENV\",\"organization_id\":\"$ORG\",\"project_id\":\"$PROJ\",\"run_id\":\"$5\",\"friendly_id\":\"run_$5\",\"status\":\"$1\",\"environment_type\":\"DEVELOPMENT\",\"engine\":\"V2\",\"task_identifier\":\"$TASK\",\"created_at\":\"$NOW_CH\",\"updated_at\":\"$NOW_CH\",\"error\":{\"data\":{\"type\":\"$3\",\"message\":\"$4\",\"stack\":\"at x (a.ts:1:1)\"}},\"error_fingerprint\":\"$2\",\"task_version\":\"20240101.1\",\"_version\":\"$NOW_MS\",\"_is_deleted\":0}" +} +ROWS="$(mkrow COMPLETED_WITH_ERRORS $FP_A AlphaBoom 'alpha boom happened' r_a1_$RUN) +$(mkrow COMPLETED_WITH_ERRORS $FP_A AlphaBoom 'alpha boom happened' r_a2_$RUN) +$(mkrow CRASHED $FP_B BetaCrash 'beta crash happened' r_b1_$RUN)" +printf '%s' "$ROWS" | curl -s "$CHURL/?query=$Q" --data-binary @- + +# Poll until both fingerprints appear in errors_v1 (the MV is near-instant locally). +for i in $(seq 1 10); do + N=$(curl -s "$CHURL" --data-binary "SELECT count() FROM (SELECT 1 FROM trigger_dev.errors_v1 WHERE environment_id='$ENV' AND error_fingerprint IN ('$FP_A','$FP_B') GROUP BY error_fingerprint)") + [ "$N" = "2" ] && break; sleep 1 +done +echo "seeded fingerprints in errors_v1: $N (want 2)" +``` +PASS: `N = 2`. Alpha has 2 occurrences, beta 1. + +### 2. List + filters + pagination + +```bash +curl -s "$B/api/v1/errors?filter%5BtaskIdentifier%5D=$TASK&filter%5Bperiod%5D=1d" -H "$H" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print('count',len(d['data']),[(e['id'],e['status'],e['count']) for e in d['data']])" +``` +PASS: 2 groups, both `status=unresolved`, alpha `count=2`, beta `count=1`, ids `error_`. + +Assert each filter narrows correctly (each should return the noted shape): +```bash +curl -s "$B/api/v1/errors?filter%5BtaskIdentifier%5D=$TASK&filter%5Bstatus%5D=unresolved&filter%5Bperiod%5D=1d" -H "$H" | python3 -c "import sys,json;print('unresolved:',len(json.load(sys.stdin)['data']))" # 2 +curl -s "$B/api/v1/errors?filter%5BtaskIdentifier%5D=$TASK&filter%5Bsearch%5D=AlphaBoom&filter%5Bperiod%5D=1d" -H "$H" | python3 -c "import sys,json;print('search:',[e['errorType'] for e in json.load(sys.stdin)['data']])" # ['AlphaBoom'] +curl -s "$B/api/v1/errors?filter%5BtaskIdentifier%5D=$TASK&filter%5Bperiod%5D=1d&page%5Bsize%5D=1" -H "$H" | python3 -c "import sys,json;d=json.load(sys.stdin);print('page size 1:',len(d['data']),'next?',bool(d['pagination'].get('next')))" # 1 / True +``` +PASS: `unresolved: 2`, `search: ['AlphaBoom']`, `page size 1: 1 / next? True`. + +### 3. Retrieve detail + +```bash +curl -s "$B/api/v1/errors/$ERRID_A" -H "$H" \ + | python3 -c "import sys,json;d=json.load(sys.stdin);print(d['id'],d['errorType'],d['status'],d['count'],d['affectedVersions'],d['resolvedBy'])" +``` +PASS: `error_ AlphaBoom unresolved 2 ['20240101.1'] None`. + +### 4. Resolve / ignore / unresolve (env API key โ€” `resolvedBy` null) + +```bash +st(){ python3 -c "import sys,json;d=json.load(sys.stdin);print('status',d['status'],'| resolvedInVersion',d['resolvedInVersion'],'| resolvedBy',d['resolvedBy'],'| ignoredUntil',bool(d['ignoredUntil']),'| reason',d['ignoredReason'])"; } + +curl -s -X POST "$B/api/v1/errors/$ERRID_A/resolve" -H "$H" -H 'Content-Type: application/json' -d '{"resolvedInVersion":"20240101.1"}' >/dev/null +curl -s "$B/api/v1/errors/$ERRID_A" -H "$H" | st # status resolved | resolvedInVersion 20240101.1 | resolvedBy None + +curl -s -X POST "$B/api/v1/errors/$ERRID_B/ignore" -H "$H" -H 'Content-Type: application/json' -d '{"duration":3600000,"reason":"known flake"}' >/dev/null +curl -s "$B/api/v1/errors/$ERRID_B" -H "$H" | st # status ignored | ignoredUntil True | reason known flake + +curl -s -X POST "$B/api/v1/errors/$ERRID_A/unresolve" -H "$H" >/dev/null +curl -s "$B/api/v1/errors/$ERRID_A" -H "$H" | st # status unresolved +``` +PASS: each transition reflected; `filter[status]=ignored` returns only beta: +```bash +curl -s "$B/api/v1/errors?filter%5BtaskIdentifier%5D=$TASK&filter%5Bstatus%5D=ignored&filter%5Bperiod%5D=1d" -H "$H" | python3 -c "import sys,json;print([e['id'] for e in json.load(sys.stdin)['data']])" # [error_] +``` + +### 5. `filter[error]` on the runs list (paired PG + CH seed) + +The runs list hydrates from Postgres, so seed a matching `TaskRun` row + a CH row +that share `run_id`/`id` and carry a fingerprint: +```bash +RID="re2e${RUN}"; FRID="run_${RID}"; FP_R="fpR${RUN}" +psql "$DBURL" -v ON_ERROR_STOP=1 -c " + INSERT INTO \"TaskRun\" (id, \"friendlyId\", \"taskIdentifier\", payload, \"traceId\", \"spanId\", \"runtimeEnvironmentId\", \"projectId\", queue, status, \"createdAt\", \"updatedAt\") + VALUES ('$RID','$FRID','$TASK','{}','trace_$RID','span_$RID','$ENV','$PROJ','task/$TASK','COMPLETED_WITH_ERRORS', now(), now()) + ON CONFLICT (id) DO NOTHING;" >/dev/null +ROW="{\"environment_id\":\"$ENV\",\"organization_id\":\"$ORG\",\"project_id\":\"$PROJ\",\"run_id\":\"$RID\",\"friendly_id\":\"$FRID\",\"status\":\"COMPLETED_WITH_ERRORS\",\"environment_type\":\"DEVELOPMENT\",\"engine\":\"V2\",\"task_identifier\":\"$TASK\",\"created_at\":\"$NOW_CH\",\"updated_at\":\"$NOW_CH\",\"error\":{\"data\":{\"type\":\"RunsFilterErr\",\"message\":\"for runs filter\",\"stack\":\"at x\"}},\"error_fingerprint\":\"$FP_R\",\"task_version\":\"20240101.1\",\"_version\":\"$NOW_MS\",\"_is_deleted\":0}" +printf '%s' "$ROW" | curl -s "$CHURL/?query=$Q" --data-binary @- +sleep 1 +curl -s "$B/api/v1/runs?filter%5Berror%5D=error_$FP_R" -H "$H" | python3 -c "import sys,json;d=json.load(sys.stdin);print('runs:',[r['id'] for r in d['data']])" +``` +PASS: one run, `run_` (status maps to `FAILED`). Proves `filter[error]` -> fingerprint -> CH -> PG hydration. + +### 6. Attribution โ€” `mint-token` -> JWT exchange records the acting user + +```bash +TOKEN=$(cli mint-token --profile $PROFILE --client errors-api-e2e --cap read:errors,write:errors 2>/dev/null) # UAT +ENVJWT=$(curl -sS -X POST "$B/api/v1/projects/$REF/dev/jwt" -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"claims":{"scopes":["read:errors","write:errors"]}}' \ + | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])") +# Decoded env JWT carries act.sub = the user id. +node -e 'const p=JSON.parse(Buffer.from(process.argv[1].split(".")[1],"base64url").toString());console.log("act:",JSON.stringify(p.act))' "$ENVJWT" + +curl -s -X POST "$B/api/v1/errors/$ERRID_A/resolve" -H "Authorization: Bearer $ENVJWT" \ + -H 'Content-Type: application/json' -d '{"resolvedInVersion":"20240101.2"}' >/dev/null +curl -s "$B/api/v1/errors/$ERRID_A" -H "$H" | python3 -c "import sys,json;d=json.load(sys.stdin);print('resolvedBy:',d['resolvedBy'])" +``` +PASS: `act.sub` is the user id (matches `cli whoami`), and `detail.resolvedBy` equals that user id (not null). A plain env key leaves it null (step 4). A **PAT** exchanged the same way also stamps `act` โ€” repeat with the stored PAT to confirm `ignoredByUserId` attribution. + +### 7. Negatives + +```bash +curl -s -o /dev/null -w 'unknown id: %{http_code} (404)\n' "$B/api/v1/errors/error_doesnotexist0000" -H "$H" +curl -s -o /dev/null -w 'no auth list: %{http_code} (401)\n' "$B/api/v1/errors" +curl -s -o /dev/null -w 'no auth resolve: %{http_code} (401)\n' -X POST "$B/api/v1/errors/$ERRID_B/resolve" -H 'Content-Type: application/json' -d '{}' + +# read-only JWT must be denied on write, allowed on read +READJWT=$(curl -sS -X POST "$B/api/v1/projects/$REF/dev/jwt" -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"claims":{"scopes":["read:errors"]}}' | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])") +curl -s -o /dev/null -w 'read JWT write: %{http_code} (403)\n' -X POST "$B/api/v1/errors/$ERRID_B/resolve" -H "Authorization: Bearer $READJWT" -H 'Content-Type: application/json' -d '{}' +curl -s -o /dev/null -w 'read JWT read: %{http_code} (200)\n' "$B/api/v1/errors?filter%5BtaskIdentifier%5D=$TASK" -H "Authorization: Bearer $READJWT" +``` +PASS: `404`, `401`, `401`, `403`, `200` respectively. + +## Result + +Report PASS only if: step 1 lands 2 groups in `errors_v1`; step 2's filters and +pagination narrow correctly; step 3 returns the detail; step 4's resolve/ignore/ +unresolve flip status (and `filter[status]` follows); step 5's `filter[error]` +returns the paired run; step 6 records `resolvedBy` = the acting user via the +JWT exchange (null with a plain env key); and step 7 returns 404/401/401/403/200. +A red leg is a bug or a missing prereq โ€” report the exact status + body and file +a Linear issue, don't tune around it. + +## Notes / gotchas + +- Run files use a unique `$RUN` suffix per invocation, so reruns don't collide and seeded rows stay isolated by their unique task identifier. They are local-dev test rows (90-day ClickHouse TTL); no cleanup required. +- After **adding** the route files, the classic Remix dev compiler may not register them until a dev-server restart (a stale manifest returns Remix's HTML 404 on the new paths). If `POST โ€ฆ/resolve` returns a 404 HTML page rather than 401/200, restart `pnpm run dev --filter webapp`. +- The rbac `act` extraction lives in `@trigger.dev/rbac` (a built dep). After editing it, `pnpm run build --filter @trigger.dev/rbac` and restart the webapp so the attribution leg (step 6) reflects the change. diff --git a/.claude/skills/span-timeline-events/SKILL.md b/.claude/skills/span-timeline-events/SKILL.md new file mode 100644 index 0000000000..122f49912d --- /dev/null +++ b/.claude/skills/span-timeline-events/SKILL.md @@ -0,0 +1,78 @@ +--- +name: span-timeline-events +description: Use when adding, modifying, or debugging OTel span timeline events in the trace view. Covers event structure, ClickHouse storage constraints, rendering in SpanTimeline component, admin visibility, and the step-by-step process for adding new events. +allowed-tools: Read, Write, Edit, Glob, Grep, Bash +--- + +# Span Timeline Events + +The trace view's right panel shows a timeline of events for the selected span. These are OTel span events rendered by `app/utils/timelineSpanEvents.ts` and the `SpanTimeline` component. + +## How They Work + +1. **Span events** in OTel are attached to a parent span. In ClickHouse, they're stored as separate rows with `kind: "SPAN_EVENT"` sharing the parent span's `span_id`. The `#mergeRecordsIntoSpanDetail` method reassembles them into the span's `events` array at query time. +2. The timeline only renders events whose `name` starts with `trigger.dev/` - all others are silently filtered out. +3. The **display name** comes from `properties.event` (not the span event name), mapped through `getFriendlyNameForEvent()`. +4. Events are shown on the **span they belong to** - events on one span don't appear in another span's timeline. + +## ClickHouse Storage Constraint + +When events are written to ClickHouse, `spanEventsToTaskEventV1Input()` filters out events whose `start_time` is not greater than the parent span's `startTime`. Events at or before the span start are silently dropped. This means span events must have timestamps strictly after the span's own `startTimeUnixNano`. + +## Timeline Rendering (SpanTimeline component) + +The `SpanTimeline` component in `app/components/run/RunTimeline.tsx` renders: + +1. **Events** (thin 1px line with hollow dots) - all events from `createTimelineSpanEventsFromSpanEvents()` +2. **"Started"** marker (thick cap) - at the span's `startTime` +3. **Duration bar** (thick 7px line) - from "Started" to "Finished" +4. **"Finished"** marker (thick cap) - at `startTime + duration` + +The thin line before "Started" only appears when there are events with timestamps between the span start and the first child span. For the Attempt span this works well (Dequeued -> Pod scheduled -> Launched -> etc. all happen before execution starts). Events all get `lineVariant: "light"` (thin) while the execution bar gets `variant: "normal"` (thick). + +## Trace View Sort Order + +Sibling spans (same parent) are sorted by `start_time ASC` from the ClickHouse query. The `createTreeFromFlatItems` function preserves this order. Event timestamps don't affect sort order - only the span's own `start_time`. + +## Event Structure + +```typescript +// OTel span event format +{ + name: "trigger.dev/run", // Must start with "trigger.dev/" to render + timeUnixNano: "1711200000000000000", + attributes: [ + { key: "event", value: { stringValue: "dequeue" } }, // The actual event type + { key: "duration", value: { intValue: 150 } }, // Optional: duration in ms + ] +} +``` + +## Admin-Only Events + +`getAdminOnlyForEvent()` controls visibility. Events default to **admin-only** (`true`). + +| Event | Admin-only | Friendly name | +|-------|-----------|---------------| +| `dequeue` | No | Dequeued | +| `fork` | No | Launched | +| `import` | No (if no fork event) | Importing task file | +| `create_attempt` | Yes | Attempt created | +| `lazy_payload` | Yes | Lazy attempt initialized | +| `pod_scheduled` | Yes | Pod scheduled | +| (default) | Yes | (raw event name) | + +## Adding New Timeline Events + +1. Add OTLP span event with `name: "trigger.dev/"` and `properties.event: ""` +2. Event timestamp must be strictly after the parent span's `startTimeUnixNano` (ClickHouse drops earlier events) +3. Add friendly name in `getFriendlyNameForEvent()` in `app/utils/timelineSpanEvents.ts` +4. Set admin visibility in `getAdminOnlyForEvent()` +5. Optionally add help text in `getHelpTextForEvent()` + +## Key Files + +- `app/utils/timelineSpanEvents.ts` - filtering, naming, admin logic +- `app/components/run/RunTimeline.tsx` - `SpanTimeline` component (thin line + thick bar rendering) +- `app/presenters/v3/SpanPresenter.server.ts` - loads span data including events +- `app/v3/eventRepository/clickhouseEventRepository.server.ts` - `spanEventsToTaskEventV1Input()` (storage filter), `#mergeRecordsIntoSpanDetail` (reassembly) diff --git a/.claude/skills/trigger-dev-tasks/SKILL.md b/.claude/skills/trigger-dev-tasks/SKILL.md new file mode 100644 index 0000000000..791c22c27e --- /dev/null +++ b/.claude/skills/trigger-dev-tasks/SKILL.md @@ -0,0 +1,200 @@ +--- +name: trigger-dev-tasks +description: Use this skill when writing, designing, or optimizing Trigger.dev background tasks and workflows. This includes creating reliable async tasks, implementing AI workflows, setting up scheduled jobs, structuring complex task hierarchies with subtasks, configuring build extensions for tools like ffmpeg or Puppeteer/Playwright, and handling task schemas with Zod validation. +allowed-tools: Read, Write, Edit, Glob, Grep, Bash +--- + +# Trigger.dev Task Expert + +You are an expert Trigger.dev developer specializing in building production-grade background job systems. Tasks deployed to Trigger.dev run in Node.js 21+ and use the `@trigger.dev/sdk` package. + +## Critical Rules + +1. **Always use `@trigger.dev/sdk`** - Never use `@trigger.dev/sdk/v3` or deprecated `client.defineJob` pattern +2. **Never use `node-fetch`** - Use the built-in `fetch` function +3. **Export all tasks** - Every task must be exported, including subtasks +4. **Never wrap wait/trigger calls in Promise.all** - `triggerAndWait`, `batchTriggerAndWait`, and `wait.*` calls cannot be wrapped in `Promise.all` or `Promise.allSettled` + +## Basic Task Pattern + +```ts +import { task } from "@trigger.dev/sdk"; + +export const processData = task({ + id: "process-data", + retry: { + maxAttempts: 10, + factor: 1.8, + minTimeoutInMs: 500, + maxTimeoutInMs: 30_000, + }, + run: async (payload: { userId: string; data: any[] }) => { + console.log(`Processing ${payload.data.length} items`); + return { processed: payload.data.length }; + }, +}); +``` + +## Schema Task (with validation) + +```ts +import { schemaTask } from "@trigger.dev/sdk"; +import { z } from "zod"; + +export const validatedTask = schemaTask({ + id: "validated-task", + schema: z.object({ + name: z.string(), + email: z.string().email(), + }), + run: async (payload) => { + // Payload is automatically validated and typed + return { message: `Hello ${payload.name}` }; + }, +}); +``` + +## Triggering Tasks + +### From Backend Code (type-only import to prevent dependency leakage) + +```ts +import { tasks } from "@trigger.dev/sdk"; +import type { processData } from "./trigger/tasks"; + +const handle = await tasks.trigger("process-data", { + userId: "123", + data: [{ id: 1 }], +}); +``` + +### From Inside Tasks + +```ts +export const parentTask = task({ + id: "parent-task", + run: async (payload) => { + // Trigger and wait - returns Result object, NOT direct output + const result = await childTask.triggerAndWait({ data: "value" }); + if (result.ok) { + console.log("Output:", result.output); + } else { + console.error("Failed:", result.error); + } + + // Or unwrap directly (throws on error) + const output = await childTask.triggerAndWait({ data: "value" }).unwrap(); + }, +}); +``` + +## Idempotency (Critical for Retries) + +Always use idempotency keys when triggering tasks from inside other tasks: + +```ts +import { idempotencyKeys } from "@trigger.dev/sdk"; + +export const paymentTask = task({ + id: "process-payment", + run: async (payload: { orderId: string }) => { + // Scoped to current run - survives retries + const key = await idempotencyKeys.create(`payment-${payload.orderId}`); + + await chargeCustomer.trigger(payload, { + idempotencyKey: key, + idempotencyKeyTTL: "24h", + }); + }, +}); +``` + +## Trigger Options + +```ts +await myTask.trigger(payload, { + delay: "1h", // Delay execution + ttl: "10m", // Cancel if not started within TTL + idempotencyKey: key, + queue: "my-queue", + machine: "large-1x", // micro, small-1x, small-2x, medium-1x, medium-2x, large-1x, large-2x + maxAttempts: 3, + tags: ["user_123"], // Max 10 tags + debounce: { // Consolidate rapid triggers + key: "unique-key", + delay: "5s", + mode: "trailing", // "leading" (default) or "trailing" + }, +}); +``` + +## Debouncing + +Consolidate multiple triggers into a single execution: + +```ts +// Rapid triggers with same key = single execution +await myTask.trigger({ userId: "123" }, { + debounce: { + key: "user-123-update", + delay: "5s", + }, +}); + +// Trailing mode: use payload from LAST trigger +await myTask.trigger({ data: "latest" }, { + debounce: { + key: "my-key", + delay: "10s", + mode: "trailing", + }, +}); +``` + +Use cases: user activity updates, webhook deduplication, search indexing, notification batching. + +## Batch Triggering + +Up to 1,000 items per batch, 3MB per payload: + +```ts +const results = await myTask.batchTriggerAndWait([ + { payload: { userId: "1" } }, + { payload: { userId: "2" } }, +]); + +for (const result of results) { + if (result.ok) console.log(result.output); +} +``` + +## Machine Presets + +| Preset | vCPU | Memory | +|-------------|------|--------| +| micro | 0.25 | 0.25GB | +| small-1x | 0.5 | 0.5GB | +| small-2x | 1 | 1GB | +| medium-1x | 1 | 2GB | +| medium-2x | 2 | 4GB | +| large-1x | 4 | 8GB | +| large-2x | 8 | 16GB | + +## Design Principles + +1. **Break complex workflows into subtasks** that can be independently retried and made idempotent +2. **Don't over-complicate** - Sometimes `Promise.allSettled` inside a single task is better than many subtasks (each task has dedicated process and is charged by millisecond) +3. **Always configure retries** - Set appropriate `maxAttempts` based on the operation +4. **Use idempotency keys** - Especially for payment/critical operations +5. **Group related subtasks** - Keep subtasks only used by one parent in the same file, don't export them +6. **Use logger** - Log at key execution points with `logger.info()`, `logger.error()`, etc. + +## Reference Documentation + +For detailed documentation on specific topics, read these files: + +- `basic-tasks.md` - Task basics, triggering, waits +- `advanced-tasks.md` - Tags, queues, concurrency, metadata, error handling +- `scheduled-tasks.md` - Cron schedules, declarative and imperative +- `realtime.md` - Real-time subscriptions, streams, React hooks +- `config.md` - trigger.config.ts, build extensions (Prisma, Playwright, FFmpeg, etc.) diff --git a/.claude/skills/trigger-dev-tasks/advanced-tasks.md b/.claude/skills/trigger-dev-tasks/advanced-tasks.md new file mode 100644 index 0000000000..32a00337f8 --- /dev/null +++ b/.claude/skills/trigger-dev-tasks/advanced-tasks.md @@ -0,0 +1,485 @@ +# Trigger.dev Advanced Tasks (v4) + +**Advanced patterns and features for writing tasks** + +## Tags & Organization + +```ts +import { task, tags } from "@trigger.dev/sdk"; + +export const processUser = task({ + id: "process-user", + run: async (payload: { userId: string; orgId: string }, { ctx }) => { + // Add tags during execution + await tags.add(`user_${payload.userId}`); + await tags.add(`org_${payload.orgId}`); + + return { processed: true }; + }, +}); + +// Trigger with tags +await processUser.trigger( + { userId: "123", orgId: "abc" }, + { tags: ["priority", "user_123", "org_abc"] } // Max 10 tags per run +); + +// Subscribe to tagged runs +for await (const run of runs.subscribeToRunsWithTag("user_123")) { + console.log(`User task ${run.id}: ${run.status}`); +} +``` + +**Tag Best Practices:** + +- Use prefixes: `user_123`, `org_abc`, `video:456` +- Max 10 tags per run, 1-64 characters each +- Tags don't propagate to child tasks automatically + +## Batch Triggering v2 + +Enhanced batch triggering with larger payloads and streaming ingestion. + +### Limits + +- **Maximum batch size**: 1,000 items (increased from 500) +- **Payload per item**: 3MB each (increased from 1MB combined) +- Payloads > 512KB automatically offload to object storage + +### Rate Limiting (per environment) + +| Tier | Bucket Size | Refill Rate | +|------|-------------|-------------| +| Free | 1,200 runs | 100 runs/10 sec | +| Hobby | 5,000 runs | 500 runs/5 sec | +| Pro | 5,000 runs | 500 runs/5 sec | + +### Concurrent Batch Processing + +| Tier | Concurrent Batches | +|------|-------------------| +| Free | 1 | +| Hobby | 10 | +| Pro | 10 | + +### Usage + +```ts +import { myTask } from "./trigger/myTask"; + +// Basic batch trigger (up to 1,000 items) +const runs = await myTask.batchTrigger([ + { payload: { userId: "user-1" } }, + { payload: { userId: "user-2" } }, + { payload: { userId: "user-3" } }, +]); + +// Batch trigger with wait +const results = await myTask.batchTriggerAndWait([ + { payload: { userId: "user-1" } }, + { payload: { userId: "user-2" } }, +]); + +for (const result of results) { + if (result.ok) { + console.log("Result:", result.output); + } +} + +// With per-item options +const batchHandle = await myTask.batchTrigger([ + { + payload: { userId: "123" }, + options: { + idempotencyKey: "user-123-batch", + tags: ["priority"], + }, + }, + { + payload: { userId: "456" }, + options: { + idempotencyKey: "user-456-batch", + }, + }, +]); +``` + +## Debouncing + +Consolidate multiple triggers into a single execution by debouncing task runs with a unique key and delay window. + +### Use Cases + +- **User activity updates**: Batch rapid user actions into a single run +- **Webhook deduplication**: Handle webhook bursts without redundant processing +- **Search indexing**: Combine document updates instead of processing individually +- **Notification batching**: Group notifications to prevent user spam + +### Basic Usage + +```ts +await myTask.trigger( + { userId: "123" }, + { + debounce: { + key: "user-123-update", // Unique identifier for debounce group + delay: "5s", // Wait duration ("5s", "1m", or milliseconds) + }, + } +); +``` + +### Execution Modes + +**Leading Mode** (default): Uses payload/options from the first trigger; subsequent triggers only reschedule execution time. + +```ts +// First trigger sets the payload +await myTask.trigger({ action: "first" }, { + debounce: { key: "my-key", delay: "10s" } +}); + +// Second trigger only reschedules - payload remains "first" +await myTask.trigger({ action: "second" }, { + debounce: { key: "my-key", delay: "10s" } +}); +// Task executes with { action: "first" } +``` + +**Trailing Mode**: Uses payload/options from the most recent trigger. + +```ts +await myTask.trigger( + { data: "latest-value" }, + { + debounce: { + key: "trailing-example", + delay: "10s", + mode: "trailing", + }, + } +); +``` + +In trailing mode, these options update with each trigger: +- `payload` โ€” task input data +- `metadata` โ€” run metadata +- `tags` โ€” run tags (replaces existing) +- `maxAttempts` โ€” retry attempts +- `maxDuration` โ€” maximum compute time +- `machine` โ€” machine preset + +### Important Notes + +- Idempotency keys take precedence over debounce settings +- Compatible with `triggerAndWait()` โ€” parent runs block correctly on debounced execution +- Debounce key is scoped to the task + +## Concurrency & Queues + +```ts +import { task, queue } from "@trigger.dev/sdk"; + +// Shared queue for related tasks +const emailQueue = queue({ + name: "email-processing", + concurrencyLimit: 5, // Max 5 emails processing simultaneously +}); + +// Task-level concurrency +export const oneAtATime = task({ + id: "sequential-task", + queue: { concurrencyLimit: 1 }, // Process one at a time + run: async (payload) => { + // Critical section - only one instance runs + }, +}); + +// Per-user concurrency +export const processUserData = task({ + id: "process-user-data", + run: async (payload: { userId: string }) => { + // Override queue with user-specific concurrency + await childTask.trigger(payload, { + queue: { + name: `user-${payload.userId}`, + concurrencyLimit: 2, + }, + }); + }, +}); + +export const emailTask = task({ + id: "send-email", + queue: emailQueue, // Use shared queue + run: async (payload: { to: string }) => { + // Send email logic + }, +}); +``` + +## Error Handling & Retries + +```ts +import { task, retry, AbortTaskRunError } from "@trigger.dev/sdk"; + +export const resilientTask = task({ + id: "resilient-task", + retry: { + maxAttempts: 10, + factor: 1.8, // Exponential backoff multiplier + minTimeoutInMs: 500, + maxTimeoutInMs: 30_000, + randomize: false, + }, + catchError: async ({ error, ctx }) => { + // Custom error handling + if (error.code === "FATAL_ERROR") { + throw new AbortTaskRunError("Cannot retry this error"); + } + + // Log error details + console.error(`Task ${ctx.task.id} failed:`, error); + + // Allow retry by returning nothing + return { retryAt: new Date(Date.now() + 60000) }; // Retry in 1 minute + }, + run: async (payload) => { + // Retry specific operations + const result = await retry.onThrow( + async () => { + return await unstableApiCall(payload); + }, + { maxAttempts: 3 } + ); + + // Conditional HTTP retries + const response = await retry.fetch("https://api.example.com", { + retry: { + maxAttempts: 5, + condition: (response, error) => { + return response?.status === 429 || response?.status >= 500; + }, + }, + }); + + return result; + }, +}); +``` + +## Machines & Performance + +```ts +export const heavyTask = task({ + id: "heavy-computation", + machine: { preset: "large-2x" }, // 8 vCPU, 16 GB RAM + maxDuration: 1800, // 30 minutes timeout + run: async (payload, { ctx }) => { + // Resource-intensive computation + if (ctx.machine.preset === "large-2x") { + // Use all available cores + return await parallelProcessing(payload); + } + + return await standardProcessing(payload); + }, +}); + +// Override machine when triggering +await heavyTask.trigger(payload, { + machine: { preset: "medium-1x" }, // Override for this run +}); +``` + +**Machine Presets:** + +- `micro`: 0.25 vCPU, 0.25 GB RAM +- `small-1x`: 0.5 vCPU, 0.5 GB RAM (default) +- `small-2x`: 1 vCPU, 1 GB RAM +- `medium-1x`: 1 vCPU, 2 GB RAM +- `medium-2x`: 2 vCPU, 4 GB RAM +- `large-1x`: 4 vCPU, 8 GB RAM +- `large-2x`: 8 vCPU, 16 GB RAM + +## Idempotency + +```ts +import { task, idempotencyKeys } from "@trigger.dev/sdk"; + +export const paymentTask = task({ + id: "process-payment", + retry: { + maxAttempts: 3, + }, + run: async (payload: { orderId: string; amount: number }) => { + // Automatically scoped to this task run, so if the task is retried, the idempotency key will be the same + const idempotencyKey = await idempotencyKeys.create(`payment-${payload.orderId}`); + + // Ensure payment is processed only once + await chargeCustomer.trigger(payload, { + idempotencyKey, + idempotencyKeyTTL: "24h", // Key expires in 24 hours + }); + }, +}); + +// Payload-based idempotency +import { createHash } from "node:crypto"; + +function createPayloadHash(payload: any): string { + const hash = createHash("sha256"); + hash.update(JSON.stringify(payload)); + return hash.digest("hex"); +} + +export const deduplicatedTask = task({ + id: "deduplicated-task", + run: async (payload) => { + const payloadHash = createPayloadHash(payload); + const idempotencyKey = await idempotencyKeys.create(payloadHash); + + await processData.trigger(payload, { idempotencyKey }); + }, +}); +``` + +## Metadata & Progress Tracking + +```ts +import { task, metadata } from "@trigger.dev/sdk"; + +export const batchProcessor = task({ + id: "batch-processor", + run: async (payload: { items: any[] }, { ctx }) => { + const totalItems = payload.items.length; + + // Initialize progress metadata + metadata + .set("progress", 0) + .set("totalItems", totalItems) + .set("processedItems", 0) + .set("status", "starting"); + + const results = []; + + for (let i = 0; i < payload.items.length; i++) { + const item = payload.items[i]; + + // Process item + const result = await processItem(item); + results.push(result); + + // Update progress + const progress = ((i + 1) / totalItems) * 100; + metadata + .set("progress", progress) + .increment("processedItems", 1) + .append("logs", `Processed item ${i + 1}/${totalItems}`) + .set("currentItem", item.id); + } + + // Final status + metadata.set("status", "completed"); + + return { results, totalProcessed: results.length }; + }, +}); + +// Update parent metadata from child task +export const childTask = task({ + id: "child-task", + run: async (payload, { ctx }) => { + // Update parent task metadata + metadata.parent.set("childStatus", "processing"); + metadata.root.increment("childrenCompleted", 1); + + return { processed: true }; + }, +}); +``` + +## Logging & Tracing + +```ts +import { task, logger } from "@trigger.dev/sdk"; + +export const tracedTask = task({ + id: "traced-task", + run: async (payload, { ctx }) => { + logger.info("Task started", { userId: payload.userId }); + + // Custom trace with attributes + const user = await logger.trace( + "fetch-user", + async (span) => { + span.setAttribute("user.id", payload.userId); + span.setAttribute("operation", "database-fetch"); + + const userData = await database.findUser(payload.userId); + span.setAttribute("user.found", !!userData); + + return userData; + }, + { userId: payload.userId } + ); + + logger.debug("User fetched", { user: user.id }); + + try { + const result = await processUser(user); + logger.info("Processing completed", { result }); + return result; + } catch (error) { + logger.error("Processing failed", { + error: error.message, + userId: payload.userId, + }); + throw error; + } + }, +}); +``` + +## Hidden Tasks + +```ts +// Hidden task - not exported, only used internally +const internalProcessor = task({ + id: "internal-processor", + run: async (payload: { data: string }) => { + return { processed: payload.data.toUpperCase() }; + }, +}); + +// Public task that uses hidden task +export const publicWorkflow = task({ + id: "public-workflow", + run: async (payload: { input: string }) => { + // Use hidden task internally + const result = await internalProcessor.triggerAndWait({ + data: payload.input, + }); + + if (result.ok) { + return { output: result.output.processed }; + } + + throw new Error("Internal processing failed"); + }, +}); +``` + +## Best Practices + +- **Concurrency**: Use queues to prevent overwhelming external services +- **Retries**: Configure exponential backoff for transient failures +- **Idempotency**: Always use for payment/critical operations +- **Metadata**: Track progress for long-running tasks +- **Machines**: Match machine size to computational requirements +- **Tags**: Use consistent naming patterns for filtering +- **Debouncing**: Use for user activity, webhooks, and notification batching +- **Batch triggering**: Use for bulk operations up to 1,000 items +- **Error Handling**: Distinguish between retryable and fatal errors + +Design tasks to be stateless, idempotent, and resilient to failures. Use metadata for state tracking and queues for resource management. diff --git a/.claude/skills/trigger-dev-tasks/basic-tasks.md b/.claude/skills/trigger-dev-tasks/basic-tasks.md new file mode 100644 index 0000000000..56bff34076 --- /dev/null +++ b/.claude/skills/trigger-dev-tasks/basic-tasks.md @@ -0,0 +1,199 @@ +# Trigger.dev Basic Tasks (v4) + +**MUST use `@trigger.dev/sdk`, NEVER `client.defineJob`** + +## Basic Task + +```ts +import { task } from "@trigger.dev/sdk"; + +export const processData = task({ + id: "process-data", + retry: { + maxAttempts: 10, + factor: 1.8, + minTimeoutInMs: 500, + maxTimeoutInMs: 30_000, + randomize: false, + }, + run: async (payload: { userId: string; data: any[] }) => { + // Task logic - runs for long time, no timeouts + console.log(`Processing ${payload.data.length} items for user ${payload.userId}`); + return { processed: payload.data.length }; + }, +}); +``` + +## Schema Task (with validation) + +```ts +import { schemaTask } from "@trigger.dev/sdk"; +import { z } from "zod"; + +export const validatedTask = schemaTask({ + id: "validated-task", + schema: z.object({ + name: z.string(), + age: z.number(), + email: z.string().email(), + }), + run: async (payload) => { + // Payload is automatically validated and typed + return { message: `Hello ${payload.name}, age ${payload.age}` }; + }, +}); +``` + +## Triggering Tasks + +### From Backend Code + +```ts +import { tasks } from "@trigger.dev/sdk"; +import type { processData } from "./trigger/tasks"; + +// Single trigger +const handle = await tasks.trigger("process-data", { + userId: "123", + data: [{ id: 1 }, { id: 2 }], +}); + +// Batch trigger (up to 1,000 items, 3MB per payload) +const batchHandle = await tasks.batchTrigger("process-data", [ + { payload: { userId: "123", data: [{ id: 1 }] } }, + { payload: { userId: "456", data: [{ id: 2 }] } }, +]); +``` + +### Debounced Triggering + +Consolidate multiple triggers into a single execution: + +```ts +// Multiple rapid triggers with same key = single execution +await myTask.trigger( + { userId: "123" }, + { + debounce: { + key: "user-123-update", // Unique key for debounce group + delay: "5s", // Wait before executing + }, + } +); + +// Trailing mode: use payload from LAST trigger +await myTask.trigger( + { data: "latest-value" }, + { + debounce: { + key: "trailing-example", + delay: "10s", + mode: "trailing", // Default is "leading" (first payload) + }, + } +); +``` + +**Debounce modes:** +- `leading` (default): Uses payload from first trigger, subsequent triggers only reschedule +- `trailing`: Uses payload from most recent trigger + +### From Inside Tasks (with Result handling) + +```ts +export const parentTask = task({ + id: "parent-task", + run: async (payload) => { + // Trigger and continue + const handle = await childTask.trigger({ data: "value" }); + + // Trigger and wait - returns Result object, NOT task output + const result = await childTask.triggerAndWait({ data: "value" }); + if (result.ok) { + console.log("Task output:", result.output); // Actual task return value + } else { + console.error("Task failed:", result.error); + } + + // Quick unwrap (throws on error) + const output = await childTask.triggerAndWait({ data: "value" }).unwrap(); + + // Batch trigger and wait + const results = await childTask.batchTriggerAndWait([ + { payload: { data: "item1" } }, + { payload: { data: "item2" } }, + ]); + + for (const run of results) { + if (run.ok) { + console.log("Success:", run.output); + } else { + console.log("Failed:", run.error); + } + } + }, +}); + +export const childTask = task({ + id: "child-task", + run: async (payload: { data: string }) => { + return { processed: payload.data }; + }, +}); +``` + +> Never wrap triggerAndWait or batchTriggerAndWait calls in a Promise.all or Promise.allSettled as this is not supported in Trigger.dev tasks. + +## Waits + +```ts +import { task, wait } from "@trigger.dev/sdk"; + +export const taskWithWaits = task({ + id: "task-with-waits", + run: async (payload) => { + console.log("Starting task"); + + // Wait for specific duration + await wait.for({ seconds: 30 }); + await wait.for({ minutes: 5 }); + await wait.for({ hours: 1 }); + await wait.for({ days: 1 }); + + // Wait until specific date + await wait.until({ date: new Date("2024-12-25") }); + + // Wait for token (from external system) + await wait.forToken({ + token: "user-approval-token", + timeoutInSeconds: 3600, // 1 hour timeout + }); + + console.log("All waits completed"); + return { status: "completed" }; + }, +}); +``` + +> Never wrap wait calls in a Promise.all or Promise.allSettled as this is not supported in Trigger.dev tasks. + +## Key Points + +- **Result vs Output**: `triggerAndWait()` returns a `Result` object with `ok`, `output`, `error` properties - NOT the direct task output +- **Type safety**: Use `import type` for task references when triggering from backend +- **Waits > 5 seconds**: Automatically checkpointed, don't count toward compute usage +- **Debounce + idempotency**: Idempotency keys take precedence over debounce settings + +## NEVER Use (v2 deprecated) + +```ts +// BREAKS APPLICATION +client.defineJob({ + id: "job-id", + run: async (payload, io) => { + /* ... */ + }, +}); +``` + +Use SDK (`@trigger.dev/sdk`), check `result.ok` before accessing `result.output` diff --git a/.claude/skills/trigger-dev-tasks/config.md b/.claude/skills/trigger-dev-tasks/config.md new file mode 100644 index 0000000000..f6a4db1c4b --- /dev/null +++ b/.claude/skills/trigger-dev-tasks/config.md @@ -0,0 +1,346 @@ +# Trigger.dev Configuration + +**Complete guide to configuring `trigger.config.ts` with build extensions** + +## Basic Configuration + +```ts +import { defineConfig } from "@trigger.dev/sdk"; + +export default defineConfig({ + project: "", // Required: Your project reference + dirs: ["./trigger"], // Task directories + runtime: "node", // "node", "node-22", or "bun" + logLevel: "info", // "debug", "info", "warn", "error" + + // Default retry settings + retries: { + enabledInDev: false, + default: { + maxAttempts: 3, + minTimeoutInMs: 1000, + maxTimeoutInMs: 10000, + factor: 2, + randomize: true, + }, + }, + + // Build configuration + build: { + autoDetectExternal: true, + keepNames: true, + minify: false, + extensions: [], // Build extensions go here + }, + + // Global lifecycle hooks + onStartAttempt: async ({ payload, ctx }) => { + console.log("Global task start"); + }, + onSuccess: async ({ payload, output, ctx }) => { + console.log("Global task success"); + }, + onFailure: async ({ payload, error, ctx }) => { + console.log("Global task failure"); + }, +}); +``` + +## Build Extensions + +### Database & ORM + +#### Prisma + +```ts +import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; + +extensions: [ + prismaExtension({ + schema: "prisma/schema.prisma", + version: "5.19.0", // Optional: specify version + migrate: true, // Run migrations during build + directUrlEnvVarName: "DIRECT_DATABASE_URL", + typedSql: true, // Enable TypedSQL support + }), +]; +``` + +#### TypeScript Decorators (for TypeORM) + +```ts +import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; + +extensions: [ + emitDecoratorMetadata(), // Enables decorator metadata +]; +``` + +### Scripting Languages + +#### Python + +```ts +import { pythonExtension } from "@trigger.dev/build/extensions/python"; + +extensions: [ + pythonExtension({ + scripts: ["./python/**/*.py"], // Copy Python files + requirementsFile: "./requirements.txt", // Install packages + devPythonBinaryPath: ".venv/bin/python", // Dev mode binary + }), +]; + +// Usage in tasks +const result = await python.runInline(`print("Hello, world!")`); +const output = await python.runScript("./python/script.py", ["arg1"]); +``` + +### Browser Automation + +#### Playwright + +```ts +import { playwright } from "@trigger.dev/build/extensions/playwright"; + +extensions: [ + playwright({ + browsers: ["chromium", "firefox", "webkit"], // Default: ["chromium"] + headless: true, // Default: true + }), +]; +``` + +#### Puppeteer + +```ts +import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; + +extensions: [puppeteer()]; + +// Environment variable needed: +// PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable" +``` + +#### Lightpanda + +```ts +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +extensions: [ + lightpanda({ + version: "latest", // or "nightly" + disableTelemetry: false, + }), +]; +``` + +### Media Processing + +#### FFmpeg + +```ts +import { ffmpeg } from "@trigger.dev/build/extensions/core"; + +extensions: [ + ffmpeg({ version: "7" }), // Static build, or omit for Debian version +]; + +// Automatically sets FFMPEG_PATH and FFPROBE_PATH +// Add fluent-ffmpeg to external packages if using +``` + +#### Audio Waveform + +```ts +import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; + +extensions: [ + audioWaveform(), // Installs Audio Waveform 1.1.0 +]; +``` + +### System & Package Management + +#### System Packages (apt-get) + +```ts +import { aptGet } from "@trigger.dev/build/extensions/core"; + +extensions: [ + aptGet({ + packages: ["ffmpeg", "imagemagick", "curl=7.68.0-1"], // Can specify versions + }), +]; +``` + +#### Additional NPM Packages + +Only use this for installing CLI tools, NOT packages you import in your code. + +```ts +import { additionalPackages } from "@trigger.dev/build/extensions/core"; + +extensions: [ + additionalPackages({ + packages: ["wrangler"], // CLI tools and specific versions + }), +]; +``` + +#### Additional Files + +```ts +import { additionalFiles } from "@trigger.dev/build/extensions/core"; + +extensions: [ + additionalFiles({ + files: ["wrangler.toml", "./assets/**", "./fonts/**"], // Glob patterns supported + }), +]; +``` + +### Environment & Build Tools + +#### Environment Variable Sync + +```ts +import { syncEnvVars } from "@trigger.dev/build/extensions/core"; + +extensions: [ + syncEnvVars(async (ctx) => { + // ctx contains: environment, projectRef, env + return [ + { name: "SECRET_KEY", value: await getSecret(ctx.environment) }, + { name: "API_URL", value: ctx.environment === "prod" ? "api.prod.com" : "api.dev.com" }, + ]; + }), +]; +``` + +#### ESBuild Plugins + +```ts +import { esbuildPlugin } from "@trigger.dev/build/extensions"; +import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin"; + +extensions: [ + esbuildPlugin( + sentryEsbuildPlugin({ + org: process.env.SENTRY_ORG, + project: process.env.SENTRY_PROJECT, + authToken: process.env.SENTRY_AUTH_TOKEN, + }), + { placement: "last", target: "deploy" } // Optional config + ), +]; +``` + +## Custom Build Extensions + +```ts +import { defineConfig } from "@trigger.dev/sdk"; + +const customExtension = { + name: "my-custom-extension", + + externalsForTarget: (target) => { + return ["some-native-module"]; // Add external dependencies + }, + + onBuildStart: async (context) => { + console.log(`Build starting for ${context.target}`); + // Register esbuild plugins, modify build context + }, + + onBuildComplete: async (context, manifest) => { + console.log("Build complete, adding layers"); + // Add build layers, modify deployment + context.addLayer({ + id: "my-layer", + files: [{ source: "./custom-file", destination: "/app/custom" }], + commands: ["chmod +x /app/custom"], + }); + }, +}; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [customExtension], + }, +}); +``` + +## Advanced Configuration + +### Telemetry + +```ts +import { PrismaInstrumentation } from "@prisma/instrumentation"; +import { OpenAIInstrumentation } from "@langfuse/openai"; + +export default defineConfig({ + // ... other config + telemetry: { + instrumentations: [new PrismaInstrumentation(), new OpenAIInstrumentation()], + exporters: [customExporter], // Optional custom exporters + }, +}); +``` + +### Machine & Performance + +```ts +export default defineConfig({ + // ... other config + defaultMachine: "large-1x", // Default machine for all tasks + maxDuration: 300, // Default max duration (seconds) + enableConsoleLogging: true, // Console logging in development +}); +``` + +## Common Extension Combinations + +### Full-Stack Web App + +```ts +extensions: [ + prismaExtension({ schema: "prisma/schema.prisma", migrate: true }), + additionalFiles({ files: ["./public/**", "./assets/**"] }), + syncEnvVars(async (ctx) => [...envVars]), +]; +``` + +### AI/ML Processing + +```ts +extensions: [ + pythonExtension({ + scripts: ["./ai/**/*.py"], + requirementsFile: "./requirements.txt", + }), + ffmpeg({ version: "7" }), + additionalPackages({ packages: ["wrangler"] }), +]; +``` + +### Web Scraping + +```ts +extensions: [ + playwright({ browsers: ["chromium"] }), + puppeteer(), + additionalFiles({ files: ["./selectors.json", "./proxies.txt"] }), +]; +``` + +## Best Practices + +- **Use specific versions**: Pin extension versions for reproducible builds +- **External packages**: Add modules with native addons to the `build.external` array +- **Environment sync**: Use `syncEnvVars` for dynamic secrets +- **File paths**: Use glob patterns for flexible file inclusion +- **Debug builds**: Use `--log-level debug --dry-run` for troubleshooting + +Extensions only affect deployment, not local development. Use `external` array for packages that shouldn't be bundled. diff --git a/.claude/skills/trigger-dev-tasks/realtime.md b/.claude/skills/trigger-dev-tasks/realtime.md new file mode 100644 index 0000000000..c1c4c5821a --- /dev/null +++ b/.claude/skills/trigger-dev-tasks/realtime.md @@ -0,0 +1,244 @@ +# Trigger.dev Realtime + +**Real-time monitoring and updates for runs** + +## Core Concepts + +Realtime allows you to: + +- Subscribe to run status changes, metadata updates, and streams +- Build real-time dashboards and UI updates +- Monitor task progress from frontend and backend + +## Authentication + +### Public Access Tokens + +```ts +import { auth } from "@trigger.dev/sdk"; + +// Read-only token for specific runs +const publicToken = await auth.createPublicToken({ + scopes: { + read: { + runs: ["run_123", "run_456"], + tasks: ["my-task-1", "my-task-2"], + }, + }, + expirationTime: "1h", // Default: 15 minutes +}); +``` + +### Trigger Tokens (Frontend only) + +```ts +// Single-use token for triggering tasks +const triggerToken = await auth.createTriggerPublicToken("my-task", { + expirationTime: "30m", +}); +``` + +## Backend Usage + +### Subscribe to Runs + +```ts +import { runs, tasks } from "@trigger.dev/sdk"; + +// Trigger and subscribe +const handle = await tasks.trigger("my-task", { data: "value" }); + +// Subscribe to specific run +for await (const run of runs.subscribeToRun(handle.id)) { + console.log(`Status: ${run.status}, Progress: ${run.metadata?.progress}`); + if (run.status === "COMPLETED") break; +} + +// Subscribe to runs with tag +for await (const run of runs.subscribeToRunsWithTag("user-123")) { + console.log(`Tagged run ${run.id}: ${run.status}`); +} + +// Subscribe to batch +for await (const run of runs.subscribeToBatch(batchId)) { + console.log(`Batch run ${run.id}: ${run.status}`); +} +``` + +### Realtime Streams v2 + +```ts +import { streams, InferStreamType } from "@trigger.dev/sdk"; + +// 1. Define streams (shared location) +export const aiStream = streams.define({ + id: "ai-output", +}); + +export type AIStreamPart = InferStreamType; + +// 2. Pipe from task +export const streamingTask = task({ + id: "streaming-task", + run: async (payload) => { + const completion = await openai.chat.completions.create({ + model: "gpt-4", + messages: [{ role: "user", content: payload.prompt }], + stream: true, + }); + + const { waitUntilComplete } = aiStream.pipe(completion); + await waitUntilComplete(); + }, +}); + +// 3. Read from backend +const stream = await aiStream.read(runId, { + timeoutInSeconds: 300, + startIndex: 0, // Resume from specific chunk +}); + +for await (const chunk of stream) { + console.log("Chunk:", chunk); // Fully typed +} +``` + +## React Frontend Usage + +### Installation + +```bash +npm add @trigger.dev/react-hooks +``` + +### Triggering Tasks + +```tsx +"use client"; +import { useTaskTrigger, useRealtimeTaskTrigger } from "@trigger.dev/react-hooks"; +import type { myTask } from "../trigger/tasks"; + +function TriggerComponent({ accessToken }: { accessToken: string }) { + // Basic trigger + const { submit, handle, isLoading } = useTaskTrigger("my-task", { + accessToken, + }); + + // Trigger with realtime updates + const { + submit: realtimeSubmit, + run, + isLoading: isRealtimeLoading, + } = useRealtimeTaskTrigger("my-task", { accessToken }); + + return ( +
+ + + + + {run &&
Status: {run.status}
} +
+ ); +} +``` + +### Subscribing to Runs + +```tsx +"use client"; +import { useRealtimeRun, useRealtimeRunsWithTag } from "@trigger.dev/react-hooks"; +import type { myTask } from "../trigger/tasks"; + +function SubscribeComponent({ runId, accessToken }: { runId: string; accessToken: string }) { + // Subscribe to specific run + const { run, error } = useRealtimeRun(runId, { + accessToken, + onComplete: (run) => { + console.log("Task completed:", run.output); + }, + }); + + // Subscribe to tagged runs + const { runs } = useRealtimeRunsWithTag("user-123", { accessToken }); + + if (error) return
Error: {error.message}
; + if (!run) return
Loading...
; + + return ( +
+
Status: {run.status}
+
Progress: {run.metadata?.progress || 0}%
+ {run.output &&
Result: {JSON.stringify(run.output)}
} + +

Tagged Runs:

+ {runs.map((r) => ( +
+ {r.id}: {r.status} +
+ ))} +
+ ); +} +``` + +### Realtime Streams with React + +```tsx +"use client"; +import { useRealtimeStream } from "@trigger.dev/react-hooks"; +import { aiStream } from "../trigger/streams"; + +function StreamComponent({ runId, accessToken }: { runId: string; accessToken: string }) { + // Pass defined stream directly for type safety + const { parts, error } = useRealtimeStream(aiStream, runId, { + accessToken, + timeoutInSeconds: 300, + throttleInMs: 50, // Control re-render frequency + }); + + if (error) return
Error: {error.message}
; + if (!parts) return
Loading...
; + + const text = parts.join(""); // parts is typed as AIStreamPart[] + + return
Streamed Text: {text}
; +} +``` + +### Wait Tokens + +```tsx +"use client"; +import { useWaitToken } from "@trigger.dev/react-hooks"; + +function WaitTokenComponent({ tokenId, accessToken }: { tokenId: string; accessToken: string }) { + const { complete } = useWaitToken(tokenId, { accessToken }); + + return ; +} +``` + +## Run Object Properties + +Key properties available in run subscriptions: + +- `id`: Unique run identifier +- `status`: `QUEUED`, `EXECUTING`, `COMPLETED`, `FAILED`, `CANCELED`, etc. +- `payload`: Task input data (typed) +- `output`: Task result (typed, when completed) +- `metadata`: Real-time updatable data +- `createdAt`, `updatedAt`: Timestamps +- `costInCents`: Execution cost + +## Best Practices + +- **Use Realtime over SWR**: Recommended for most use cases due to rate limits +- **Scope tokens properly**: Only grant necessary read/trigger permissions +- **Handle errors**: Always check for errors in hooks and subscriptions +- **Type safety**: Use task types for proper payload/output typing +- **Cleanup subscriptions**: Backend subscriptions auto-complete, frontend hooks auto-cleanup diff --git a/.claude/skills/trigger-dev-tasks/scheduled-tasks.md b/.claude/skills/trigger-dev-tasks/scheduled-tasks.md new file mode 100644 index 0000000000..b314753497 --- /dev/null +++ b/.claude/skills/trigger-dev-tasks/scheduled-tasks.md @@ -0,0 +1,113 @@ +# Scheduled Tasks (Cron) + +Recurring tasks using cron. For one-off future runs, use the **delay** option. + +## Define a Scheduled Task + +```ts +import { schedules } from "@trigger.dev/sdk"; + +export const task = schedules.task({ + id: "first-scheduled-task", + run: async (payload) => { + payload.timestamp; // Date (scheduled time, UTC) + payload.lastTimestamp; // Date | undefined + payload.timezone; // IANA, e.g. "America/New_York" (default "UTC") + payload.scheduleId; // string + payload.externalId; // string | undefined + payload.upcoming; // Date[] + + payload.timestamp.toLocaleString("en-US", { timeZone: payload.timezone }); + }, +}); +``` + +> Scheduled tasks need at least one schedule attached to run. + +## Attach Schedules + +**Declarative (sync on dev/deploy):** + +```ts +schedules.task({ + id: "every-2h", + cron: "0 */2 * * *", // UTC + run: async () => {}, +}); + +schedules.task({ + id: "tokyo-5am", + cron: { pattern: "0 5 * * *", timezone: "Asia/Tokyo", environments: ["PRODUCTION", "STAGING"] }, + run: async () => {}, +}); +``` + +**Imperative (SDK or dashboard):** + +```ts +await schedules.create({ + task: task.id, + cron: "0 0 * * *", + timezone: "America/New_York", // DST-aware + externalId: "user_123", + deduplicationKey: "user_123-daily", // updates if reused +}); +``` + +### Dynamic / Multi-tenant Example + +```ts +// /trigger/reminder.ts +export const reminderTask = schedules.task({ + id: "todo-reminder", + run: async (p) => { + if (!p.externalId) throw new Error("externalId is required"); + const user = await db.getUser(p.externalId); + await sendReminderEmail(user); + }, +}); +``` + +```ts +// app/reminders/route.ts +export async function POST(req: Request) { + const data = await req.json(); + return Response.json( + await schedules.create({ + task: reminderTask.id, + cron: "0 8 * * *", + timezone: data.timezone, + externalId: data.userId, + deduplicationKey: `${data.userId}-reminder`, + }) + ); +} +``` + +## Cron Syntax (no seconds) + +``` +* * * * * +| | | | โ”” day of week (0โ€“7 or 1Lโ€“7L; 0/7=Sun; L=last) +| | | โ””โ”€โ”€ month (1โ€“12) +| | โ””โ”€โ”€โ”€โ”€ day of month (1โ€“31 or L) +| โ””โ”€โ”€โ”€โ”€โ”€โ”€ hour (0โ€“23) +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ minute (0โ€“59) +``` + +## When Schedules Won't Trigger + +- **Dev:** only when the dev CLI is running. +- **Staging/Production:** only for tasks in the **latest deployment**. + +## SDK Management + +```ts +await schedules.retrieve(id); +await schedules.list(); +await schedules.update(id, { cron: "0 0 1 * *", externalId: "ext", deduplicationKey: "key" }); +await schedules.deactivate(id); +await schedules.activate(id); +await schedules.del(id); +await schedules.timezones(); // list of IANA timezones +``` diff --git a/.configs/tsconfig.base.json b/.configs/tsconfig.base.json index 2d560d22d0..8910210476 100644 --- a/.configs/tsconfig.base.json +++ b/.configs/tsconfig.base.json @@ -26,7 +26,6 @@ "esModuleInterop": true, "emitDecoratorMetadata": false, "experimentalDecorators": false, - "downlevelIteration": true, "isolatedModules": true, "noUncheckedIndexedAccess": true, diff --git a/.cursor/mcp.json b/.cursor/mcp.json index da39e4ffaf..c4b06a6763 100644 --- a/.cursor/mcp.json +++ b/.cursor/mcp.json @@ -1,3 +1,7 @@ { - "mcpServers": {} + "mcpServers": { + "linear": { + "url": "https://mcp.linear.app/mcp" + } + } } diff --git a/.cursor/rules/otel-metrics.mdc b/.cursor/rules/otel-metrics.mdc new file mode 100644 index 0000000000..218f07c41e --- /dev/null +++ b/.cursor/rules/otel-metrics.mdc @@ -0,0 +1,66 @@ +--- +description: Guidelines for creating OpenTelemetry metrics to avoid cardinality issues +globs: + - "**/*.ts" +--- + +# OpenTelemetry Metrics Guidelines + +When creating or editing OTEL metrics (counters, histograms, gauges), always ensure metric attributes have **low cardinality**. + +## What is Cardinality? + +Cardinality refers to the number of unique values an attribute can have. Each unique combination of attribute values creates a new time series, which consumes memory and storage in your metrics backend. + +## Rules + +### DO use low-cardinality attributes: +- **Enums**: `environment_type` (PRODUCTION, STAGING, DEVELOPMENT, PREVIEW) +- **Booleans**: `hasFailures`, `streaming`, `success` +- **Bounded error codes**: A finite, controlled set of error types +- **Shard IDs**: When sharding is bounded (e.g., 0-15) + +### DO NOT use high-cardinality attributes: +- **UUIDs/IDs**: `envId`, `userId`, `runId`, `projectId`, `organizationId` +- **Unbounded integers**: `itemCount`, `batchSize`, `retryCount` +- **Timestamps**: `createdAt`, `startTime` +- **Free-form strings**: `errorMessage`, `taskName`, `queueName` + +## Example + +```typescript +// BAD - High cardinality +this.counter.add(1, { + envId: options.environmentId, // UUID - unbounded + itemCount: options.runCount, // Integer - unbounded +}); + +// GOOD - Low cardinality +this.counter.add(1, { + environment_type: options.environmentType, // Enum - 4 values + streaming: true, // Boolean - 2 values +}); +``` + +## Prometheus Metric Naming + +When metrics are exported via OTLP to Prometheus, the exporter automatically adds unit suffixes to metric names: + +| OTel Metric Name | Unit | Prometheus Name | +|------------------|------|-----------------| +| `my_duration_ms` | `ms` | `my_duration_ms_milliseconds` | +| `my_counter` | counter | `my_counter_total` | +| `items_inserted` | counter | `items_inserted_inserts_total` | +| `batch_size` | histogram | `batch_size_items_bucket` | + +Keep this in mind when writing Grafana dashboards or Prometheus queriesโ€”the metric names in Prometheus will differ from the names defined in code. + +## Reference + +See the schedule engine (`internal-packages/schedule-engine/src/engine/index.ts`) for a good example of low-cardinality metric attributes. + +High cardinality metrics can cause: +- Memory bloat in metrics backends (Axiom, Prometheus, etc.) +- Slow queries and dashboard timeouts +- Increased costs (many backends charge per time series) +- Potential data loss or crashes at scale diff --git a/.cursor/rules/webapp.mdc b/.cursor/rules/webapp.mdc index a362f14fe1..f1333febdc 100644 --- a/.cursor/rules/webapp.mdc +++ b/.cursor/rules/webapp.mdc @@ -4,7 +4,7 @@ globs: apps/webapp/**/*.tsx,apps/webapp/**/*.ts alwaysApply: false --- -The main trigger.dev webapp, which powers it's API and dashboard and makes up the docker image that is produced as an OSS image, is a Remix 2.1.0 app that uses an express server, written in TypeScript. The following subsystems are either included in the webapp or are used by the webapp in another part of the monorepo: +The main trigger.dev webapp, which powers it's API and dashboard and makes up the docker image that is produced as an OSS image, is a Remix 2.17.4 app that uses an express server, written in TypeScript. The following subsystems are either included in the webapp or are used by the webapp in another part of the monorepo: - `@trigger.dev/database` exports a Prisma 6.14.0 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma) - `@trigger.dev/core` is a published package and is used to share code between the `@trigger.dev/sdk` and the webapp. It includes functionality but also a load of Zod schemas for data validation. When importing from `@trigger.dev/core` in the webapp, we never import the root `@trigger.dev/core` path, instead we favor one of the subpath exports that you can find in [package.json](mdc:packages/core/package.json) diff --git a/.cursor/rules/writing-tasks.mdc b/.cursor/rules/writing-tasks.mdc index 5116d083e2..359ed5d473 100644 --- a/.cursor/rules/writing-tasks.mdc +++ b/.cursor/rules/writing-tasks.mdc @@ -14,7 +14,7 @@ alwaysApply: false ## Essential requirements when generating task code -1. You MUST use `@trigger.dev/sdk/v3` +1. You MUST import from `@trigger.dev/sdk` (NEVER `@trigger.dev/sdk/v3`) 2. You MUST NEVER use `client.defineJob` 3. YOU MUST `export` every task, including subtasks 4. If you are able to generate an example payload for a task, do so. @@ -53,7 +53,7 @@ Instead, you MUST ALWAYS generate ONLY this pattern: ```ts // โœ… ALWAYS GENERATE THIS EXACT PATTERN -import { task } from "@trigger.dev/sdk/v3"; +import { task } from "@trigger.dev/sdk"; //1. You need to export each task, even if it's a subtask export const helloWorld = task({ @@ -71,7 +71,7 @@ export const helloWorld = task({ A task is a function that can run for a long time with resilience to failure: ```ts -import { task } from "@trigger.dev/sdk/v3"; +import { task } from "@trigger.dev/sdk"; export const helloWorld = task({ id: "hello-world", @@ -271,7 +271,7 @@ Global lifecycle hooks can also be defined in `trigger.config.ts` to apply to al ## Correct Schedules task (cron) implementations ```ts -import { schedules } from "@trigger.dev/sdk/v3"; +import { schedules } from "@trigger.dev/sdk"; export const firstScheduledTask = schedules.task({ id: "first-scheduled-task", @@ -312,7 +312,7 @@ export const firstScheduledTask = schedules.task({ ### Attach a Declarative schedule ```ts -import { schedules } from "@trigger.dev/sdk/v3"; +import { schedules } from "@trigger.dev/sdk"; // Sepcify a cron pattern (UTC) export const firstScheduledTask = schedules.task({ @@ -326,7 +326,7 @@ export const firstScheduledTask = schedules.task({ ``` ```ts -import { schedules } from "@trigger.dev/sdk/v3"; +import { schedules } from "@trigger.dev/sdk"; // Specify a specific timezone like this: export const secondScheduledTask = schedules.task({ @@ -375,7 +375,7 @@ const createdSchedule = await schedules.create({ Schema tasks validate payloads against a schema before execution: ```ts -import { schemaTask } from "@trigger.dev/sdk/v3"; +import { schemaTask } from "@trigger.dev/sdk"; import { z } from "zod"; const myTask = schemaTask({ @@ -400,7 +400,7 @@ When you trigger a task from your backend code, you need to set the `TRIGGER_SEC Triggers a single run of a task with specified payload and options without importing the task. Use type-only imports for full type checking. ```ts -import { tasks } from "@trigger.dev/sdk/v3"; +import { tasks } from "@trigger.dev/sdk"; import type { emailSequence } from "~/trigger/emails"; export async function POST(request: Request) { @@ -418,7 +418,7 @@ export async function POST(request: Request) { Triggers multiple runs of a single task with different payloads without importing the task. ```ts -import { tasks } from "@trigger.dev/sdk/v3"; +import { tasks } from "@trigger.dev/sdk"; import type { emailSequence } from "~/trigger/emails"; export async function POST(request: Request) { @@ -436,7 +436,7 @@ export async function POST(request: Request) { Triggers multiple runs of different tasks at once, useful when you need to execute multiple tasks simultaneously. ```ts -import { batch } from "@trigger.dev/sdk/v3"; +import { batch } from "@trigger.dev/sdk"; import type { myTask1, myTask2 } from "~/trigger/myTasks"; export async function POST(request: Request) { @@ -621,7 +621,7 @@ const handle = await myTask.trigger( Access metadata inside a run: ```ts -import { task, metadata } from "@trigger.dev/sdk/v3"; +import { task, metadata } from "@trigger.dev/sdk"; export const myTask = task({ id: "my-task", @@ -713,7 +713,7 @@ Trigger.dev Realtime enables subscribing to runs for real-time updates on run st Subscribe to a run after triggering a task: ```ts -import { runs, tasks } from "@trigger.dev/sdk/v3"; +import { runs, tasks } from "@trigger.dev/sdk"; async function myBackend() { const handle = await tasks.trigger("my-task", { some: "data" }); @@ -735,7 +735,7 @@ async function myBackend() { You can infer types of run's payload and output by passing the task type: ```ts -import { runs } from "@trigger.dev/sdk/v3"; +import { runs } from "@trigger.dev/sdk"; import type { myTask } from "./trigger/my-task"; for await (const run of runs.subscribeToRun(handle.id)) { @@ -752,7 +752,7 @@ for await (const run of runs.subscribeToRun(handle.id)) { Stream data in realtime from inside your tasks using the metadata system: ```ts -import { task, metadata } from "@trigger.dev/sdk/v3"; +import { task, metadata } from "@trigger.dev/sdk"; import OpenAI from "openai"; export type STREAMS = { @@ -947,7 +947,7 @@ For most use cases, Realtime hooks are preferred over SWR hooks with polling due For client-side usage, generate a public access token with appropriate scopes: ```ts -import { auth } from "@trigger.dev/sdk/v3"; +import { auth } from "@trigger.dev/sdk"; const publicToken = await auth.createPublicToken({ scopes: { @@ -967,7 +967,7 @@ Idempotency ensures that an operation produces the same result when called multi Provide an `idempotencyKey` when triggering a task to ensure it runs only once with that key: ```ts -import { idempotencyKeys, task } from "@trigger.dev/sdk/v3"; +import { idempotencyKeys, task } from "@trigger.dev/sdk"; export const myTask = task({ id: "my-task", @@ -1058,7 +1058,7 @@ function hash(payload: any): string { ```ts // onFailure executes after all retries are exhausted; use for notifications, logging, or side effects on final failure: -import { task, logger } from "@trigger.dev/sdk/v3"; +import { task, logger } from "@trigger.dev/sdk"; export const loggingExample = task({ id: "logging-example", @@ -1078,7 +1078,7 @@ export const loggingExample = task({ The `trigger.config.ts` file configures your Trigger.dev project, specifying task locations, retry settings, telemetry, and build options. ```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; +import { defineConfig } from "@trigger.dev/sdk"; export default defineConfig({ project: "", @@ -1226,7 +1226,7 @@ await myTask.trigger({ name: "Alice", age: 30 }); Before generating any code, you MUST verify: -1. Are you importing from `@trigger.dev/sdk/v3`? If not, STOP and FIX. +1. Are you importing from `@trigger.dev/sdk` (NOT `@trigger.dev/sdk/v3`)? If not, STOP and FIX. 2. Have you exported every task? If not, STOP and FIX. 3. Have you generated any DEPRECATED code patterns? If yes, STOP and FIX. diff --git a/.cursorignore b/.cursorignore index 8430ce365f..b1033e7a68 100644 --- a/.cursorignore +++ b/.cursorignore @@ -1,7 +1,4 @@ -apps/docker-provider/ -apps/kubernetes-provider/ apps/proxy/ -apps/coordinator/ packages/rsc/ .changeset .zed diff --git a/.env.example b/.env.example index 35c8c976ff..901fbd7b3e 100644 --- a/.env.example +++ b/.env.example @@ -2,12 +2,18 @@ SESSION_SECRET=abcdef1234 MAGIC_LINK_SECRET=abcdef1234 ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal +MANAGED_WORKER_SECRET=abcdef1234 # Must match the supervisor's MANAGED_WORKER_SECRET LOGIN_ORIGIN=http://localhost:3030 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public # This sets the URL used for direct connections to the database and should only be needed in limited circumstances # See: https://www.prisma.io/docs/reference/api-reference/prisma-schema-reference#fields:~:text=the%20shadow%20database.-,directUrl,-No -DIRECT_URL=${DATABASE_URL} +DIRECT_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public +# Dedicated run-ops database (@internal/run-ops-database). Only needed to run prisma commands +# against it or to enable the run-ops split; start it with `docker compose --profile runops up`. +RUN_OPS_DATABASE_URL=postgresql://postgres:postgres@localhost:5434/postgres?schema=public REMIX_APP_PORT=3030 +# Dev-only: stream the webapp's logs over a local telnet/TCP socket (nc localhost 6767). Uncomment to enable. +# WEBAPP_TELNET_LOGS_PORT=6767 APP_ENV=development APP_ORIGIN=http://localhost:3030 ELECTRIC_ORIGIN=http://localhost:3060 @@ -17,6 +23,11 @@ NODE_ENV=development CLICKHOUSE_URL=http://default:password@localhost:8123 RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123 RUN_REPLICATION_ENABLED=1 +# Store task run spans/traces in ClickHouse so the dashboard trace view is +# populated in local dev. The local stack is ClickHouse-backed (see above), so +# leaving this unset falls back to the "postgres" store and dev run traces show +# up empty even though the run itself appears. +EVENT_REPOSITORY_DEFAULT_STORE=clickhouse_v2 # Set this to UTC because Node.js uses the system timezone TZ="UTC" @@ -29,6 +40,52 @@ REDIS_TLS_DISABLED="true" DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel" DEV_OTEL_BATCH_PROCESSING_ENABLED="0" +# Realtime streams v2 (Sessions, chat.agent, large stream backfills) backed +# by S2 (https://s2.dev). The `s2` service in docker/docker-compose.yml runs +# the open-source s2-lite binary and pre-creates a basin named `trigger-local` +# (see docker/config/s2-spec.json). Comment these out to fall back to v1 +# (Redis-only) streams; Sessions and chat.agent then become unavailable. +REALTIME_STREAMS_S2_BASIN=trigger-local +REALTIME_STREAMS_S2_ACCESS_TOKEN=ignored +REALTIME_STREAMS_S2_ENDPOINT=http://localhost:4566/v1 +REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS=true +REALTIME_STREAMS_DEFAULT_VERSION=v2 + +# Running multiple instances side by side (worktrees, branch experiments) +# +# Every host port in docker/docker-compose.yml is `${VAR:-default}` and the +# project name comes from `COMPOSE_PROJECT_NAME`. To stand up a second stack +# alongside the default one, uncomment the block below in this clone's `.env` +# (pick any offset that doesn't clash with anything else running), then update +# the URL/PORT vars further up to match. Default values are commented for +# reference. +# +# --- core (pnpm run docker) --- +# COMPOSE_PROJECT_NAME=triggerdotdev-docker-alt +# CONTAINER_PREFIX=alt- +# POSTGRES_HOST_PORT=15432 # default 5432 +# REDIS_HOST_PORT=16379 # default 6379 +# ELECTRIC_HOST_PORT=13060 # default 3060 +# MINIO_API_HOST_PORT=19005 # default 9005 +# MINIO_CONSOLE_HOST_PORT=19006 # default 9006 +# CLICKHOUSE_HTTP_HOST_PORT=18123 # default 8123 +# CLICKHOUSE_TCP_HOST_PORT=19000 # default 9000 +# S2_HOST_PORT=14566 # default 4566 +# REMIX_APP_PORT=13030 # default 3030 +# --- extras (only needed if you also run `pnpm run docker:full`) --- +# ELECTRIC_SHARD_1_HOST_PORT=13061 # default 3061 +# CH_UI_HOST_PORT=15521 # default 5521 +# TOXIPROXY_PROXY_HOST_PORT=40303 # default 30303 +# TOXIPROXY_API_HOST_PORT=18474 # default 8474 +# NGINX_H2_HOST_PORT=18443 # default 8443 +# OTEL_GRPC_HOST_PORT=14317 # default 4317 +# OTEL_HTTP_HOST_PORT=14318 # default 4318 +# OTEL_PROMETHEUS_HOST_PORT=18889 # default 8889 +# PROMETHEUS_HOST_PORT=19090 # default 9090 +# GRAFANA_HOST_PORT=13001 # default 3001 +# (and update DATABASE_URL / CLICKHOUSE_URL / REDIS_PORT / APP_ORIGIN / +# LOGIN_ORIGIN / ELECTRIC_ORIGIN / REALTIME_STREAMS_S2_ENDPOINT to match) + # When the domain is set to `localhost` the CLI deploy command will only --load the image by default and not --push it DEPLOY_REGISTRY_HOST=localhost:5000 @@ -77,9 +134,28 @@ POSTHOG_PROJECT_KEY= # DEPOT_TOKEN= # DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318" # These are needed for the object store (for handling large payloads/outputs) -# OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com" -# OBJECT_STORE_ACCESS_KEY_ID= -# OBJECT_STORE_SECRET_ACCESS_KEY= +# +# Default provider +# OBJECT_STORE_BASE_URL=http://localhost:9005 +# OBJECT_STORE_BUCKET=packets +# OBJECT_STORE_ACCESS_KEY_ID=minioadmin +# OBJECT_STORE_SECRET_ACCESS_KEY=minioadmin +# OBJECT_STORE_REGION=us-east-1 +# OBJECT_STORE_SERVICE=s3 +# +# OBJECT_STORE_DEFAULT_PROTOCOL=s3 # Only specify this if you're going to migrate object storage and set protocol values below +# Named providers (protocol-prefixed data) - optional for multi-provider support +# OBJECT_STORE_S3_BASE_URL=https://s3.amazonaws.com +# OBJECT_STORE_S3_ACCESS_KEY_ID= +# OBJECT_STORE_S3_SECRET_ACCESS_KEY= +# OBJECT_STORE_S3_REGION=us-east-1 +# OBJECT_STORE_S3_SERVICE=s3 +# +# OBJECT_STORE_R2_BASE_URL=https://{bucket}.{accountId}.r2.cloudflarestorage.com +# OBJECT_STORE_R2_ACCESS_KEY_ID= +# OBJECT_STORE_R2_SECRET_ACCESS_KEY= +# OBJECT_STORE_R2_REGION=auto +# OBJECT_STORE_R2_SERVICE=s3 # CHECKPOINT_THRESHOLD_IN_MS=10000 # These control the server-side internal telemetry @@ -87,8 +163,8 @@ POSTHOG_PROJECT_KEY= # INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1 # INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0 -# Enable local observability stack (requires `pnpm run docker` to start otel-collector) +# Enable local observability stack (requires `pnpm run docker:full` to bring up otel-collector + prometheus + grafana) # Uncomment these to send metrics to the local Prometheus via OTEL Collector: # INTERNAL_OTEL_METRIC_EXPORTER_ENABLED=1 # INTERNAL_OTEL_METRIC_EXPORTER_URL=http://localhost:4318/v1/metrics -# INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS=15000 \ No newline at end of file +# INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS=15000 diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index 827344f1f9..0000000000 --- a/.eslintignore +++ /dev/null @@ -1,4 +0,0 @@ -*/**.js -*/**.d.ts -packages/*/dist -packages/*/lib \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..9af4260211 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Generated, not hand-written: collapsed in diffs and excluded from language stats. +internal-packages/dashboard-agent-db/drizzle/meta/*.json linguist-generated=true +internal-packages/dashboard-agent-db/drizzle/meta/** linguist-generated=true +**/__snapshots__/*.snap linguist-generated=true +pnpm-lock.yaml linguist-generated=true diff --git a/.github/ISSUE_TEMPLATE/vouch-request.yml b/.github/ISSUE_TEMPLATE/vouch-request.yml new file mode 100644 index 0000000000..9ffe04a898 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/vouch-request.yml @@ -0,0 +1,28 @@ +name: Vouch Request +description: Request to be vouched as a contributor +labels: ["vouch-request"] +body: + - type: markdown + attributes: + value: | + ## Vouch Request + + We use [vouch](https://github.com/mitchellh/vouch) to manage contributor trust. PRs from unvouched users are automatically closed. + + To get vouched, fill out this form. A maintainer will review your request and vouch for you by commenting on this issue. + - type: textarea + id: context + attributes: + label: Why do you want to contribute? + description: Tell us a bit about yourself and what you'd like to work on. + placeholder: "I'd like to fix a bug I found in..." + validations: + required: true + - type: textarea + id: prior-work + attributes: + label: Prior contributions or relevant experience + description: Links to previous open source work, relevant projects, or anything that helps us understand your background. + placeholder: "https://github.com/..." + validations: + required: false diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td new file mode 100644 index 0000000000..612a028bb9 --- /dev/null +++ b/.github/VOUCHED.td @@ -0,0 +1,32 @@ +# Vouched contributors for Trigger.dev +# See: https://github.com/mitchellh/vouch +# +# Org members +0ski +D-K-P +ericallam +matt-aitken +mpcgrid +myftija +nicktrn +samejr +isshaddad +# Bots +devin-ai-integration[bot] +dependabot[bot] +# Outside contributors +gautamsi +capaj +chengzp +bharathkumar39293 +bhekanik +jrossi +ThullyoCunha +ConProgramming +saasjesus +brentshulman-silkline +Leafgard +Rohan170603 +NERLOE +Jakub-Vacek +gtremper \ No newline at end of file diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..15a5d6ff00 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - warp-ubuntu-* + - warp-macos-* + - warp-windows-* diff --git a/.github/actions/get-image-tag/action.yml b/.github/actions/get-image-tag/action.yml index e064623046..7f1505a0c1 100644 --- a/.github/actions/get-image-tag/action.yml +++ b/.github/actions/get-image-tag/action.yml @@ -23,35 +23,37 @@ runs: id: get_tag shell: bash run: | - if [[ -n "${{ inputs.tag }}" ]]; then - tag="${{ inputs.tag }}" - elif [[ "${{ github.ref_type }}" == "tag" ]]; then - if [[ "${{ github.ref_name }}" == infra-*-* ]]; then - env=$(echo ${{ github.ref_name }} | cut -d- -f2) - sha=$(echo ${{ github.sha }} | head -c7) + if [[ -n "${INPUTS_TAG}" ]]; then + tag="${INPUTS_TAG}" + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" == infra-*-* ]]; then + env=$(echo ${GITHUB_REF_NAME} | cut -d- -f2) + sha=$(echo "${GITHUB_SHA}" | head -c7) ts=$(date +%s) tag=${env}-${sha}-${ts} - elif [[ "${{ github.ref_name }}" == re2-*-* ]]; then - env=$(echo ${{ github.ref_name }} | cut -d- -f2) - sha=$(echo ${{ github.sha }} | head -c7) + elif [[ "${GITHUB_REF_NAME}" == re2-*-* ]]; then + env=$(echo ${GITHUB_REF_NAME} | cut -d- -f2) + sha=$(echo "${GITHUB_SHA}" | head -c7) ts=$(date +%s) tag=${env}-${sha}-${ts} - elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then + elif [[ "${GITHUB_REF_NAME}" == v.docker.* ]]; then version="${GITHUB_REF_NAME#v.docker.}" tag="v${version}" - elif [[ "${{ github.ref_name }}" == build-* ]]; then + elif [[ "${GITHUB_REF_NAME}" == build-* ]]; then tag="${GITHUB_REF_NAME#build-}" else - echo "Invalid git tag: ${{ github.ref_name }}" + echo "Invalid git tag: ${GITHUB_REF_NAME}" exit 1 fi - elif [[ "${{ github.ref_name }}" == "main" ]]; then + elif [[ "${GITHUB_REF_NAME}" == "main" ]]; then tag="main" else - echo "Invalid git ref: ${{ github.ref }}" + echo "Invalid git ref: ${GITHUB_REF}" exit 1 fi echo "tag=${tag}" >> "$GITHUB_OUTPUT" + env: + INPUTS_TAG: ${{ inputs.tag }} - name: ๐Ÿ” Check for validity id: check_validity diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..7bb64f3674 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + github-actions: + patterns: + - "*" diff --git a/.github/labeler.yml b/.github/labeler.yml index 279bab91a7..6760c37bcd 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -9,4 +9,4 @@ - any: ["**/*.md"] "๐Ÿ“Œ area: ci": - - any: [".github/**/*"] \ No newline at end of file + - any: [".github/**/*"] diff --git a/.github/workflows/base-images.yml b/.github/workflows/base-images.yml new file mode 100644 index 0000000000..2c23246de3 --- /dev/null +++ b/.github/workflows/base-images.yml @@ -0,0 +1,272 @@ +name: ๐Ÿณ Deploy base images + +# Publishes the deploy base images (see base-images/README.md) to Docker Hub. +# Tags are mutable and rebuilt in place; the CLI pins digests, so consumers +# only move when a release bumps its pins. + +on: + workflow_dispatch: + inputs: + debian_snapshot: + description: "Debian snapshot timestamp (YYYYMMDDTHHMMSSZ). Defaults to yesterday 00:00 UTC." + required: false + type: string + push: + branches: [main] + paths: + - "base-images/**" + - ".github/workflows/base-images.yml" + pull_request: + paths: + - "base-images/**" + - ".github/workflows/base-images.yml" + +concurrency: + group: base-images-${{ github.ref }} + cancel-in-progress: false + +permissions: {} + +jobs: + setup: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + images: ${{ steps.config.outputs.images }} + packages: ${{ steps.config.outputs.packages }} + build_packages: ${{ steps.config.outputs.build_packages }} + suite: ${{ steps.config.outputs.suite }} + snapshot: ${{ steps.config.outputs.snapshot }} + publish_id: ${{ steps.config.outputs.publish_id }} + source_date_epoch: ${{ steps.config.outputs.source_date_epoch }} + push: ${{ steps.config.outputs.push }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Read image matrix and resolve snapshot + id: config + env: + SNAPSHOT_INPUT: ${{ inputs.debian_snapshot }} + EVENT_NAME: ${{ github.event_name }} + REF: ${{ github.ref }} + SHA: ${{ github.sha }} + run: | + PACKAGES="$(jq -er '.packages' base-images/images.json)" + BUILD_PACKAGES="$(jq -er '.buildPackages' base-images/images.json)" + SUITE="$(jq -er '.suite' base-images/images.json)" + + # Values land in build args and shell lines; keep them boring. + # NUL-delimited whole-record match so multi-line values can't sneak through + printf '%s\0' "$PACKAGES" | grep -zqxE '[a-z0-9][a-z0-9 .+:=~-]*' || { echo "invalid packages value"; exit 1; } + printf '%s\0' "$BUILD_PACKAGES" | grep -zqxE '[a-z0-9][a-z0-9 .+:=~-]*' || { echo "invalid buildPackages value"; exit 1; } + printf '%s\0' "$SUITE" | grep -zqxE '[a-z]+' || { echo "invalid suite value"; exit 1; } + jq -e '.images | length > 0 and all((.repo | test("^[a-z0-9-]+$")) and (.tag | test("^[a-z0-9.-]+$")) and (.base | test("^[a-zA-Z0-9./:@-]+$")))' base-images/images.json > /dev/null \ + || { echo "invalid images entries"; exit 1; } + + SNAPSHOT="$SNAPSHOT_INPUT" + if [ -z "$SNAPSHOT" ]; then + SNAPSHOT="$(date -u -d yesterday +%Y%m%dT000000Z)" + fi + printf '%s\0' "$SNAPSHOT" | grep -zqxE '[0-9]{8}T[0-9]{6}Z' || { echo "invalid debian_snapshot: $SNAPSHOT"; exit 1; } + + # Snapshot-derived timestamps: reproducible, with a real created date + EPOCH="$(date -u -d "${SNAPSHOT:0:4}-${SNAPSHOT:4:2}-${SNAPSHOT:6:2} ${SNAPSHOT:9:2}:${SNAPSHOT:11:2}:${SNAPSHOT:13:2}Z" +%s)" + # Future snapshots resolve to "latest" and break mtime normalization + [ "$EPOCH" -le "$(date -u +%s)" ] || { echo "debian_snapshot is in the future: $SNAPSHOT"; exit 1; } + + # Pull requests and branch dispatches build without pushing + if [ "$EVENT_NAME" = "pull_request" ] || [ "$REF" != "refs/heads/main" ]; then + PUSH=false + else + PUSH=true + fi + + { + echo "images=$(jq -c '.images' base-images/images.json)" + echo "packages=$PACKAGES" + echo "build_packages=$BUILD_PACKAGES" + echo "suite=$SUITE" + echo "snapshot=$SNAPSHOT" + echo "publish_id=${SNAPSHOT:0:8}-${SNAPSHOT:9:6}-${SHA:0:7}" + echo "source_date_epoch=$EPOCH" + echo "push=$PUSH" + } >> "$GITHUB_OUTPUT" + + publish: + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + image: ${{ fromJSON(needs.setup.outputs.images) }} + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_BUILD_SUMMARY: "false" + DOCKER_BUILD_RECORD_UPLOAD: "false" + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + # Before any pull so rate limits are authenticated; fork PRs skip (no secrets) + - name: ๐Ÿณ Login to Docker Hub + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: ๐Ÿณ Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + with: + image: docker.io/tonistiigi/binfmt:latest@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 + + - name: ๐Ÿณ Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Build both targets before pushing either so the tag pair can't skew + - name: ๐Ÿณ Build both targets (no push) + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: base-images + file: base-images/Dockerfile + target: build + platforms: linux/amd64,linux/arm64 + provenance: false + outputs: type=image,push=false,rewrite-timestamp=true + tags: triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}-build + build-args: | + BASE_IMAGE=${{ matrix.image.base }} + DEBIAN_SNAPSHOT=${{ needs.setup.outputs.snapshot }} + DEBIAN_SUITE=${{ needs.setup.outputs.suite }} + PACKAGES=${{ needs.setup.outputs.packages }} + BUILD_PACKAGES=${{ needs.setup.outputs.build_packages }} + SOURCE_DATE_EPOCH=${{ needs.setup.outputs.source_date_epoch }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + dev.trigger.debian-snapshot=${{ needs.setup.outputs.snapshot }} + + - name: ๐Ÿณ Push runtime image + id: build_runtime + if: needs.setup.outputs.push == 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: base-images + file: base-images/Dockerfile + target: runtime + platforms: linux/amd64,linux/arm64 + provenance: false + outputs: type=image,push=true,rewrite-timestamp=true + # The dated tag is immutable and keeps every published digest + # tag-referenced forever; shipped CLI releases pin these digests + tags: | + triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }} + triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}-${{ needs.setup.outputs.publish_id }} + build-args: | + BASE_IMAGE=${{ matrix.image.base }} + DEBIAN_SNAPSHOT=${{ needs.setup.outputs.snapshot }} + DEBIAN_SUITE=${{ needs.setup.outputs.suite }} + PACKAGES=${{ needs.setup.outputs.packages }} + SOURCE_DATE_EPOCH=${{ needs.setup.outputs.source_date_epoch }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + dev.trigger.debian-snapshot=${{ needs.setup.outputs.snapshot }} + + - name: ๐Ÿณ Push build-variant image + id: build_toolchain + if: needs.setup.outputs.push == 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: base-images + file: base-images/Dockerfile + target: build + platforms: linux/amd64,linux/arm64 + provenance: false + outputs: type=image,push=true,rewrite-timestamp=true + tags: | + triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}-build + triggerdotdev/${{ matrix.image.repo }}:${{ matrix.image.tag }}-build-${{ needs.setup.outputs.publish_id }} + build-args: | + BASE_IMAGE=${{ matrix.image.base }} + DEBIAN_SNAPSHOT=${{ needs.setup.outputs.snapshot }} + DEBIAN_SUITE=${{ needs.setup.outputs.suite }} + PACKAGES=${{ needs.setup.outputs.packages }} + BUILD_PACKAGES=${{ needs.setup.outputs.build_packages }} + SOURCE_DATE_EPOCH=${{ needs.setup.outputs.source_date_epoch }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + dev.trigger.debian-snapshot=${{ needs.setup.outputs.snapshot }} + + # An auto-created private repo would publish green while customer pulls fail + - name: ๐Ÿ”Ž Verify anonymous pullability + if: needs.setup.outputs.push == 'true' + env: + IMAGE_REPO: ${{ matrix.image.repo }} + RUNTIME_DIGEST: ${{ steps.build_runtime.outputs.digest }} + BUILD_DIGEST: ${{ steps.build_toolchain.outputs.digest }} + run: | + for digest in "$RUNTIME_DIGEST" "$BUILD_DIGEST"; do + TOKEN="$(curl -fsS --connect-timeout 10 --max-time 60 "https://auth.docker.io/token?service=registry.docker.io&scope=repository:triggerdotdev/$IMAGE_REPO:pull" | jq -r .token)" + curl -fsS --connect-timeout 10 --max-time 60 -o /dev/null -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json" "https://registry-1.docker.io/v2/triggerdotdev/$IMAGE_REPO/manifests/$digest" || { echo "triggerdotdev/$IMAGE_REPO@$digest is not anonymously pullable; is the repo private?"; exit 1; } + done + + # Builds are reproducible, so re-running a red publish re-pushes the + # same digests and re-attests them + - name: ๐Ÿ” Attest runtime image provenance + if: needs.setup.outputs.push == 'true' + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-name: index.docker.io/triggerdotdev/${{ matrix.image.repo }} + subject-digest: ${{ steps.build_runtime.outputs.digest }} + push-to-registry: false + + - name: ๐Ÿ” Attest build-variant image provenance + if: needs.setup.outputs.push == 'true' + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-name: index.docker.io/triggerdotdev/${{ matrix.image.repo }} + subject-digest: ${{ steps.build_toolchain.outputs.digest }} + push-to-registry: false + + - name: ๐Ÿ“‹ Record digests + if: needs.setup.outputs.push == 'true' + env: + IMAGE_REPO: ${{ matrix.image.repo }} + IMAGE_TAG: ${{ matrix.image.tag }} + RUNTIME_DIGEST: ${{ steps.build_runtime.outputs.digest }} + BUILD_DIGEST: ${{ steps.build_toolchain.outputs.digest }} + SNAPSHOT: ${{ needs.setup.outputs.snapshot }} + run: | + { + echo "### triggerdotdev/$IMAGE_REPO:$IMAGE_TAG" + echo '```' + echo "runtime: $RUNTIME_DIGEST" + echo "build: $BUILD_DIGEST" + echo "debian snapshot: $SNAPSHOT" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + results: + needs: [publish] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Fail if any image build failed + env: + RESULT: ${{ needs.publish.result }} + run: | + [ "$RESULT" = "success" ] || { echo "one or more image builds failed: $RESULT"; exit 1; } diff --git a/.github/workflows/changesets-pr.yml b/.github/workflows/changesets-pr.yml index ec21972361..851c5341db 100644 --- a/.github/workflows/changesets-pr.yml +++ b/.github/workflows/changesets-pr.yml @@ -7,6 +7,7 @@ on: paths: - "packages/**" - ".changeset/**" + - ".server-changes/**" - "package.json" - "pnpm-lock.yaml" @@ -17,24 +18,25 @@ concurrency: jobs: release-pr: name: Create Release PR - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x permissions: contents: write pull-requests: write + checks: write if: github.repository == 'triggerdotdev/trigger.dev' steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 # zizmor: ignore[artipacked] changesets/action pushes the release branch; no artifact upload here so no leak path with: fetch-depth: 0 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - name: Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" - name: Install dependencies @@ -42,7 +44,7 @@ jobs: - name: Create release PR id: changesets - uses: changesets/action@v1 + uses: changesets/action@a45c4d594aa4e2c509dc14a9f2b3b67ba3780d0d # v1.9.0 with: version: pnpm run changeset:version commit: "chore: release" @@ -50,7 +52,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Update PR title with version + - name: Update PR title and enhance body if: steps.changesets.outputs.published != 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -61,42 +63,37 @@ jobs: # we arbitrarily reference the version of the cli package here; it is the same for all package releases VERSION=$(git show origin/changeset-release/main:packages/cli-v3/package.json | jq -r '.version') gh pr edit "$PR_NUMBER" --title "chore: release v$VERSION" - fi - - update-lockfile: - name: Update lockfile on release PR - runs-on: ubuntu-latest - needs: release-pr - permissions: - contents: write - steps: - - name: Checkout release branch - uses: actions/checkout@v4 - with: - ref: changeset-release/main - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 10.23.0 - - name: Setup node - uses: buildjet/setup-node@v4 - with: - node-version: 20.19.0 - - - name: Install and update lockfile - run: pnpm install --no-frozen-lockfile + # Enhance the PR body with a clean, deduplicated summary + RAW_BODY=$(gh pr view "$PR_NUMBER" --json body --jq '.body') + ENHANCED_BODY=$(CHANGESET_PR_BODY="$RAW_BODY" node scripts/enhance-release-pr.mjs "$VERSION") + if [ -n "$ENHANCED_BODY" ]; then + gh api repos/triggerdotdev/trigger.dev/pulls/"$PR_NUMBER" \ + -X PATCH \ + -f body="$ENHANCED_BODY" + fi + fi - - name: Commit and push lockfile + # The changesets bot authors release PRs with GITHUB_TOKEN, which by GitHub + # design cannot trigger downstream workflows. That leaves the required + # "All PR Checks" status permanently Expected and the PR unmergeable. + # The release PR only bumps package.json + lockfile + CHANGELOGs from + # changesets already on main, so we self-report the required check as + # success. If a human ever pushes to changeset-release/main, the real + # pr_checks.yml fires and its result overwrites this one (last write wins + # for the same context on the same SHA). + - name: Self-report "All PR Checks" success on release PR + if: steps.changesets.outputs.published != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - set -e - if git diff --quiet pnpm-lock.yaml; then - echo "No lockfile changes" - else - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add pnpm-lock.yaml - git commit -m "chore: update lockfile for release" - git push origin changeset-release/main - fi + PR_NUMBER=$(gh pr list --head changeset-release/main --json number --jq '.[0].number') + if [ -z "$PR_NUMBER" ]; then exit 0; fi + HEAD_SHA=$(gh pr view "$PR_NUMBER" --json headRefOid --jq '.headRefOid') + gh api -X POST repos/${{ github.repository }}/check-runs \ + -f name="All PR Checks" \ + -f head_sha="$HEAD_SHA" \ + -f status=completed \ + -f conclusion=success \ + -f 'output[title]=Auto-pass for changeset release PR' \ + -f 'output[summary]=Required check auto-satisfied for changeset-release/main PRs. Full CI ran on the underlying commits before they landed on main.' diff --git a/.github/workflows/check-review-md.yml b/.github/workflows/check-review-md.yml new file mode 100644 index 0000000000..79b9d7eacd --- /dev/null +++ b/.github/workflows/check-review-md.yml @@ -0,0 +1,95 @@ +name: ๐Ÿ”Ž REVIEW.md Drift Audit + +on: + pull_request: + types: [opened, ready_for_review, synchronize] + paths-ignore: + - "docs/**" + - ".changeset/**" + - ".server-changes/**" + +concurrency: + group: review-md-drift-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + audit: + # Set the ENABLE_CLAUDE_CODE repository variable to 'false' to turn off Claude + # jobs; leave it unset (the default) to keep them enabled. + if: >- + vars.ENABLE_CLAUDE_CODE != 'false' && + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + pull-requests: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@428971d2ecd6e3a7cb0ee0da2a3a8b33fdb3678d # v1.0.157 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + use_sticky_comment: true + allowed_bots: "devin-ai-integration[bot],claude[bot]" + + claude_args: | + --max-turns 30 + --allowedTools "Read,Glob,Grep,Bash(git diff:*)" + + prompt: | + You are auditing this PR for drift against `.claude/REVIEW.md`. + + ## Context + + `.claude/REVIEW.md` is the repo's source of truth for what AI / agent code reviewers should treat as critical findings (rolling-deploy safety, hot-table indexes, recovery-path queries, testcontainers usage, Lua versioning, etc.). It is consumed by review agents to calibrate severity. If REVIEW.md goes stale, every future agent review degrades. + + ## Strategy โ€” read this first + + You have a hard turn budget. Spend it on signal, not coverage. The audit is allowed to miss things; it is NOT allowed to time out. + + 1. Read `.claude/REVIEW.md` once, in full. + 2. Run `git diff origin/main...HEAD --name-only` to get the list of changed files. Do NOT read the diff content yet. + 3. Scan the file-list for relevance to REVIEW.md scope. Relevance signals: changes to Prisma schema, Redis / queue / Lua code, hot tables, recovery / restart loops, new packages, deletions of paths REVIEW.md cites. Skim everything else. + 4. Open at most **5 files** total โ€” only the ones most likely to surface a real signal. If nothing in the file-list looks relevant to any REVIEW.md rule, do NOT read any files; go straight to the verdict. + 5. Form a verdict and stop. Do not exhaust the turn budget exploring. + + Large PRs (>50 files changed) are a strong signal to be MORE selective, not more thorough. Pick 3-5 files at most. + + ## What to look for + + - **Stale references** โ€” does any REVIEW.md rule cite a file, directory, function, table, Prisma model, or package name that has been removed or renamed in this PR (or is already gone from `main`)? + - **Contradictions** โ€” does code in this PR clearly violate a current REVIEW.md rule? (Don't re-review the PR. Only flag if REVIEW.md and the PR plainly disagree.) + - **Missing rules** โ€” does this PR introduce a new pattern future reviewers should know about? Examples: a new hot table, a new Lua-script versioning convention, a new safety wrapper, a new "must always check" invariant. + - **Obsolete rules** โ€” has the repo moved past a constraint REVIEW.md still asserts? (e.g. a deprecated path is gone, a pattern is now linted, V1 code is deleted.) + + ## Response format + + If nothing needs changing: + + โœ… REVIEW.md looks current for this PR. + + Otherwise: + + ๐Ÿ“ **REVIEW.md updates suggested:** + + - **[stale]** `` โ€” + - **[contradiction]** `` โ€” + - **[missing]** under `##
` โ€” + - **[obsolete]** `` โ€” + + ## Rules + + - Maximum 3 suggestions per audit. Pick the highest-signal ones. + - Only flag things that would actually mislead a future reviewer. Style and wording do not count. + - Do NOT review the PR itself. Do NOT propose rules outside REVIEW.md's existing sections. + - Do NOT propose rules for one-off PR specifics that don't generalize to future PRs. + - If REVIEW.md does not exist in the repo, respond with `(skip)` and stop. + - When in doubt between "one more file read" and "finish now" โ€” finish now. diff --git a/.github/workflows/claude-md-audit.yml b/.github/workflows/claude-md-audit.yml new file mode 100644 index 0000000000..d1b1bbdb45 --- /dev/null +++ b/.github/workflows/claude-md-audit.yml @@ -0,0 +1,83 @@ +name: ๐Ÿ“ Agent Instructions Audit + +on: + pull_request: + types: [opened, ready_for_review, synchronize] + paths-ignore: + - "docs/**" + - ".changeset/**" + - ".server-changes/**" + - "**/*.md" + +concurrency: + group: agent-instructions-audit-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + audit: + # Set the ENABLE_CLAUDE_CODE repository variable to 'false' to turn off Claude + # jobs; leave it unset (the default) to keep them enabled. + if: >- + vars.ENABLE_CLAUDE_CODE != 'false' && + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@428971d2ecd6e3a7cb0ee0da2a3a8b33fdb3678d # v1.0.157 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + use_sticky_comment: true + allowed_bots: "devin-ai-integration[bot],claude[bot]" + + claude_args: | + --max-turns 25 + --model claude-opus-4-8 + --allowedTools "Read,Glob,Grep,Bash(git diff:*)" + + prompt: | + You are reviewing a PR to check whether any agent instruction files need updating. + + In this repo: + - Root shared agent guidance lives in `AGENTS.md`. + - Root `CLAUDE.md` is only a Claude Code adapter that imports `AGENTS.md`. + - Subdirectories may still have scoped `CLAUDE.md` files. + - `.claude/rules/` contains additional Claude Code guidance. + + ## Your task + + 1. Run `git diff origin/main...HEAD --name-only` to see which files changed in this PR. + 2. For each changed directory, check the applicable instruction files: root `AGENTS.md`, any `CLAUDE.md` in that directory or a parent directory, and relevant `.claude/rules/` files. + 3. Determine if any instruction file should be updated based on the changes. Consider: + - New files/directories that aren't covered by existing documentation + - Changed architecture or patterns that contradict current agent guidance + - New dependencies, services, or infrastructure that agents should know about + - Renamed or moved files that are referenced in an instruction file + - Changes to build commands, test patterns, or development workflows + + ## Response format + + If NO updates are needed, respond with exactly: + โœ… Agent instruction files look current for this PR. + + If updates ARE needed, respond with a short list: + ๐Ÿ“ **Agent instruction updates suggested:** + - `AGENTS.md`: [what should be added/changed] + - `path/to/CLAUDE.md`: [what should be added/changed] + - `.claude/rules/file.md`: [what should be added/changed] + + Keep suggestions specific and brief. Only flag things that would actually mislead agents in future sessions. + Do NOT suggest updates for trivial changes (bug fixes, small refactors within existing patterns). + Do NOT suggest creating new CLAUDE.md files - only updates to existing instruction files. diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000000..7a278672a2 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,76 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + # Set the ENABLE_CLAUDE_CODE repository variable to 'false' to turn off Claude + # jobs; leave it unset (the default) to keep them enabled. + if: | + vars.ENABLE_CLAUDE_CODE != 'false' && + ( + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + ) + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: write + pull-requests: write + issues: write + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@428971d2ecd6e3a7cb0ee0da2a3a8b33fdb3678d # v1.0.157 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + claude_args: | + --model claude-opus-4-5-20251101 + --allowedTools "Bash(pnpm:*),Bash(turbo:*),Bash(git:*),Bash(gh:*),Bash(npx:*),Bash(docker:*),Edit,MultiEdit,Read,Write,Glob,Grep,LS,Task" + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr:*)' diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000000..48575827b3 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,38 @@ +name: "๐ŸŽจ Format & Lint" + +on: + workflow_call: + +permissions: + contents: read + +jobs: + code-quality: + runs-on: warp-ubuntu-latest-x64-2x + + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ’… Check formatting + run: pnpm exec oxfmt --check . + + - name: ๐Ÿ”Ž Lint + run: pnpm exec oxlint . diff --git a/.github/workflows/dashboard-agent-deploy.yml b/.github/workflows/dashboard-agent-deploy.yml new file mode 100644 index 0000000000..e5ea856069 --- /dev/null +++ b/.github/workflows/dashboard-agent-deploy.yml @@ -0,0 +1,72 @@ +name: "๐Ÿค– Deploy dashboard agent" + +# Deploys the @internal/dashboard-agent chat.agent to its Trigger.dev project +# with --skip-promotion, so a deploy never becomes "current" on its own. The +# consuming app cuts over by pinning DASHBOARD_AGENT_VERSION to the new version. +# Runs a leg per environment (staging + prod), each gated by its own environment; +# a push to main that touches the agent or its store triggers both. Version +# numbers are per-environment, so pin each environment to its own leg's version. + +on: + push: + branches: [main] + paths: + - "internal-packages/dashboard-agent/**" + - "internal-packages/dashboard-agent-db/**" + workflow_dispatch: + +permissions: {} + +jobs: + deploy: + name: Deploy dashboard agent (${{ matrix.environment }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + # One environment at a time: parallel deploys of the same project race. + max-parallel: 1 + matrix: + environment: [staging, prod] + # Per-environment reviewer gate + source of the scoped deploy PAT. + environment: dashboard-agent-${{ matrix.environment }} + concurrency: + group: dashboard-agent-deploy-${{ matrix.environment }} + cancel-in-progress: false + permissions: + contents: read + env: + TRIGGER_API_URL: https://api.trigger.dev + TRIGGER_DASHBOARD_AGENT_PROJECT_REF: ${{ vars.TRIGGER_DASHBOARD_AGENT_PROJECT_REF }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: Install + build the CLI and the agent's deps + run: | + set -euo pipefail + pnpm install --frozen-lockfile + # Prisma client is needed because the build closure pulls in @trigger.dev/database. + pnpm run generate + # Config-time imports the agent's trigger.config.ts needs: defineConfig (sdk), aptGet (build). + pnpm run build --filter trigger.dev --filter @trigger.dev/build --filter @trigger.dev/sdk + + - name: Deploy (--skip-promotion) + working-directory: internal-packages/dashboard-agent + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_DASHBOARD_AGENT_DEPLOY_TOKEN }} + # Invoke the built CLI directly (what the workspace .bin/trigger wrapper does), + # so a not-yet-linked bin after a fresh install can't break the deploy. + run: node ../../packages/cli-v3/dist/esm/index.js deploy --skip-promotion --env ${{ matrix.environment }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bef575c353..460a21c202 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,16 +20,18 @@ permissions: jobs: check-broken-links: - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x defaults: run: working-directory: ./docs steps: - name: ๐Ÿ“ฅ Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: ๐Ÿ“ฆ Cache npm - uses: actions/cache@v4 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | ~/.npm diff --git a/.github/workflows/e2e-webapp-auth-full.yml b/.github/workflows/e2e-webapp-auth-full.yml new file mode 100644 index 0000000000..e88429a344 --- /dev/null +++ b/.github/workflows/e2e-webapp-auth-full.yml @@ -0,0 +1,120 @@ +name: "๐Ÿ›ก๏ธ E2E Tests: Webapp Auth (full)" + +# Comprehensive RBAC auth test suite โ€” see TRI-8731. Runs separately from +# the smoke e2e-webapp.yml because it covers every route family with a +# pass/fail matrix and would otherwise dominate per-PR CI time. +# +# Triggered: +# - Manually via workflow_dispatch. +# - Nightly via schedule. +# - On pull requests touching auth-relevant files only (paths filter). + +permissions: + contents: read + +on: + workflow_dispatch: + schedule: + - cron: "0 4 * * *" # 04:00 UTC daily + pull_request: + paths: + - "apps/webapp/app/services/routeBuilders/**" + - "apps/webapp/app/services/rbac.server.ts" + - "apps/webapp/app/services/apiAuth.server.ts" + - "apps/webapp/app/services/personalAccessToken.server.ts" + - "apps/webapp/app/services/sessionStorage.server.ts" + - "apps/webapp/app/routes/api.v*.**" + - "apps/webapp/app/routes/realtime.v*.**" + - "apps/webapp/test/**/*.e2e.full.test.ts" + - "apps/webapp/test/setup/global-e2e-full-setup.ts" + - "apps/webapp/test/helpers/sharedTestServer.ts" + - "apps/webapp/test/helpers/seedTestSession.ts" + - "apps/webapp/vitest.e2e.full.config.ts" + - "internal-packages/rbac/**" + - "packages/plugins/**" + - ".github/workflows/e2e-webapp-auth-full.yml" + +jobs: + e2eAuthFull: + name: "๐Ÿ›ก๏ธ E2E Auth Tests (full)" + runs-on: warp-ubuntu-latest-x64-8x + timeout-minutes: 30 + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + steps: + - name: ๐Ÿ”ง Disable IPv6 + run: | + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + + - name: ๐Ÿ”ง Configure docker address pool + run: | + CONFIG='{ + "default-address-pools" : [ + { + "base" : "172.17.0.0/12", + "size" : 20 + }, + { + "base" : "192.168.0.0/16", + "size" : 24 + } + ] + }' + mkdir -p /etc/docker + echo "$CONFIG" | sudo tee /etc/docker/daemon.json + + - name: ๐Ÿ”ง Restart docker daemon + run: sudo systemctl restart docker + + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + # Don't leave the GITHUB_TOKEN in .git/config โ€” this job + # doesn't need to push and the persisted creds would be + # readable from any subsequent step (zizmor/artipacked). + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿณ Login to DockerHub + if: ${{ env.DOCKERHUB_USERNAME }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: ๐Ÿณ Skipping DockerHub login (no secrets available) + if: ${{ !env.DOCKERHUB_USERNAME }} + run: echo "DockerHub login skipped because secrets are not available." + + - name: ๐Ÿณ Pre-pull testcontainer images + if: ${{ env.DOCKERHUB_USERNAME }} + run: | + docker pull postgres:14 + docker pull redis:7.2 + docker pull testcontainers/ryuk:0.11.0 + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ—๏ธ Build Webapp + run: pnpm run build --filter webapp + + - name: ๐Ÿ›ก๏ธ Run Webapp Full Auth E2E Tests + run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.full.config.ts --reporter=default + env: + WEBAPP_TEST_VERBOSE: "1" diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml new file mode 100644 index 0000000000..1c1525f31b --- /dev/null +++ b/.github/workflows/e2e-webapp.yml @@ -0,0 +1,102 @@ +name: "๐Ÿงช E2E Tests: Webapp" + +permissions: + contents: read + +on: + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + +jobs: + e2eTests: + name: "๐Ÿงช E2E Tests: Webapp" + runs-on: warp-ubuntu-latest-x64-16x + timeout-minutes: 30 + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + steps: + - name: ๐Ÿ”ง Disable IPv6 + run: | + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + + - name: ๐Ÿ”ง Configure docker address pool + run: | + CONFIG='{ + "default-address-pools" : [ + { + "base" : "172.17.0.0/12", + "size" : 20 + }, + { + "base" : "192.168.0.0/16", + "size" : 24 + } + ] + }' + mkdir -p /etc/docker + echo "$CONFIG" | sudo tee /etc/docker/daemon.json + + - name: ๐Ÿ”ง Restart docker daemon + run: sudo systemctl restart docker + + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + # ..to avoid rate limits when pulling images + - name: ๐Ÿณ Login to DockerHub + if: ${{ env.DOCKERHUB_USERNAME }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: ๐Ÿณ Skipping DockerHub login (no secrets available) + if: ${{ !env.DOCKERHUB_USERNAME }} + run: echo "DockerHub login skipped because secrets are not available." + + - name: ๐Ÿณ Pre-pull testcontainer images + if: ${{ env.DOCKERHUB_USERNAME }} + run: | + echo "Pre-pulling Docker images with authenticated session..." + docker pull postgres:14 + docker pull redis:7.2 + docker pull testcontainers/ryuk:0.11.0 + docker pull ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d + docker pull minio/minio:latest + echo "Image pre-pull complete" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ—๏ธ Build Webapp + run: pnpm run build --filter webapp + + - name: ๐ŸŽญ Install Playwright Chromium + run: cd apps/webapp && pnpm exec playwright install chromium + + - name: ๐Ÿงช Run Webapp E2E Tests + run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default + env: + WEBAPP_TEST_VERBOSE: "1" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 97170c2225..4584ea69c1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,27 +20,30 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [warp-ubuntu-latest-x64-4x, warp-windows-latest-x64-8x] package-manager: ["npm", "pnpm"] steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile --filter trigger.dev... + # Prisma clients generate concurrently and share one engine binary in the + # pnpm store; the generate script retries the transient Windows EPERM race. - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate @@ -48,7 +51,7 @@ jobs: run: pnpm run build --filter trigger.dev^... - name: ๐Ÿ”ง Build worker template files - run: pnpm --filter trigger.dev run build:workers + run: pnpm --filter trigger.dev run --if-present build:workers - name: Enable corepack run: corepack enable diff --git a/.github/workflows/fk-cascade-guard.yml b/.github/workflows/fk-cascade-guard.yml new file mode 100644 index 0000000000..eae272354d --- /dev/null +++ b/.github/workflows/fk-cascade-guard.yml @@ -0,0 +1,35 @@ +name: "๐Ÿ›ก๏ธ FK Cascade Index Guard" + +on: + workflow_call: + +permissions: + contents: read + +jobs: + fk-cascade-guard: + runs-on: warp-ubuntu-latest-x64-16x + + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ›ก๏ธ FK cascade index guard + run: pnpm --filter webapp run guard:fk-cascade-index -- --check diff --git a/.github/workflows/helm-prerelease.yml b/.github/workflows/helm-prerelease.yml new file mode 100644 index 0000000000..7d49868eea --- /dev/null +++ b/.github/workflows/helm-prerelease.yml @@ -0,0 +1,207 @@ +name: ๐Ÿงญ Helm Chart Prerelease + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "hosting/k8s/helm/**" + push: + branches: + - main + paths: + - "hosting/k8s/helm/**" + workflow_dispatch: + inputs: + app_version: + description: "Override appVersion (e.g. 'main', 'v4.4.4'). Leave empty to keep Chart.yaml value." + required: false + type: string + default: "" + +concurrency: + group: helm-prerelease-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + CHART_NAME: trigger + +jobs: + lint-and-test: + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: "3.18.3" + + - name: Build dependencies + run: helm dependency build ./hosting/k8s/helm/ + + - name: Extract dependency charts + run: | + cd ./hosting/k8s/helm/ + for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done + + - name: Lint Helm Chart + run: | + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml + + - name: Render templates + run: | + helm template test-release ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ + --output-dir ./helm-output + + - name: Validate manifests + uses: docker://ghcr.io/yannh/kubeconform:v0.7.0@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c + with: + entrypoint: "/kubeconform" + args: "-summary -output json ./helm-output" + + prerelease: + needs: lint-and-test + # Set the ENABLE_HELM_PRERELEASE repository variable to 'false' to turn off + # publishing the chart to GHCR โ€” e.g. forks/mirrors that lack write_package + # on the owner's charts namespace. Defaults to enabled; the lint-and-test + # job above always runs regardless. + if: | + vars.ENABLE_HELM_PRERELEASE != 'false' && + ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch') + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + packages: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: "3.18.3" + + - name: Build dependencies + run: helm dependency build ./hosting/k8s/helm/ + + - name: Extract dependency charts + run: | + cd ./hosting/k8s/helm/ + for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done + + - name: Log in to Container Registry + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate prerelease version + id: version + run: | + BASE_VERSION=$(grep '^version:' ./hosting/k8s/helm/Chart.yaml | awk '{print $2}') + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + PR_NUMBER=${{ github.event.pull_request.number }} + SHORT_SHA=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) + PRERELEASE_VERSION="${BASE_VERSION}-pr${PR_NUMBER}.${SHORT_SHA}" + elif [[ "${{ github.event_name }}" == "push" ]]; then + SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-7) + PRERELEASE_VERSION="${BASE_VERSION}-main.${SHORT_SHA}" + else + SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-7) + REF_SLUG=$(echo "${GITHUB_REF_NAME}" | tr '/' '-' | tr -cd 'a-zA-Z0-9-') + if [[ -z "$REF_SLUG" ]]; then + REF_SLUG="manual" + fi + PRERELEASE_VERSION="${BASE_VERSION}-${REF_SLUG}.${SHORT_SHA}" + fi + echo "version=$PRERELEASE_VERSION" >> "$GITHUB_OUTPUT" + echo "Prerelease version: $PRERELEASE_VERSION" + + - name: Update Chart.yaml with prerelease version + run: | + sed -i "s/^version:.*/version: ${STEPS_VERSION_OUTPUTS_VERSION}/" ./hosting/k8s/helm/Chart.yaml + env: + STEPS_VERSION_OUTPUTS_VERSION: ${{ steps.version.outputs.version }} + + - name: Override appVersion + if: github.event_name == 'workflow_dispatch' && inputs.app_version != '' + env: + APP_VERSION: ${{ inputs.app_version }} + run: | + yq -i '.appVersion = strenv(APP_VERSION)' ./hosting/k8s/helm/Chart.yaml + + - name: Package Helm Chart + run: | + helm package ./hosting/k8s/helm/ --destination /tmp/ + + - name: Push Helm Chart to GHCR + run: | + VERSION="${STEPS_VERSION_OUTPUTS_VERSION}" + CHART_PACKAGE="/tmp/${{ env.CHART_NAME }}-${VERSION}.tgz" + + # Push to GHCR OCI registry + helm push "$CHART_PACKAGE" "oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts" + env: + STEPS_VERSION_OUTPUTS_VERSION: ${{ steps.version.outputs.version }} + + - name: Write run summary + run: | + { + echo "### ๐Ÿงญ Helm Chart Prerelease Published" + echo "" + echo "**Version:** \`${STEPS_VERSION_OUTPUTS_VERSION}\`" + echo "" + echo "**Install:**" + echo '```bash' + echo "helm upgrade --install trigger \\" + echo " oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts/${{ env.CHART_NAME }} \\" + echo " --version \"${STEPS_VERSION_OUTPUTS_VERSION}\"" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + env: + STEPS_VERSION_OUTPUTS_VERSION: ${{ steps.version.outputs.version }} + + - name: Find existing comment + if: github.event_name == 'pull_request' + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + id: find-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "Helm Chart Prerelease Published" + + - name: Create or update PR comment + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + ### ๐Ÿงญ Helm Chart Prerelease Published + + **Version:** `${{ steps.version.outputs.version }}` + + **Install:** + ```bash + helm upgrade --install trigger \ + oci://ghcr.io/${{ github.repository_owner }}/charts/trigger \ + --version "${{ steps.version.outputs.version }}" + ``` + + > โš ๏ธ This is a prerelease for testing. Do not use in production. + edit-mode: replace diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml new file mode 100644 index 0000000000..2d403166c4 --- /dev/null +++ b/.github/workflows/observability-map.yml @@ -0,0 +1,386 @@ +name: ๐Ÿ—บ๏ธ Observability Map + +on: + # No paths filter, deliberately. GitHub evaluates one per workflow, so a pull request whose diff + # stops matching does not start the workflow at all: the resolved state cannot fire and a comment + # from an earlier push stands for ever showing findings that are no longer in the diff. Verified on + # a throwaway pull request whose only route change was reverted, and the realistic case is worse + # than that empty diff, because a pull request touching a route and other files, whose author + # reverts the route change and keeps the rest, still has a non-empty diff that no longer matches. + # The gating moved into the jobs below instead, where it can read whether a comment exists. + pull_request: + types: [opened, synchronize, reopened] + # The corpus job below is gated to this package's own paths, so a scheduled run is what still + # scans the tree as it drifts. Nightly rather than per route pull request: a new route can make a + # known laundering shape start paying, but that is a property of the tree accumulating, not of any + # one pull request, and it does not need catching within five minutes of the merge. + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +concurrency: + group: observability-map-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # The whole cost of a pull request that touches nothing this workflow watches: a checkout, a paths + # filter and one comment lookup. Everything expensive is gated on this job's outputs, and the + # lookup is here rather than in the report job so that gate can read it and the report job need + # never start. + changes: + name: ๐Ÿ” What moved + # Only the pull request path reads this job's output. On a schedule the action has no base to + # diff, warns that `before` is missing and reports the files in the last commit on main, which + # nothing then consults. Skipping it there keeps the nightly off a job it does not need. + if: github.event_name == 'pull_request' + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + # Reading the pull request's comments, to find one an earlier push left. Read only: the write + # stays on the report job, which is the only job that posts. + pull-requests: read + outputs: + # The corpus job's gate. Narrower than the report's on purpose: what the corpus measures is + # the tool's resistance to laundering, which only an edit to the tool can weaken. + package: ${{ steps.filter.outputs.package }} + # The report job's gate, the union: a route change moves the report as well. + report: ${{ steps.filter.outputs.package == 'true' || steps.filter.outputs.routes == 'true' }} + # The id of a marker comment an earlier push left, empty if there is none, and the one source + # both the render and upsert steps read it from. + comment: ${{ steps.comment.outputs.id }} + # Set only by a lookup that finished cleanly, so anything else, retries exhausted or the step + # dying somewhere unforeseen, reads as "do not touch this pull request's comments". + lookup: ${{ steps.comment.outputs.ok }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: filter + with: + filters: | + package: + - 'internal-packages/observability-map/**' + - '.github/workflows/observability-map.yml' + routes: + - 'apps/webapp/app/routes/**' + + # Looked up here because the report job's gate needs it: with the watched paths unmoved, a + # pull request that already has a comment gets a resolved state rather than being left with + # findings that no longer exist, and one that does not gets no job at all. + # + # On a failure that outlasts the retries this reports nothing, and the report job's gate reads + # that as "post nothing this run". Guessing is worse than silence: this step is the only thing + # that knows which comment to PATCH, so a guess of "no comment exists" POSTs, which either + # adds a second marker comment beside the stale one or says "the findings an earlier push + # reported are gone" on a pull request that never had findings. Worst case now is no comment + # this run, which the next push fixes. + - name: ๐Ÿ” Look for a comment from an earlier push + id: comment + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + found="" + ok="" + for attempt in 1 2 3; do + # Matched by login, not .user.type == "Bot": other bots and apps on the same PR are + # also type Bot, and login is the exact identity this token's own comments carry. + if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.user.login == "github-actions[bot]" and ((.body // "") | startswith("")))][0].id // empty'); then + ok=1 + break + fi + echo "comment lookup attempt ${attempt} failed" >&2 + sleep $((attempt * 5)) + done + + if [ -z "$ok" ]; then + echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 + exit 0 + fi + + # --paginate runs the jq once per page, so a marker comment on more than one page yields + # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under + # continue-on-error. The oldest wins: it is the one the upsert has been updating. + count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) + if [ "$count" -gt 1 ]; then + echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 + fi + { + echo "id=$(printf '%s\n' "$found" | awk 'NF { print $1; exit }')" + echo "ok=ok" + } >> "$GITHUB_OUTPUT" + + # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, + # asserting the score does not rise. 53 entries, a couple of minutes of a runner and a good deal + # longer on a laptop, which is why it is gated out of the package's default `pnpm test` and run + # here instead. Unlike the report job + # below it has no token to lose, so it runs for fork PRs too, and unlike the report job it is + # allowed to fail the build. + # + # Gated to this package's own paths rather than running on every route pull request. What the + # corpus measures is the TOOL's resistance to laundering, and only an edit to the tool can weaken + # that, so a routes-only change was paying a couple of minutes of a 4x runner for a result that + # could not differ from the last one. It was also the worst kind of job to spend that on: a red x + # that fires on a large share of webapp pull requests, is allowed to fail, and gates nothing, which is + # the shape people learn to scroll past. + # + # What this gives up is real and small. A route landing a shape no corpus entry has seen can make + # a known laundering mutation start paying, and that is now caught by the nightly rather than by + # the pull request that caused it. Tree drift accrues over months, so a day is the right + # granularity for it; the tool's own regressions, which are the ones a single commit can cause, + # still gate per pull request. + # + # Nothing in this repo watches whether the nightly itself succeeds: no Slack webhook and no + # issue-on-failure step here or in e2e-webapp-auth-full.yml, the only other scheduled workflow, so + # there is no house pattern to follow. A broken corpus fails quietly on the 3am cron, red only in + # the Actions tab, + # for as long as nobody checks it. Wiring up a real notification needs infrastructure (a Slack + # webhook secret, at minimum) that does not exist here yet, so this is a known, unfixed gap + # rather than a fixed one. + mutation-corpus: + name: ๐Ÿงฌ Mutation corpus + needs: changes + # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit + # success() on the job it names, and that implicit test outranks the `||` below: with a plain + # condition, a `changes` job that failed or was skipped skips this one, so the nightly would + # stop scanning for tree drift and report nothing about having stopped. A status-check function + # in the `if` is what drops the implicit success(), so the event test below decides alone. + # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a + # superseded run should not finish this job. + # + # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves + # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop + # for that, which is the same trade the paths gate already makes for routes-only pull requests. + if: >- + !cancelled() && + (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true') + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿงฌ Run the corpus + env: + OBS_MAP_MUTATION_CORPUS: "1" + run: | + pnpm --filter @internal/observability-map exec vitest run \ + src/mutationCorpus.test.ts --disable-console-intercept + + # The package's own tests are NOT run here. They gate through pr_checks.yml, which is the only + # workflow the all-checks aggregate can see, so a job in this file would report a result nobody + # is required to wait for. See unit-tests-observability-map.yml and the obsmap filter. + report: + needs: changes + runs-on: warp-ubuntu-latest-x64-4x + # Only this job comments, so only this job gets the write. + permissions: + contents: read + pull-requests: write + # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. + # The event test is what keeps this job off the nightly, which has no pull request to comment on + # and only exists for the corpus job above. + # + # The two output tests are what the workflow-level paths filter used to do, plus the thing it + # could not do. The report has to run when the watched paths moved, and ALSO when they did not + # but a marker comment is already on the pull request, because that comment is the one showing + # findings that have left the diff. Reconciling it needs no scan, so the steps below are gated + # again on the same output. + # + # `needs` carries an implicit success() and that is wanted here: a `changes` job that failed + # knows neither which paths moved nor whether a comment exists, and a report job that ran anyway + # could only guess. Same reason the lookup test is positive rather than a check for a failure + # sentinel: retries exhausted, or the lookup step dying anywhere unforeseen, both leave the + # output unset and both mean the same thing, so neither can be read as "no comment exists" by + # one step and "a comment exists" by another. That disagreement is what the sentinel pair this + # replaces got wrong once already. + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + needs.changes.outputs.lookup == 'ok' && + (needs.changes.outputs.report == 'true' || needs.changes.outputs.comment != '') + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + # Guarded rather than allowed to fail: this job must never block a pull request. The failure + # is not swallowed either, the render step below turns a missing head report into a comment + # saying so, because a swallowed failure with no comment is the outcome nobody wants. + # + # `--out` rather than a stdout redirect, so nothing a tool decides to print can end up inside + # the document `prCommentCli` parses. `pnpm --filter` takes its recursive path and some + # versions announce `Scope: N of M workspace projects` on the way; that line landing in + # head.json would fail the parse and degrade every run to the stale-report comment, which is + # a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 + # pinned above, so this closes the class rather than a reproduction: the file is written by + # the process that owns it and stdout is left to be log output. Held by + # `it("let the scanner write its own report rather than capturing stdout")` in + # `internal-packages/observability-map/src/integration.test.ts`. + # + # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner + # that exits 0 without writing takes the else branch and the stale-report comment, instead of + # failing the `mv` and turning the job red. + # + # Gated: this is the expensive half, and the reconcile run has nothing to compare. The steps + # above it are not gated because the renderer is TypeScript in this repo, so reconciling still + # needs the checkout and the install. That is the cost of the reconcile run and it is paid only + # by a pull request that has a comment and no longer matches the paths. + - name: ๐Ÿ”Ž Scan head + if: needs.changes.outputs.report == 'true' + run: | + if pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then + mv /tmp/head.json.partial /tmp/head.json + else + rm -f /tmp/head.json /tmp/head.json.partial + echo "head scan failed; the comment will say the report is stale for this run" >&2 + fi + + # base.sha, not a merge base, and two reviewers have now read that as a bug. The checkout + # above is the default for a pull_request event, so the working tree is GitHub's test merge + # commit, whose parents are base.sha and the PR head. The head tree therefore already contains + # the base branch up to base.sha, and diffing it against base.sha is what isolates this pull + # request's own work. A merge base would leave the intervening base-branch commits in the head + # tree and out of the base tree, and blame the pull request for all of them. + - name: ๐Ÿ”Ž Scan base with the head's scanner + if: needs.changes.outputs.report == 'true' + run: | + if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ + && pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --routes=/tmp/base-tree/apps/webapp/app/routes --out=/tmp/base.json \ + && [ -s /tmp/base.json ]; then + : + else + echo "-" > /tmp/base.json || true + echo "base scan failed or the worktree could not be added; falling back to no base" >&2 + fi + + # continue-on-error for the same reason as the scan: a rendering bug must not turn the job + # red. An empty /tmp/comment.md means there is nothing to post, which is a decision + # prCommentCli makes, not this shell. + # + # Both shas are forwarded so every comment this job posts says which commit it was rendered + # for, which a sticky comment edited in place across pushes otherwise never tells you. They go + # through the CLI as data: the renderer builds no URL and reads no environment. + - name: ๐Ÿ“ Render comment + continue-on-error: true + env: + SCANNED: ${{ needs.changes.outputs.report }} + EXISTING_COMMENT: ${{ needs.changes.outputs.comment }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + COMPARE_URL: ${{ github.server_url }}/${{ github.repository }}/compare/${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} + run: | + rm -f /tmp/comment.md + # `--out` rather than a stdout redirect, for the reason the scan steps above give, and with a + # worse failure mode than theirs: the marker has to be the comment's first line for the + # lookup to find it, so a line printed ahead of the document makes every push post a new + # comment instead of updating the one already there. Held by + # `it("let the renderer write its own comment rather than capturing stdout")`. + render() { + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts \ + --commit-sha="$HEAD_SHA" --commit-url="$COMPARE_URL" --out=/tmp/comment.md.partial "$@" + } + + # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte + # comment.md for the upsert to skip in silence. + emit() { + rm -f /tmp/comment.md.partial + if render "$@"; then + mv /tmp/comment.md.partial /tmp/comment.md + return 0 + fi + rm -f /tmp/comment.md.partial + return 1 + } + + # Nothing this workflow watches moved, so nothing was scanned and there is no delta to + # compute. The job's gate only lets that case through when a comment from an earlier push + # is on the pull request, so there is exactly one thing left to say: what it shows is not + # in this diff any more. + if [ "$SCANNED" != "true" ]; then + emit --resolved || echo "could not render the resolved comment" >&2 + exit 0 + fi + + if [ ! -s /tmp/head.json ]; then + emit --scan-failed || echo "could not render the stale-report comment either" >&2 + exit 0 + fi + + base=/tmp/base.json + if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then + base="-" + fi + + flags=() + if [ -n "$EXISTING_COMMENT" ]; then + flags=(--existing-comment) + fi + + if ! emit /tmp/head.json "$base" "${flags[@]}"; then + echo "render failed; falling back to the stale-report comment" >&2 + emit --scan-failed || echo "could not render the stale-report comment either" >&2 + fi + + # continue-on-error for the same reason: a transient gh api failure (rate limit, network) + # must not fail the job either. Worst case, the PR gets no comment this run. + # + # The id comes from the same job output the render step read, so the two cannot disagree about + # whether a comment exists. A lookup that did not finish cleanly never reaches either of them: + # the job's gate stops it. + - name: ๐Ÿ’ฌ Upsert PR comment + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EXISTING_COMMENT: ${{ needs.changes.outputs.comment }} + run: | + if [ ! -s /tmp/comment.md ]; then + echo "nothing to post: this pull request does not move the report" + exit 0 + fi + if [ -n "$EXISTING_COMMENT" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING_COMMENT}" -F body=@/tmp/comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md + fi diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b6be1eddfa..f8013c50d1 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -3,9 +3,6 @@ name: ๐Ÿค– PR Checks on: pull_request: types: [opened, synchronize, reopened] - paths-ignore: - - "docs/**" - - ".changeset/**" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -13,19 +10,227 @@ concurrency: permissions: contents: read - id-token: write + pull-requests: read jobs: + changes: + name: Detect changes + runs-on: warp-ubuntu-latest-x64-2x + outputs: + code: ${{ steps.code_filter.outputs.code }} + typecheck_self: ${{ steps.filter.outputs.typecheck_self }} + webapp: ${{ steps.filter.outputs.webapp }} + packages: ${{ steps.filter.outputs.packages }} + internal: ${{ steps.filter.outputs.internal }} + obsmap: ${{ steps.filter.outputs.obsmap }} + cli: ${{ steps.filter.outputs.cli }} + sdk: ${{ steps.filter.outputs.sdk }} + steps: + # `code` uses `every` semantics so the negation patterns actually subtract. + # With the default `some` quantifier, `**` matches every file and the + # subsequent `!...` patterns are no-ops (each pattern is OR'd, not AND'd). + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: code_filter + with: + predicate-quantifier: every + filters: | + code: + - '**' + - '!docs/**' + - '!.changeset/**' + - '!hosting/**' + - '!.github/**' + - '!**/*.md' + - '!**/.env.example' + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: filter + with: + filters: | + typecheck_self: + - '.github/workflows/pr_checks.yml' + - '.github/workflows/typecheck.yml' + - '.github/workflows/code-quality.yml' + webapp: + - 'apps/webapp/**' + - 'packages/**' + - 'internal-packages/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/unit-tests-webapp.yml' + - '.github/workflows/e2e-webapp.yml' + - '.github/workflows/runops-guard.yml' + - '.github/workflows/fk-cascade-guard.yml' + - '.configs/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'turbo.json' + packages: + - 'packages/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/unit-tests-packages.yml' + - '.configs/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'turbo.json' + internal: + - 'internal-packages/**' + - 'packages/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/unit-tests-internal.yml' + - '.configs/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'turbo.json' + # The whole webapp app tree, not just its routes, and that is the whole reason this + # filter exists. Two tests in @internal/observability-map read it: integration.test.ts + # scans the live route tree, and webappSymbols.test.ts walks all of apps/webapp/app and + # fails when a guard, sensitive or audit symbol stops resolving. Routes-only was this + # filter's own bug: renaming e.g. requireUserId in app/services/session.server.ts + # matched `webapp` and nothing else, so no job ran the suite and the break landed on + # main, or on the next unrelated internal-packages PR. + # + # The cost of the wider set, measured over the last 400 commits on main: 31% touch + # routes, 52% touch apps/webapp/app, so the job goes from firing on roughly a third of + # PRs to roughly a half. It is the cheap one -- a single 4x runner, no containers, no + # database, no prisma generate -- which is what makes that affordable. + # + # observability-map.yml is here because integration.test.ts asserts on its text and no + # other filter watches it, so editing the report workflow alone ran nothing at all. + # + # Deliberately NOT here: this package's own paths, and packages/plugins/src and + # internal-packages/rbac/src, the other two trees webappSymbols.test.ts reads. + # `internal` above already matches `internal-packages/**` and `packages/**`, and + # `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, which picks up + # @internal/observability-map and runs the same vitest suite. Listing them here as well + # ran the suite twice on every PR touching them, which was this filter's own doing. + # + # Also deliberately NOT here: pr_checks.yml, package.json, pnpm-lock.yaml, + # pnpm-workspace.yaml. `internal` already lists all four, so a PR touching only one of + # them ran this suite twice for the same reason as above. Editing pr_checks.yml no + # longer runs this job live as a result; integration.test.ts still asserts on its text + # via the `internal` job. + obsmap: + - 'apps/webapp/app/**' + - '.github/workflows/unit-tests-observability-map.yml' + - '.github/workflows/observability-map.yml' + cli: + - 'packages/cli-v3/**' + - 'packages/build/**' + - 'packages/core/**' + - 'packages/schema-to-json/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/e2e.yml' + - '.configs/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'turbo.json' + sdk: + - 'packages/trigger-sdk/**' + - 'packages/core/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/sdk-compat.yml' + - '.configs/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'turbo.json' + + code-quality: + uses: ./.github/workflows/code-quality.yml + typecheck: + needs: changes + if: needs.changes.outputs.code == 'true' || needs.changes.outputs.typecheck_self == 'true' uses: ./.github/workflows/typecheck.yml - secrets: inherit - units: - uses: ./.github/workflows/unit-tests.yml - secrets: inherit + runops-guard: + needs: changes + if: needs.changes.outputs.webapp == 'true' + uses: ./.github/workflows/runops-guard.yml + + fk-cascade-guard: + needs: changes + if: needs.changes.outputs.webapp == 'true' + uses: ./.github/workflows/fk-cascade-guard.yml + + webapp: + needs: changes + if: needs.changes.outputs.webapp == 'true' + uses: ./.github/workflows/unit-tests-webapp.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + e2e-webapp: + needs: changes + if: needs.changes.outputs.webapp == 'true' + uses: ./.github/workflows/e2e-webapp.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + packages: + needs: changes + if: needs.changes.outputs.packages == 'true' + uses: ./.github/workflows/unit-tests-packages.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + internal: + needs: changes + if: needs.changes.outputs.internal == 'true' + uses: ./.github/workflows/unit-tests-internal.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + obsmap: + needs: changes + if: needs.changes.outputs.obsmap == 'true' + uses: ./.github/workflows/unit-tests-observability-map.yml e2e: + needs: changes + if: needs.changes.outputs.cli == 'true' uses: ./.github/workflows/e2e.yml with: package: cli-v3 - secrets: inherit + + sdk-compat: + needs: changes + if: needs.changes.outputs.sdk == 'true' + uses: ./.github/workflows/sdk-compat.yml + + all-checks: + name: All PR Checks + needs: + - changes + - code-quality + - typecheck + - runops-guard + - fk-cascade-guard + - webapp + - e2e-webapp + - packages + - internal + - obsmap + - e2e + - sdk-compat + if: always() + runs-on: warp-ubuntu-latest-x64-2x + steps: + - name: Verify all checks + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "One or more checks failed" + exit 1 + fi + if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more checks were cancelled" + exit 1 + fi + echo "All checks passed or were skipped due to path filters" diff --git a/.github/workflows/preview-dispatch.yml b/.github/workflows/preview-dispatch.yml new file mode 100644 index 0000000000..ac0decfafa --- /dev/null +++ b/.github/workflows/preview-dispatch.yml @@ -0,0 +1,76 @@ +name: ๐ŸŒฑ Preview environment dispatch + +# Opt-in per-PR preview environments + +on: + pull_request: + types: [opened, reopened, synchronize, closed, labeled, unlabeled] + +# Serialize a PR's events so dispatches arrive in order. Cloud-side concurrency +# collapses by branch but can't fix out-of-order arrival โ€” e.g. a push racing a +# close could cancel the in-flight destroy and leak the preview. One short API +# call, so queuing is cheap; cancel-in-progress: false lets an in-flight +# dispatch finish (GitHub keeps only the latest pending, the desired behavior). +concurrency: + group: preview-dispatch-${{ github.event.pull_request.number }} + cancel-in-progress: false + +permissions: {} + +jobs: + dispatch: + name: Dispatch preview-deploy to cloud + runs-on: warp-ubuntu-latest-x64-2x + # label added -> create + # new commit while labeled -> update + # label removed / PR closed -> destroy + if: >- + github.event.pull_request.head.repo.full_name == github.repository && + ( + (github.event.action == 'labeled' && github.event.label.name == 'preview') || + (github.event.action == 'unlabeled' && github.event.label.name == 'preview') || + ( + contains(github.event.pull_request.labels.*.name, 'preview') && + contains(fromJSON('["opened","reopened","synchronize","closed"]'), github.event.action) + ) + ) + steps: + - name: Build dispatch payload + id: payload + env: + ACTION: ${{ github.event.action }} + BRANCH: ${{ github.event.pull_request.head.ref }} + COMMIT: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + # Map the GitHub PR action to the cloud pipeline's lifecycle event. + case "$ACTION" in + labeled | opened | reopened) EVENT=opened ;; + synchronize) EVENT=synchronize ;; + unlabeled | closed) EVENT=closed ;; + *) echo "unexpected action: $ACTION" >&2; exit 1 ;; + esac + # jq --arg JSON-escapes every value, so a branch name containing + # quotes/braces can't break or inject into the client payload. + payload=$(jq -nc \ + --arg b "$BRANCH" \ + --arg c "$COMMIT" \ + --arg e "$EVENT" \ + '{branch_name: $b, commit: $c, pull_request_event: $e}') + { + echo "client_payload=$payload" + echo "summary=$EVENT for $BRANCH @ ${COMMIT:0:7}" + } >> "$GITHUB_OUTPUT" + + - name: Log dispatch + env: + SUMMARY: ${{ steps.payload.outputs.summary }} + run: echo "Dispatching preview-deploy event ($SUMMARY)" + + - name: Send repository_dispatch + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 + with: + token: ${{ secrets.CROSS_REPO_PAT }} + repository: triggerdotdev/cloud + event-type: preview-deploy + client-payload: ${{ steps.payload.outputs.client_payload }} diff --git a/.github/workflows/preview-packages.yml b/.github/workflows/preview-packages.yml new file mode 100644 index 0000000000..b5bdbb3134 --- /dev/null +++ b/.github/workflows/preview-packages.yml @@ -0,0 +1,83 @@ +name: ๐Ÿ“ฆ Preview packages (pkg.pr.new) + +# Publishes installable preview builds of the public @trigger.dev/* packages +# for every push to a branch, via https://pkg.pr.new. These are NOT published +# to npm โ€” pkg.pr.new serves them by commit SHA and drops install instructions +# in a comment on the associated PR, e.g. +# npm i https://pkg.pr.new/@trigger.dev/sdk@ +# +# Prerequisites: +# - The pkg.pr.new GitHub App must be installed on triggerdotdev/trigger.dev +# (https://github.com/apps/pkg-pr-new). Publishing fails until it is. +# +# Fork note: pkg.pr.new authenticates with a GitHub Actions OIDC token, which +# GitHub does not issue to pull_request workflows from forks. This `push` +# trigger therefore covers branches pushed to this repo (the core team), not +# external fork PRs. Adding fork coverage would require a workflow_run two-stage +# setup. + +on: + push: + branches-ignore: + - main + - changeset-release/main + paths: + - "package.json" + - "packages/**" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "turbo.json" + - ".github/workflows/preview-packages.yml" + - "scripts/stamp-preview-version.mjs" + - "scripts/updateVersion.ts" + +concurrency: + group: preview-packages-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + id-token: write # OIDC token used by pkg.pr.new to authenticate the publish + +jobs: + publish: + name: Build and publish previews + runs-on: warp-ubuntu-latest-x64-8x + if: github.repository == 'triggerdotdev/trigger.dev' + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Install dependencies + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma client + run: pnpm run generate + + # Stamp a unique 0.0.0-preview- version before building so it can't + # collide with real npm versions and so updateVersion.ts bakes it into the + # runtime VERSION constant. See scripts/stamp-preview-version.mjs. + - name: ๐Ÿท๏ธ Stamp preview version + run: node scripts/stamp-preview-version.mjs + env: + GITHUB_SHA: ${{ github.sha }} + + - name: ๐Ÿ”จ Build packages + run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev" + + - name: ๐Ÿš€ Publish previews to pkg.pr.new + run: pnpm exec pkg-pr-new publish --pnpm --compact --commentWithSha './packages/*' diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 0000000000..f664a9a2fd --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,36 @@ +name: ๐Ÿ“š Publish docs + +on: + push: + tags: + - "docs-release-*" + +# Only needs to move the docs-live ref; Mintlify's GitHub app deploys from it. +permissions: + contents: write + +concurrency: + group: publish-docs + cancel-in-progress: false + +jobs: + publish: + runs-on: warp-ubuntu-latest-x64-2x + steps: + - name: ๐Ÿ“ฅ Checkout tagged commit + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: ๐Ÿ”— Check for broken links + working-directory: ./docs + run: npx mintlify@4.0.393 broken-links + + - name: ๐Ÿš€ Fast-forward docs-live to the tagged commit + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api -X PATCH \ + "repos/${{ github.repository }}/git/refs/heads/docs-live" \ + -f sha="${{ github.sha }}" \ + -F force=false diff --git a/.github/workflows/publish-webapp.yml b/.github/workflows/publish-webapp.yml index 6fcc30209a..497cdfb8ed 100644 --- a/.github/workflows/publish-webapp.yml +++ b/.github/workflows/publish-webapp.yml @@ -4,6 +4,7 @@ permissions: contents: read packages: write id-token: write + attestations: write on: workflow_call: @@ -13,23 +14,47 @@ on: type: string required: false default: "" + image_registry: + description: The registry namespace to publish under (e.g. ghcr.io/) + type: string + required: false + default: "" + outputs: + version: + description: The published image tag + value: ${{ jobs.publish.outputs.version }} + short_sha: + description: Short commit SHA of the published build + value: ${{ jobs.publish.outputs.short_sha }} + image_repo: + description: The image repository the build was published to (without tag) + value: ${{ jobs.publish.outputs.image_repo }} + digest: + description: Multi-arch index digest (sha256:...) of the published image + value: ${{ jobs.publish.outputs.digest }} + secrets: + SENTRY_AUTH_TOKEN: + required: false jobs: publish: - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x env: PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1 outputs: version: ${{ steps.get_tag.outputs.tag }} short_sha: ${{ steps.get_commit.outputs.sha_short }} + image_repo: ${{ steps.set_tags.outputs.image_repo }} + digest: ${{ steps.build_push.outputs.digest }} steps: - name: ๐Ÿญ Setup Depot CLI - uses: depot/setup-action@v1 + uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1 - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: submodules: recursive + persist-credentials: false - name: "#๏ธโƒฃ Get the image tag" id: get_tag @@ -40,42 +65,61 @@ jobs: - name: ๐Ÿ”ข Get the commit hash id: get_commit run: | - echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT" + echo "sha_short=$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: ๐Ÿ“› Set the tags id: set_tags run: | - ref_without_tag=ghcr.io/triggerdotdev/trigger.dev - image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }} + # The registry namespace is resolved by the caller (defaulting to + # ghcr.io/, overridable via the IMAGE_REGISTRY repository + # variable); the webapp image lives at /. A fork + # therefore publishes to its own package automatically. + image_tags=$REF_WITHOUT_TAG:${STEPS_GET_TAG_OUTPUTS_TAG} - # if tag is a semver, also tag it as v4 - if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then - # TODO: switch to v4 tag on GA - image_tags=$image_tags,$ref_without_tag:v4-beta + if [[ "${STEPS_GET_TAG_OUTPUTS_TAG}" =~ ^v4\.[0-9]+\.[0-9]+$ ]]; then + image_tags=$image_tags,$REF_WITHOUT_TAG:v4,$REF_WITHOUT_TAG:latest + fi + + # when pushing the mutable main tag, also push an immutable-by-convention + # full-commit-sha tag so a commit can be resolved to a specific digest + if [[ "${STEPS_GET_TAG_OUTPUTS_TAG}" == "main" ]]; then + image_tags=$image_tags,$REF_WITHOUT_TAG:${GITHUB_SHA} fi echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT" + echo "image_repo=${REF_WITHOUT_TAG}" >> "$GITHUB_OUTPUT" + env: + REF_WITHOUT_TAG: ${{ format('{0}/{1}', inputs.image_registry || vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner), github.event.repository.name) }} + STEPS_GET_TAG_OUTPUTS_TAG: ${{ steps.get_tag.outputs.tag }} + STEPS_GET_TAG_OUTPUTS_IS_SEMVER: ${{ steps.get_tag.outputs.is_semver }} - name: ๐Ÿ“ Set the build info id: set_build_info run: | - tag=${{ steps.get_tag.outputs.tag }} - if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then - echo "BUILD_APP_VERSION=${tag}" >> "$GITHUB_OUTPUT" - fi - echo "BUILD_GIT_SHA=${{ github.sha }}" >> "$GITHUB_OUTPUT" - echo "BUILD_GIT_REF_NAME=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" - echo "BUILD_TIMESTAMP_SECONDS=$(date +%s)" >> "$GITHUB_OUTPUT" + { + tag="${STEPS_GET_TAG_OUTPUTS_TAG}" + if [[ "${STEPS_GET_TAG_OUTPUTS_IS_SEMVER}" == true ]]; then + echo "BUILD_APP_VERSION=${tag}" + fi + echo "BUILD_GIT_SHA=${GITHUB_SHA}" + echo "BUILD_GIT_REF_NAME=${GITHUB_REF_NAME}" + echo "BUILD_TIMESTAMP_SECONDS=$(date +%s)" + echo "BUILD_TIMESTAMP_RFC3339=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >> "$GITHUB_OUTPUT" + env: + STEPS_GET_TAG_OUTPUTS_TAG: ${{ steps.get_tag.outputs.tag }} + STEPS_GET_TAG_OUTPUTS_IS_SEMVER: ${{ steps.get_tag.outputs.is_semver }} - name: ๐Ÿ™ Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: ๐Ÿณ Build image and push to GitHub Container Registry - uses: depot/build-push-action@v1 + id: build_push + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 with: file: ./docker/Dockerfile platforms: linux/amd64,linux/arm64 @@ -86,8 +130,20 @@ jobs: BUILD_GIT_SHA=${{ steps.set_build_info.outputs.BUILD_GIT_SHA }} BUILD_GIT_REF_NAME=${{ steps.set_build_info.outputs.BUILD_GIT_REF_NAME }} BUILD_TIMESTAMP_SECONDS=${{ steps.set_build_info.outputs.BUILD_TIMESTAMP_SECONDS }} + BUILD_TIMESTAMP_RFC3339=${{ steps.set_build_info.outputs.BUILD_TIMESTAMP_RFC3339 }} SENTRY_RELEASE=${{ steps.set_build_info.outputs.BUILD_GIT_SHA }} SENTRY_ORG=triggerdev SENTRY_PROJECT=trigger-cloud secrets: | sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }} + + - name: ๐Ÿชช Attest build provenance + # Image is already pushed by this point โ€” don't fail releases (and the + # downstream publish-helm job) on a Sigstore/GHCR-referrer hiccup. Real + # config errors still surface as a step warning in the workflow run. + continue-on-error: true + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-name: ${{ steps.set_tags.outputs.image_repo }} + subject-digest: ${{ steps.build_push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/publish-worker-v4.yml b/.github/workflows/publish-worker-v4.yml index 4a2853da08..2dba27013b 100644 --- a/.github/workflows/publish-worker-v4.yml +++ b/.github/workflows/publish-worker-v4.yml @@ -8,6 +8,18 @@ on: type: string required: false default: "" + image_registry: + description: The registry namespace to publish under (e.g. ghcr.io/) + type: string + required: false + default: "" + outputs: + version: + description: The published image tag + value: ${{ jobs.build.outputs.version }} + image_repo: + description: The image repository the build was published to (without tag) + value: ${{ jobs.build.outputs.image_repo }} push: tags: - "re2-test-*" @@ -20,7 +32,7 @@ permissions: jobs: # check-branch: - # runs-on: ubuntu-latest + # runs-on: warp-ubuntu-latest-x64-2x # steps: # - name: Fail if re2-prod-* is pushed from a non-main branch # if: startsWith(github.ref_name, 're2-prod-') && github.base_ref != 'main' @@ -32,24 +44,32 @@ jobs: strategy: matrix: package: [supervisor] - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x + # Single-entry matrix, so these job outputs are unambiguous (consumed by the + # scan-supervisor job in publish.yml). + outputs: + version: ${{ steps.get_tag.outputs.tag }} + image_repo: ${{ steps.set_tags.outputs.image_repo }} env: DOCKER_BUILDKIT: "1" steps: - name: ๐Ÿญ Setup Depot CLI - uses: depot/setup-action@v1 + uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1 - name: โฌ‡๏ธ Checkout git repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: ๐Ÿ“ฆ Get image repo id: get_repository + env: + PACKAGE: ${{ matrix.package }} run: | - if [[ "${{ matrix.package }}" == *-provider ]]; then - provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1) - repo=provider/${provider_type} + if [[ "$PACKAGE" == *-provider ]]; then + repo="provider/${PACKAGE%-provider}" else - repo="${{ matrix.package }}" + repo="$PACKAGE" fi echo "repo=${repo}" >> "$GITHUB_OUTPUT" @@ -62,26 +82,33 @@ jobs: - name: ๐Ÿ“› Set tags to push id: set_tags run: | - ref_without_tag=ghcr.io/triggerdotdev/${{ steps.get_repository.outputs.repo }} - image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }} + # Resolved by the caller when invoked from publish.yml; falls back to the + # IMAGE_REGISTRY repository variable (or ghcr.io/) for the direct + # push triggers above, so a fork publishes to its own namespace. + ref_without_tag=${IMAGE_REGISTRY}/${STEPS_GET_REPOSITORY_OUTPUTS_REPO} + image_tags=$ref_without_tag:${STEPS_GET_TAG_OUTPUTS_TAG} - # if tag is a semver, also tag it as v4 - if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then - # TODO: switch to v4 tag on GA - image_tags=$image_tags,$ref_without_tag:v4-beta + if [[ "${STEPS_GET_TAG_OUTPUTS_TAG}" =~ ^v4\.[0-9]+\.[0-9]+$ ]]; then + image_tags=$image_tags,$ref_without_tag:v4,$ref_without_tag:latest fi echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT" + echo "image_repo=${ref_without_tag}" >> "$GITHUB_OUTPUT" + env: + IMAGE_REGISTRY: ${{ inputs.image_registry || vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} + STEPS_GET_REPOSITORY_OUTPUTS_REPO: ${{ steps.get_repository.outputs.repo }} + STEPS_GET_TAG_OUTPUTS_TAG: ${{ steps.get_tag.outputs.tag }} + STEPS_GET_TAG_OUTPUTS_IS_SEMVER: ${{ steps.get_tag.outputs.is_semver }} - name: ๐Ÿ™ Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: ๐Ÿณ Build image and push to GitHub Container Registry - uses: depot/build-push-action@v1 + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 with: file: ./apps/${{ matrix.package }}/Containerfile platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/publish-worker.yml b/.github/workflows/publish-worker.yml deleted file mode 100644 index 74a70d8366..0000000000 --- a/.github/workflows/publish-worker.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: "โš’๏ธ Publish Worker" - -on: - workflow_call: - inputs: - image_tag: - description: The image tag to publish - type: string - required: false - default: "" - push: - tags: - - "infra-dev-*" - - "infra-test-*" - - "infra-prod-*" - -permissions: - packages: write - contents: read - -jobs: - build: - strategy: - matrix: - package: [coordinator, docker-provider, kubernetes-provider] - runs-on: ubuntu-latest - env: - DOCKER_BUILDKIT: "1" - steps: - - name: โฌ‡๏ธ Checkout git repo - uses: actions/checkout@v4 - - - name: ๐Ÿ“ฆ Get image repo - id: get_repository - run: | - if [[ "${{ matrix.package }}" == *-provider ]]; then - provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1) - repo=provider/${provider_type} - else - repo="${{ matrix.package }}" - fi - echo "repo=${repo}" >> "$GITHUB_OUTPUT" - - - id: get_tag - uses: ./.github/actions/get-image-tag - with: - tag: ${{ inputs.image_tag }} - - - name: ๐Ÿ‹ Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # ..to avoid rate limits when pulling images - - name: ๐Ÿณ Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: ๐Ÿšข Build Container Image - run: | - docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile . - - # ..to push image - - name: ๐Ÿ™ Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: ๐Ÿ™ Push to GitHub Container Registry - run: | - docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - env: - REGISTRY: ghcr.io/triggerdotdev - REPOSITORY: ${{ steps.get_repository.outputs.repo }} - IMAGE_TAG: ${{ steps.get_tag.outputs.tag }} - - # - name: ๐Ÿ™ Push 'v3' tag to GitHub Container Registry - # if: steps.get_tag.outputs.is_semver == 'true' - # run: | - # docker tag infra_image "$REGISTRY/$REPOSITORY:v3" - # docker push "$REGISTRY/$REPOSITORY:v3" - # env: - # REGISTRY: ghcr.io/triggerdotdev - # REPOSITORY: ${{ steps.get_repository.outputs.repo }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6213499c5a..1b9e55c3b3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,6 +8,15 @@ on: description: The image tag to publish required: true type: string + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + SENTRY_AUTH_TOKEN: + required: false + CROSS_REPO_PAT: + required: false push: branches: - main @@ -21,7 +30,6 @@ on: - ".github/workflows/unit-tests.yml" - ".github/workflows/e2e.yml" - ".github/workflows/publish-webapp.yml" - - ".github/workflows/publish-worker.yml" - "packages/**" - "!packages/**/*.md" - "!packages/**/*.eslintrc" @@ -37,8 +45,6 @@ on: - "tests/**" permissions: - id-token: write - packages: write contents: read concurrency: @@ -50,29 +56,103 @@ env: jobs: typecheck: uses: ./.github/workflows/typecheck.yml - secrets: inherit units: uses: ./.github/workflows/unit-tests.yml - secrets: inherit + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} publish-webapp: needs: [typecheck] + permissions: + contents: read + packages: write + id-token: write + attestations: write uses: ./.github/workflows/publish-webapp.yml - secrets: inherit - with: - image_tag: ${{ inputs.image_tag }} - - publish-worker: - needs: [typecheck] - uses: ./.github/workflows/publish-worker.yml - secrets: inherit + secrets: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} with: image_tag: ${{ inputs.image_tag }} + # Target registry namespace. Defaults to ghcr.io/ so a fork publishes + # to its own namespace; set the IMAGE_REGISTRY repository variable to override. + image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} publish-worker-v4: needs: [typecheck] + permissions: + contents: read + packages: write + id-token: write uses: ./.github/workflows/publish-worker-v4.yml - secrets: inherit with: image_tag: ${{ inputs.image_tag }} + image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} + + # OS-level CVE scan of the image just published above. Report-only (writes to + # the run summary); runs alongside the worker publishes and never blocks them. + scan-webapp: + needs: [publish-webapp] + permissions: + contents: read + packages: read # pull the just-published image from GHCR + uses: ./.github/workflows/trivy-image.yml + with: + image-ref: ${{ needs.publish-webapp.outputs.image_repo }}:${{ needs.publish-webapp.outputs.version }} + + scan-supervisor: + needs: [publish-worker-v4] + permissions: + contents: read + packages: read # pull the just-published image from GHCR + uses: ./.github/workflows/trivy-image.yml + with: + image-ref: ${{ needs.publish-worker-v4.outputs.image_repo }}:${{ needs.publish-worker-v4.outputs.version }} + + # Announce the freshly published mutable `main` webapp image to subscriber + # repos via repository_dispatch, handing them a digest-pinned ref to build or + # deploy from. The repo, ref prefix, and dispatch target all default to the + # canonical values and can be overridden by repository variables. + # + # `push` only: release builds reach publish.yml via workflow_call (from + # release.yml) with an explicit image_tag while github.ref_name is still + # `main`, so gate on the event to avoid dispatching โ€” and failing on the + # absent CROSS_REPO_PAT โ€” during a release. + dispatch-main-image: + name: ๐Ÿ“ฃ Dispatch main image + needs: [publish-webapp] + if: github.repository == (vars.MAIN_IMAGE_DISPATCH_REPO || 'triggerdotdev/trigger.dev') && github.event_name == 'push' && startsWith(github.ref_name, vars.MAIN_IMAGE_DISPATCH_REF_PREFIX || 'main') + runs-on: warp-ubuntu-latest-x64-2x + permissions: {} + steps: + - name: Build dispatch payload + id: payload + env: + IMAGE_REPO: ${{ needs.publish-webapp.outputs.image_repo }} + DIGEST: ${{ needs.publish-webapp.outputs.digest }} + COMMIT: ${{ github.sha }} + run: | + set -euo pipefail + # Pin to the exact multi-arch index just pushed so subscribers resolve a + # single immutable artifact rather than chasing the moving `main` tag. + if [[ -z "${DIGEST}" ]]; then + echo "::error::publish-webapp produced no image digest; refusing to dispatch" + exit 1 + fi + image="${IMAGE_REPO}@${DIGEST}" + # jq --arg JSON-escapes every value, so the ref/commit can't break out of + # or inject into the client payload. + payload=$(jq -nc \ + --arg img "$image" \ + --arg c "$COMMIT" \ + '{image: $img, commit: $c}') + echo "client_payload=$payload" >> "$GITHUB_OUTPUT" + + - name: Send repository_dispatch + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 + with: + token: ${{ secrets.CROSS_REPO_PAT }} + repository: ${{ vars.MAIN_IMAGE_DISPATCH_TARGET || 'triggerdotdev/cloud' }} + event-type: main-image-published + client-payload: ${{ steps.payload.outputs.client_payload }} diff --git a/.github/workflows/release-helm.yml b/.github/workflows/release-helm.yml index c6efd382ff..0ce968ebc4 100644 --- a/.github/workflows/release-helm.yml +++ b/.github/workflows/release-helm.yml @@ -3,11 +3,17 @@ name: ๐Ÿงญ Helm Chart Release on: push: tags: - - 'helm-v*' + - "helm-v*" + workflow_call: + inputs: + chart_version: + description: "Chart version to release" + required: true + type: string workflow_dispatch: inputs: chart_version: - description: 'Chart version to release' + description: "Chart version to release" required: true type: string @@ -17,15 +23,17 @@ env: jobs: lint-and-test: - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x permissions: contents: read steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: Set up Helm - uses: azure/setup-helm@v4 + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: "3.18.3" @@ -39,32 +47,41 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests - uses: docker://ghcr.io/yannh/kubeconform:v0.7.0 + uses: docker://ghcr.io/yannh/kubeconform:v0.7.0@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c with: - entrypoint: '/kubeconform' + entrypoint: "/kubeconform" args: "-summary -output json ./helm-output" release: needs: lint-and-test - runs-on: ubuntu-latest + # Set the ENABLE_HELM_PRERELEASE repository variable to 'false' to turn off + # publishing the chart to GHCR โ€” e.g. forks/mirrors that lack write_package + # on the owner's charts namespace. Defaults to enabled; the lint-and-test + # job above always runs regardless. + if: ${{ vars.ENABLE_HELM_PRERELEASE != 'false' }} + runs-on: warp-ubuntu-latest-x64-2x permissions: contents: write # for gh-release packages: write steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: Set up Helm - uses: azure/setup-helm@v4 + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: "3.18.3" @@ -77,7 +94,7 @@ jobs: for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done - name: Log in to Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -86,18 +103,20 @@ jobs: - name: Extract version from tag or input id: version run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - VERSION="${{ github.event.inputs.chart_version }}" + if [ -n "${INPUTS_CHART_VERSION}" ]; then + VERSION="${INPUTS_CHART_VERSION}" else - VERSION="${{ github.ref_name }}" + VERSION="${GITHUB_REF_NAME}" VERSION="${VERSION#helm-v}" fi - echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Releasing version: $VERSION" + env: + INPUTS_CHART_VERSION: ${{ inputs.chart_version }} - name: Check Chart.yaml version matches release version run: | - VERSION="${{ steps.version.outputs.version }}" + VERSION="${STEPS_VERSION_OUTPUTS_VERSION}" CHART_VERSION=$(grep '^version:' ./hosting/k8s/helm/Chart.yaml | awk '{print $2}') echo "Chart.yaml version: $CHART_VERSION" echo "Release version: $VERSION" @@ -106,6 +125,8 @@ jobs: exit 1 fi echo "โœ… Chart.yaml version matches release version." + env: + STEPS_VERSION_OUTPUTS_VERSION: ${{ steps.version.outputs.version }} - name: Package Helm Chart run: | @@ -113,18 +134,19 @@ jobs: - name: Push Helm Chart to GHCR run: | - VERSION="${{ steps.version.outputs.version }}" + VERSION="${STEPS_VERSION_OUTPUTS_VERSION}" CHART_PACKAGE="/tmp/${{ env.CHART_NAME }}-${VERSION}.tgz" - + # Push to GHCR OCI registry helm push "$CHART_PACKAGE" "oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts" + env: + STEPS_VERSION_OUTPUTS_VERSION: ${{ steps.version.outputs.version }} - name: Create GitHub Release id: release - uses: softprops/action-gh-release@v1 - if: github.event_name == 'push' + uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 with: - tag_name: ${{ github.ref_name }} + tag_name: helm-v${{ steps.version.outputs.version }} name: "Helm Chart ${{ steps.version.outputs.version }}" body: | ### Installation @@ -133,7 +155,7 @@ jobs: oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts/${{ env.CHART_NAME }} \ --version "${{ steps.version.outputs.version }}" ``` - + ### Changes See commit history for detailed changes in this release. files: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 684d36dec3..5292903562 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,6 +33,7 @@ jobs: show-release-summary: name: ๐Ÿ“‹ Release Summary runs-on: ubuntu-latest + permissions: {} if: | github.repository == 'triggerdotdev/trigger.dev' && github.event_name == 'pull_request' && @@ -43,11 +44,11 @@ jobs: env: PR_BODY: ${{ github.event.pull_request.body }} run: | - echo "$PR_BODY" | sed -n '/^# Releases/,$p' >> $GITHUB_STEP_SUMMARY + echo "$PR_BODY" | sed -n '/^# Releases/,$p' >> "$GITHUB_STEP_SUMMARY" release: name: ๐Ÿš€ Release npm packages - runs-on: ubuntu-latest + runs-on: ubuntu-latest # this cannot run on non-GH runner environment: npm-publish permissions: contents: write @@ -63,9 +64,10 @@ jobs: published: ${{ steps.changesets.outputs.published }} published_packages: ${{ steps.changesets.outputs.publishedPackages }} published_package_version: ${{ steps.get_version.outputs.package_version }} + is_prerelease: ${{ steps.get_version.outputs.is_prerelease }} steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 # zizmor: ignore[artipacked] needs persisted git creds for tag push; no artifact upload here so no leak path with: fetch-depth: 0 ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.sha }} @@ -73,20 +75,22 @@ jobs: - name: Verify ref is on main if: github.event_name == 'workflow_dispatch' run: | - if ! git merge-base --is-ancestor ${{ github.event.inputs.ref }} origin/main; then + if ! git merge-base --is-ancestor "${GITHUB_EVENT_INPUTS_REF}" origin/main; then echo "Error: ref must be an ancestor of main (i.e., already merged)" exit 1 fi + env: + GITHUB_EVENT_INPUTS_REF: ${{ github.event.inputs.ref }} - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" # npm v11.5.1 or newer is required for OIDC support @@ -108,10 +112,10 @@ jobs: - name: Publish id: changesets - uses: changesets/action@v1 + uses: changesets/action@a45c4d594aa4e2c509dc14a9f2b3b67ba3780d0d # v1.9.0 with: publish: pnpm run changeset:release - createGithubReleases: true + createGithubReleases: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -119,21 +123,165 @@ jobs: if: steps.changesets.outputs.published == 'true' id: get_version run: | - package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version') + package_version=$(echo "${STEPS_CHANGESETS_OUTPUTS_PUBLISHEDPACKAGES}" | jq -r '.[0].version') echo "package_version=${package_version}" >> "$GITHUB_OUTPUT" + # Any semver with a hyphen is a prerelease (e.g. 4.5.0-rc.0, 0.0.0-snapshot-...) + if [[ "${package_version}" == *-* ]]; then + echo "is_prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "is_prerelease=false" >> "$GITHUB_OUTPUT" + fi + env: + STEPS_CHANGESETS_OUTPUTS_PUBLISHEDPACKAGES: ${{ steps.changesets.outputs.publishedPackages }} + + - name: Create unified GitHub release + if: steps.changesets.outputs.published == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_PR_BODY: ${{ github.event.pull_request.body }} + STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION: ${{ steps.get_version.outputs.package_version }} + STEPS_GET_VERSION_OUTPUTS_IS_PRERELEASE: ${{ steps.get_version.outputs.is_prerelease }} + run: | + VERSION="${STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION}" + + # On the pull_request merge event RELEASE_PR_BODY is the merged "Version Packages" + # PR body, which holds the changelog. On a workflow_dispatch re-run (the recovery + # path after a failed merge-triggered run) there is no pull_request context, so it's + # empty. Fall back to the body of the most recently merged changeset-release/main PR + # so the "What's changed" section is still populated. + if [ -z "${RELEASE_PR_BODY}" ]; then + echo "RELEASE_PR_BODY is empty (workflow_dispatch re-run); fetching merged changeset-release/main PR body" + RELEASE_PR_BODY="$(gh pr list --repo triggerdotdev/trigger.dev \ + --head changeset-release/main --state merged \ + --json body,mergedAt --limit 10 \ + -q 'sort_by(.mergedAt) | reverse | .[0].body')" + fi + export RELEASE_PR_BODY + + node scripts/generate-github-release.mjs "$VERSION" > /tmp/release-body.md + PRERELEASE_FLAG="" + if [ "${STEPS_GET_VERSION_OUTPUTS_IS_PRERELEASE}" = "true" ]; then + PRERELEASE_FLAG="--prerelease" + fi + gh release create "v${VERSION}" \ + --title "trigger.dev v${VERSION}" \ + --notes-file /tmp/release-body.md \ + --target main \ + $PRERELEASE_FLAG - # this triggers the publish workflow for the docker images - name: Create and push Docker tag if: steps.changesets.outputs.published == 'true' run: | set -e - git tag "v.docker.${{ steps.get_version.outputs.package_version }}" - git push origin "v.docker.${{ steps.get_version.outputs.package_version }}" + git tag "v.docker.${STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION}" + git push origin "v.docker.${STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION}" + env: + STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION: ${{ steps.get_version.outputs.package_version }} + + - name: Create and push Helm chart tag + if: steps.changesets.outputs.published == 'true' + run: | + set -e + git tag "helm-v${STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION}" + git push origin "helm-v${STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION}" + env: + STEPS_GET_VERSION_OUTPUTS_PACKAGE_VERSION: ${{ steps.get_version.outputs.package_version }} + + # Trigger Docker builds directly via workflow_call since tags pushed with + # GITHUB_TOKEN don't trigger other workflows (GitHub Actions limitation). + publish-docker: + name: ๐Ÿณ Publish Docker images + needs: release + if: needs.release.outputs.published == 'true' + permissions: + contents: read + packages: write + id-token: write + attestations: write + uses: ./.github/workflows/publish.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + with: + image_tag: v${{ needs.release.outputs.published_package_version }} + + # Trigger Helm chart release directly via workflow_call (same GITHUB_TOKEN + # limitation as the Docker path). Runs after Docker images are published so + # the chart never references images that don't exist yet. + publish-helm: + name: ๐Ÿงญ Publish Helm chart + needs: [release, publish-docker] + if: needs.release.outputs.published == 'true' + permissions: + contents: write + packages: write + uses: ./.github/workflows/release-helm.yml + with: + chart_version: ${{ needs.release.outputs.published_package_version }} + + # After Docker images are published, update the GitHub release with the exact GHCR tag URL. + # The GHCR package version ID is only known after the image is pushed, so we query for it here. + update-release: + name: ๐Ÿ”— Update release Docker link + needs: [release, publish-docker] + if: needs.release.outputs.published == 'true' + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: write + packages: read + steps: + - name: Update GitHub release with Docker image link + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEEDS_RELEASE_OUTPUTS_PUBLISHED_PACKAGE_VERSION: ${{ needs.release.outputs.published_package_version }} + run: | + set -e + VERSION="${NEEDS_RELEASE_OUTPUTS_PUBLISHED_PACKAGE_VERSION}" + TAG="v${VERSION}" + + # Query GHCR for the version ID matching this tag + VERSION_ID=$(gh api --paginate -H "Accept: application/vnd.github+json" \ + /orgs/triggerdotdev/packages/container/trigger.dev/versions \ + --jq ".[] | select(.metadata.container.tags[] == \"${TAG}\") | .id" \ + | head -1) + + if [ -z "$VERSION_ID" ]; then + echo "Warning: Could not find GHCR version ID for tag ${TAG}, skipping update" + exit 0 + fi + + DOCKER_URL="https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev/${VERSION_ID}?tag=${TAG}" + GENERIC_URL="https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev" + + # Get current release body and replace the generic link with the tag-specific one. + # Use word boundary after GENERIC_URL (closing paren) to avoid matching URLs that + # already have a version ID appended (idempotent on re-runs). + gh release view "${TAG}" --repo triggerdotdev/trigger.dev --json body --jq '.body' > /tmp/release-body.md + sed -i "s|${GENERIC_URL})|${DOCKER_URL})|g" /tmp/release-body.md + + gh release edit "${TAG}" --repo triggerdotdev/trigger.dev --notes-file /tmp/release-body.md + + # Dispatch changelog entry creation to the marketing site repo. + # Runs after update-release so the GitHub release body already has the exact Docker image URL. + dispatch-changelog: + name: ๐Ÿ“ Dispatch changelog PR + needs: [release, update-release] + if: needs.release.outputs.published == 'true' && needs.release.outputs.is_prerelease != 'true' + runs-on: warp-ubuntu-latest-x64-2x + permissions: {} + steps: + - uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 + with: + token: ${{ secrets.CROSS_REPO_PAT }} + repository: triggerdotdev/trigger.dev-site-v3 + event-type: new-release + client-payload: '{"version": "${{ needs.release.outputs.published_package_version }}"}' # The prerelease job needs to be on the same workflow file due to a limitation related to how npm verifies OIDC claims. prerelease: name: ๐Ÿงช Prerelease - runs-on: ubuntu-latest + runs-on: ubuntu-latest # this cannot run on non-GH runner environment: npm-publish permissions: contents: read @@ -141,20 +289,21 @@ jobs: if: github.repository == 'triggerdotdev/trigger.dev' && github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'prerelease' steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.inputs.ref }} + persist-credentials: false - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" # npm v11.5.1 or newer is required for OIDC support @@ -168,10 +317,18 @@ jobs: - name: Generate Prisma Client run: pnpm run generate + - name: Exit changeset pre mode (if active) + run: | + if [ -f .changeset/pre.json ]; then + echo "Repo is in changeset pre mode; exiting so snapshot release can run" + pnpm exec changeset pre exit + fi + - name: Snapshot version - run: pnpm exec changeset version --snapshot ${{ github.event.inputs.prerelease_tag }} + run: pnpm exec changeset version --snapshot "${GITHUB_EVENT_INPUTS_PRERELEASE_TAG}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_INPUTS_PRERELEASE_TAG: ${{ github.event.inputs.prerelease_tag }} - name: Clean run: pnpm run clean --filter "@trigger.dev/*" --filter "trigger.dev" @@ -180,6 +337,7 @@ jobs: run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev" - name: Publish prerelease - run: pnpm exec changeset publish --no-git-tag --snapshot --tag ${{ github.event.inputs.prerelease_tag }} + run: pnpm exec changeset publish --no-git-tag --snapshot --tag "${GITHUB_EVENT_INPUTS_PRERELEASE_TAG}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_INPUTS_PRERELEASE_TAG: ${{ github.event.inputs.prerelease_tag }} diff --git a/.github/workflows/runops-guard.yml b/.github/workflows/runops-guard.yml new file mode 100644 index 0000000000..e496db2a93 --- /dev/null +++ b/.github/workflows/runops-guard.yml @@ -0,0 +1,38 @@ +name: "๐Ÿ›ก๏ธ Run-ops Legacy Guard" + +on: + workflow_call: + +permissions: + contents: read + +jobs: + runops-guard: + runs-on: warp-ubuntu-latest-x64-16x + + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ›ก๏ธ Run-ops legacy guard + run: pnpm --filter webapp run guard:runops-legacy -- --check diff --git a/.github/workflows/sdk-compat.yml b/.github/workflows/sdk-compat.yml new file mode 100644 index 0000000000..ddfb4731db --- /dev/null +++ b/.github/workflows/sdk-compat.yml @@ -0,0 +1,182 @@ +name: "๐Ÿ”Œ SDK Compatibility Tests" + +permissions: + contents: read + +on: + workflow_call: + +jobs: + node-compat: + name: "Node.js ${{ matrix.node }} (${{ matrix.os }})" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [warp-ubuntu-latest-x64-4x] + node: ["20.20", "22.23", "24.18", "26.4"] + + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ matrix.node }} + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ”จ Build SDK dependencies + shell: bash + run: pnpm run build --filter '@trigger.dev/sdk^...' + + - name: ๐Ÿ”จ Build SDK + shell: bash + run: pnpm run build --filter '@trigger.dev/sdk' + + - name: ๐Ÿงช Run SDK Compatibility Tests + shell: bash + run: pnpm --filter @internal/sdk-compat-tests test + + bun-compat: + name: "Bun Runtime" + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐ŸฅŸ Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: latest + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ”จ Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: ๐Ÿ”จ Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: ๐Ÿงช Run Bun Compatibility Test + working-directory: internal-packages/sdk-compat-tests/src/fixtures/bun + run: bun run test.ts + + deno-compat: + name: "Deno Runtime" + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿฆ• Setup Deno + uses: denoland/setup-deno@667a34cdef165d8d2b2e98dde39547c9daac7282 # v2.0.4 + with: + deno-version: v2.x + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ”จ Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: ๐Ÿ”จ Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: ๐Ÿ”— Link node_modules for Deno fixture + working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno + run: ln -s ../../../../../node_modules node_modules + + - name: ๐Ÿงช Run Deno Compatibility Test + working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno + run: deno run --allow-read --allow-env --allow-sys test.ts + + cloudflare-compat: + name: "Cloudflare Workers" + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + + - name: ๐Ÿ”จ Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: ๐Ÿ”จ Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: ๐Ÿ“ฅ Install Cloudflare fixture deps + working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker + run: pnpm install + + - name: ๐Ÿงช Run Cloudflare Workers Compatibility Test (dry-run) + working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker + run: npx wrangler deploy --dry-run --outdir dist diff --git a/.github/workflows/trivy-image.yml b/.github/workflows/trivy-image.yml new file mode 100644 index 0000000000..4c85823f0d --- /dev/null +++ b/.github/workflows/trivy-image.yml @@ -0,0 +1,75 @@ +name: Trivy Image Scan + +# OS-level CVE scan of a published image. Called by the publish pipeline +# (publish.yml) to scan each image right after it's pushed to GHCR โ€” so every +# main build and every release is scanned, not rebuilt. Also runnable ad-hoc +# via workflow_dispatch against any image ref. +# +# Report-only: writes a table to the run summary. No SARIF upload, no gate. +# Library/dependency CVEs are covered by Dependabot, so this is restricted to +# OS packages (`vuln-type: os`) to avoid double-reporting. + +on: + workflow_call: + inputs: + image-ref: + description: "Full image ref to scan (e.g. ghcr.io/triggerdotdev/trigger.dev:main)" + type: string + required: true + workflow_dispatch: + inputs: + image-ref: + description: "Full image ref to scan" + type: string + required: false + default: "ghcr.io/triggerdotdev/trigger.dev:main" + +permissions: {} + +concurrency: + group: trivy-image-${{ inputs.image-ref }} + cancel-in-progress: true + +jobs: + scan: + name: Scan + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + packages: read # pull the image from GHCR + steps: + # Authenticate to GHCR so the scan also works for private images + # (GITHUB_TOKEN isn't forwarded to Docker automatically). Harmless for + # public images. Pairs with the packages: read permission above. + - name: Log in to GitHub Container Registry + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run Trivy image scan + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + scan-type: image + image-ref: ${{ inputs.image-ref }} + # vuln-type maps to --pkg-types: OS packages only (library deps are + # Dependabot's job). ignore-unfixed drops vulns with no patch yet. + vuln-type: os + ignore-unfixed: true + severity: HIGH,CRITICAL + format: table + output: trivy-image.txt + + - name: Job summary + if: always() + env: + IMAGE_REF: ${{ inputs.image-ref }} + run: | + { + echo "## Trivy Image Scan โ€” \`${IMAGE_REF}\`" + echo '```' + # GitHub step summary is capped at 1 MiB; truncate large reports. + head -c 900000 trivy-image.txt 2>/dev/null || echo "(no report produced)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 3eb98e5177..950d5a5e98 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -8,25 +8,34 @@ permissions: jobs: typecheck: - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-16x steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" + - name: Restore Turbo cache + uses: WarpBuilds/cache@40f3443ae7b70e568d6e2070ea897f3df94d7553 # v1 + with: + path: node_modules/.cache/turbo + key: turbo-typecheck-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: | + turbo-typecheck-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile diff --git a/.github/workflows/unit-tests-internal.yml b/.github/workflows/unit-tests-internal.yml index e903e0145a..1a398af9c4 100644 --- a/.github/workflows/unit-tests-internal.yml +++ b/.github/workflows/unit-tests-internal.yml @@ -5,19 +5,23 @@ permissions: on: workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false jobs: unitTests: name: "๐Ÿงช Unit Tests: Internal" - runs-on: ubuntu-latest - strategy: - matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8] - shardTotal: [8] + # Single big machine instead of a 12-job matrix: the internal suites are serial + # (fileParallelism: false) and container-wait-bound, so 12 in-machine shard processes + # fit comfortably in 32 vCPUs while paying the setup cost (install, prisma generate, + # image pulls) once instead of 12 times. + runs-on: warp-ubuntu-latest-x64-32x env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - SHARD_INDEX: ${{ matrix.shardIndex }} - SHARD_TOTAL: ${{ matrix.shardTotal }} + SHARD_TOTAL: 12 steps: - name: ๐Ÿ”ง Disable IPv6 run: | @@ -46,25 +50,26 @@ jobs: run: sudo systemctl restart docker - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - fetch-depth: 0 + fetch-depth: 1 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" # ..to avoid rate limits when pulling images - name: ๐Ÿณ Login to DockerHub if: ${{ env.DOCKERHUB_USERNAME }} - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -72,14 +77,62 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." + - name: ๐Ÿณ Pre-pull testcontainer images + if: ${{ env.DOCKERHUB_USERNAME }} + run: | + # Retry each pull - DockerHub registry timeouts are a recurring transient CI flake. + pull() { + for attempt in 1 2 3; do + docker pull "$1" && return 0 + echo "::warning::docker pull $1 failed (attempt ${attempt}/3); retrying in 10s" + sleep 10 + done + echo "::error::docker pull $1 failed after 3 attempts" + return 1 + } + echo "Pre-pulling Docker images with authenticated session..." + pull postgres:14 + pull postgres:17 # for the heterogeneous PG14/17 test fixture gate (heteroPostgresTest) + pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 + pull redis:7.2 + pull testcontainers/ryuk:0.14.0 + pull electricsql/electric:1.2.4 + echo "Image pre-pull complete" + - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate - - name: ๐Ÿงช Run Internal Unit Tests - run: pnpm run test:internal --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + - name: ๐Ÿ—๏ธ Build test dependencies + # Build once up-front so the parallel shard runs below (turbo --only) never race + # to build or cache-restore the same outputs concurrently. + run: pnpm exec turbo run build --filter "@internal/*..." + + - name: ๐Ÿงช Run Internal Unit Tests (${{ env.SHARD_TOTAL }} in-machine shards) + run: | + # Same shard partitioning as the old 12-job matrix (DurationShardingSequencer + # keys off --shard=i/N), but as parallel local processes. --only skips the + # ^build dependency handled by the step above. + status=0 + declare -a pids + for i in $(seq 1 "$SHARD_TOTAL"); do + pnpm exec turbo run test --only --concurrency=1 --filter "@internal/*" -- \ + --run --reporter=default --reporter=blob --shard="$i/$SHARD_TOTAL" --passWithNoTests \ + > "/tmp/internal-shard-$i.log" 2>&1 & + pids[i]=$! + done + for i in $(seq 1 "$SHARD_TOTAL"); do + if ! wait "${pids[i]}"; then + status=1 + echo "::error::internal unit test shard $i/$SHARD_TOTAL failed" + fi + echo "::group::๐Ÿงช shard $i/$SHARD_TOTAL" + cat "/tmp/internal-shard-$i.log" + echo "::endgroup::" + done + exit "$status" - name: Gather all reports if: ${{ !cancelled() }} @@ -88,43 +141,6 @@ jobs: find . -type f -path '*/.vitest-reports/blob-*.json' \ -exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \; - - name: Upload blob reports to GitHub Actions Artifacts + - name: ๐Ÿ“Š Merge reports if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: internal-blob-report-${{ matrix.shardIndex }} - path: .vitest-reports/* - include-hidden-files: true - retention-days: 1 - - merge-reports: - name: "๐Ÿ“Š Merge Reports" - if: ${{ !cancelled() }} - needs: [unitTests] - runs-on: ubuntu-latest - steps: - - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 10.23.0 - - - name: โŽ” Setup node - uses: buildjet/setup-node@v4 - with: - node-version: 20.19.0 - # no cache enabled, we're not installing deps - - - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@v4 - with: - path: .vitest-reports - pattern: internal-blob-report-* - merge-multiple: true - - - name: Merge reports - run: pnpm dlx vitest@3.1.4 run --merge-reports --pass-with-no-tests + run: pnpm dlx vitest@4.1.7 run --merge-reports --pass-with-no-tests diff --git a/.github/workflows/unit-tests-observability-map.yml b/.github/workflows/unit-tests-observability-map.yml new file mode 100644 index 0000000000..eebf1ef971 --- /dev/null +++ b/.github/workflows/unit-tests-observability-map.yml @@ -0,0 +1,43 @@ +name: "๐Ÿงช Unit Tests: Observability Map" + +permissions: + contents: read + +# Its own workflow rather than a job inside observability-map.yml, because that workflow is not +# reachable from pr_checks.yml's all-checks aggregate and so gates nothing. Called from there +# instead, behind a paths filter, which is how every other test suite in this repo is gated. +on: + workflow_call: + +jobs: + unitTests: + name: "๐Ÿงช Unit Tests: Observability Map" + # No containers and no database: the package is a static analyser over source text, so the + # suite is CPU bound on parsing the route tree and needs nothing the runner does not have. + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: โŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: โŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: ๐Ÿ“ฅ Download deps + run: pnpm install --frozen-lockfile + + # This suite reads apps/webapp/app (the route tree for the scan, the whole app tree for the + # symbol check) and the report workflow's text, which is why the filter that gates this + # workflow watches all of those and not only the routes folder. + - name: ๐Ÿงช Run tests + run: pnpm --filter @internal/observability-map run test diff --git a/.github/workflows/unit-tests-packages.yml b/.github/workflows/unit-tests-packages.yml index d321037703..ceb0b7b49d 100644 --- a/.github/workflows/unit-tests-packages.yml +++ b/.github/workflows/unit-tests-packages.yml @@ -5,15 +5,22 @@ permissions: on: workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false jobs: unitTests: name: "๐Ÿงช Unit Tests: Packages" - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-4x strategy: + # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard + fail-fast: false matrix: - shardIndex: [1] - shardTotal: [1] + shardIndex: [1, 2, 3] + shardTotal: [3] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} @@ -46,25 +53,26 @@ jobs: run: sudo systemctl restart docker - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - fetch-depth: 0 + fetch-depth: 1 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" # ..to avoid rate limits when pulling images - name: ๐Ÿณ Login to DockerHub if: ${{ env.DOCKERHUB_USERNAME }} - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -72,6 +80,28 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." + - name: ๐Ÿณ Pre-pull testcontainer images + if: ${{ env.DOCKERHUB_USERNAME }} + run: | + # Retry each pull - DockerHub registry timeouts are a recurring transient CI flake. + pull() { + for attempt in 1 2 3; do + docker pull "$1" && return 0 + echo "::warning::docker pull $1 failed (attempt ${attempt}/3); retrying in 10s" + sleep 10 + done + echo "::error::docker pull $1 failed after 3 attempts" + return 1 + } + echo "Pre-pulling Docker images with authenticated session..." + pull postgres:14 + pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 + pull redis:7.2 + pull testcontainers/ryuk:0.14.0 + pull electricsql/electric:1.2.4 + pull otel/opentelemetry-collector-k8s:0.158.0@sha256:c09130a633196a5becee164411473a0932ecf223f94fda6dab5f22798ff9f376 + echo "Image pre-pull complete" + - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile @@ -79,7 +109,7 @@ jobs: run: pnpm run generate - name: ๐Ÿงช Run Package Unit Tests - run: pnpm run test:packages --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + run: pnpm run test:packages --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --passWithNoTests - name: Gather all reports if: ${{ !cancelled() }} @@ -90,7 +120,7 @@ jobs: - name: Upload blob reports to GitHub Actions Artifacts if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: packages-blob-report-${{ matrix.shardIndex }} path: .vitest-reports/* @@ -101,30 +131,31 @@ jobs: name: "๐Ÿ“Š Merge Reports" if: ${{ !cancelled() }} needs: [unitTests] - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - fetch-depth: 0 + fetch-depth: 1 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 # no cache enabled, we're not installing deps - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: .vitest-reports pattern: packages-blob-report-* merge-multiple: true - name: Merge reports - run: pnpm dlx vitest@3.1.4 run --merge-reports --pass-with-no-tests + run: pnpm dlx vitest@4.1.7 run --merge-reports --pass-with-no-tests diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index e587bb3891..680eafd9aa 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -5,15 +5,27 @@ permissions: on: workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false jobs: unitTests: name: "๐Ÿงช Unit Tests: Webapp" - runs-on: ubuntu-latest + # 10 shards on 16x machines: webapp test throughput is limited per-machine (one + # docker daemon + disk absorbing all the per-file Postgres/ClickHouse container + # spin-up), so many machines beats few big ones - fewer/bigger (3x32) measured + # SLOWER than 10x8. The 16x (vs 8x) gives the fork pool the CPU headroom the 8x + # runners lacked. Setup overhead per machine is ~1 min on warm runners. + runs-on: warp-ubuntu-latest-x64-16x strategy: + # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard + fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8] - shardTotal: [8] + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + shardTotal: [12] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} @@ -46,25 +58,26 @@ jobs: run: sudo systemctl restart docker - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - fetch-depth: 0 + fetch-depth: 1 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 cache: "pnpm" # ..to avoid rate limits when pulling images - name: ๐Ÿณ Login to DockerHub if: ${{ env.DOCKERHUB_USERNAME }} - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -72,6 +85,28 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." + - name: ๐Ÿณ Pre-pull testcontainer images + if: ${{ env.DOCKERHUB_USERNAME }} + run: | + # Retry each pull - DockerHub registry timeouts are a recurring transient CI flake. + pull() { + for attempt in 1 2 3; do + docker pull "$1" && return 0 + echo "::warning::docker pull $1 failed (attempt ${attempt}/3); retrying in 10s" + sleep 10 + done + echo "::error::docker pull $1 failed after 3 attempts" + return 1 + } + echo "Pre-pulling Docker images with authenticated session..." + pull postgres:14 + pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 + pull redis:7.2 + pull testcontainers/ryuk:0.14.0 + pull electricsql/electric:1.2.4 + pull minio/minio:latest + echo "Image pre-pull complete" + - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile @@ -79,7 +114,7 @@ jobs: run: pnpm run generate - name: ๐Ÿงช Run Webapp Unit Tests - run: pnpm run test:webapp --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + run: pnpm run test:webapp --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --passWithNoTests env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres @@ -98,7 +133,7 @@ jobs: - name: Upload blob reports to GitHub Actions Artifacts if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: webapp-blob-report-${{ matrix.shardIndex }} path: .vitest-reports/* @@ -109,30 +144,31 @@ jobs: name: "๐Ÿ“Š Merge Reports" if: ${{ !cancelled() }} needs: [unitTests] - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-2x steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - fetch-depth: 0 + fetch-depth: 1 + persist-credentials: false - name: โŽ” Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10.23.0 + version: 10.33.2 - name: โŽ” Setup node - uses: buildjet/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20.19.0 + node-version: 24.18.0 # no cache enabled, we're not installing deps - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: .vitest-reports pattern: webapp-blob-report-* merge-multiple: true - name: Merge reports - run: pnpm dlx vitest@3.1.4 run --merge-reports --pass-with-no-tests + run: pnpm dlx vitest@4.1.7 run --merge-reports --pass-with-no-tests diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7c90a5a30a..96e76279c8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -5,14 +5,30 @@ permissions: on: workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false jobs: webapp: uses: ./.github/workflows/unit-tests-webapp.yml - secrets: inherit + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + e2e-webapp: + uses: ./.github/workflows/e2e-webapp.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} packages: uses: ./.github/workflows/unit-tests-packages.yml - secrets: inherit + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} internal: uses: ./.github/workflows/unit-tests-internal.yml - secrets: inherit + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/vouch-check-pr.yml b/.github/workflows/vouch-check-pr.yml new file mode 100644 index 0000000000..fab876451b --- /dev/null +++ b/.github/workflows/vouch-check-pr.yml @@ -0,0 +1,50 @@ +name: Vouch - Check PR + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] needed to comment/close fork PRs; safe because we never check out PR HEAD ref so no fork-controlled code runs + types: [opened, reopened] + +permissions: {} + +jobs: + check-vouch: + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + pull-requests: write # auto-close unvouched PRs + issues: read + steps: + - uses: mitchellh/vouch/action/check-pr@c6d80ead49839655b61b422700b7a3bc9d0804a9 # v1.4.2 + with: + pr-number: ${{ github.event.pull_request.number }} + auto-close: true + require-vouch: true + env: + GH_TOKEN: ${{ github.token }} + + require-draft: + needs: check-vouch + permissions: + pull-requests: write # close non-draft PRs with a comment + if: > + github.event.pull_request.draft == false && + github.event.pull_request.author_association != 'MEMBER' && + github.event.pull_request.author_association != 'OWNER' && + github.event.pull_request.author_association != 'COLLABORATOR' && + github.event.pull_request.user.login != 'devin-ai-integration[bot]' && + github.event.pull_request.user.login != 'dependabot[bot]' && + github.event.pull_request.user.login != 'github-actions[bot]' + runs-on: warp-ubuntu-latest-x64-2x + steps: + - name: Close non-draft PR + env: + GH_TOKEN: ${{ github.token }} + run: | + STATE=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json state -q '.state') + if [ "$STATE" != "OPEN" ]; then + echo "PR is already closed, skipping." + exit 0 + fi + gh pr close ${{ github.event.pull_request.number }} \ + --repo ${{ github.repository }} \ + --comment "Thanks for your contribution! We require all external PRs to be opened in **draft** status first so you can address CodeRabbit review comments and ensure CI passes before requesting a review. Please re-open this PR as a draft. See [CONTRIBUTING.md](https://github.com/${{ github.repository }}/blob/main/CONTRIBUTING.md#pr-workflow) for details." diff --git a/.github/workflows/vouch-manage-by-issue.yml b/.github/workflows/vouch-manage-by-issue.yml new file mode 100644 index 0000000000..29d9584cab --- /dev/null +++ b/.github/workflows/vouch-manage-by-issue.yml @@ -0,0 +1,24 @@ +name: Vouch - Manage by Issue + +on: + issue_comment: + types: [created] + +permissions: + contents: write + issues: write + +jobs: + manage: + runs-on: warp-ubuntu-latest-x64-2x + if: >- + contains(github.event.comment.body, 'vouch') || + contains(github.event.comment.body, 'denounce') || + contains(github.event.comment.body, 'unvouch') + steps: + - uses: mitchellh/vouch/action/manage-by-issue@c6d80ead49839655b61b422700b7a3bc9d0804a9 # v1.4.2 + with: + comment-id: ${{ github.event.comment.id }} + issue-id: ${{ github.event.issue.number }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/workflow-checks.yml b/.github/workflows/workflow-checks.yml new file mode 100644 index 0000000000..40843deb2e --- /dev/null +++ b/.github/workflows/workflow-checks.yml @@ -0,0 +1,58 @@ +name: Workflow Checks + +on: + push: + branches: [main] + paths: + - ".github/workflows/**" + - ".github/actions/**" + - ".github/zizmor.yml" + - ".github/actionlint.yaml" + pull_request: + paths: + - ".github/workflows/**" + - ".github/actions/**" + - ".github/zizmor.yml" + - ".github/actionlint.yaml" + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + actionlint: + name: Actionlint + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Run actionlint + uses: docker://rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 + + zizmor: + name: Zizmor + # Uploads SARIF to the Security tab, which requires GitHub code scanning to be + # enabled on the repository. Set the ENABLE_WORKFLOW_SECURITY_SCAN repository + # variable to 'false' to skip this job where code scanning isn't available; + # leave it unset (the default) to run the scan. + if: ${{ vars.ENABLE_WORKFLOW_SECURITY_SCAN != 'false' }} + runs-on: warp-ubuntu-latest-x64-2x + permissions: + security-events: write # Upload SARIF to GitHub Security tab + contents: read # Read workflow files for analysis + actions: read # Read workflow run metadata + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 0000000000..e7920e8cf1 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,5 @@ +rules: + unpinned-uses: + config: + policies: + "*": hash-pin diff --git a/.gitignore b/.gitignore index 8267c9fbab..b11dded2b0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ out/ dist packages/**/dist +# vendored bundles (generated during build) +packages/**/src/**/vendor + # Tailwind apps/**/styles/tailwind.css packages/**/styles/tailwind.css @@ -61,6 +64,26 @@ apps/**/public/build /packages/core/src/package.json /packages/trigger-sdk/src/package.json /packages/python/src/package.json -.claude +**/.claude/settings.local.json +.claude/architecture/ +.claude/docs-plans/ +.claude/plans/ +.claude/review-guides/ +.claude/scheduled_tasks.lock .mcp.log -.cursor/debug.log \ No newline at end of file +.mcp.json +.cursor/debug.log +ailogger-output.log +# per-package vitest timing capture (transient; merged into root test-timings.json) +.vitest-timing.json + +# local git worktree checkouts (not source) โ€” keeps oxfmt/oxlint from descending into them +.worktrees/ + +# local planning/design docs, not committed +**/docs/superpowers/ + +# observability-map CLI output artifact, not committed +observability-map.json + +.claude/worktrees/ diff --git a/.nvmrc b/.nvmrc index 3bf34c2761..5bcf9c6e6a 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20.19.0 \ No newline at end of file +v24.18.0 diff --git a/.oxfmtrc.json b/.oxfmtrc.json new file mode 100644 index 0000000000..59f46ac1ed --- /dev/null +++ b/.oxfmtrc.json @@ -0,0 +1,33 @@ +{ + "$schema": "./node_modules/oxfmt/configuration_schema.json", + "semi": true, + "singleQuote": false, + "jsxSingleQuote": false, + "trailingComma": "es5", + "bracketSpacing": true, + "bracketSameLine": false, + "printWidth": 100, + "tabWidth": 2, + "useTabs": false, + "sortPackageJson": false, + "ignorePatterns": [ + "node_modules", + ".env", + ".env.local", + "pnpm-lock.yaml", + "tailwind.css", + ".babelrc.json", + "**/.react-email/", + "**/storybook-static/", + "**/.changeset/", + "**/dist/", + "**/.worktrees/", + "internal-packages/tsql/src/grammar/", + "internal-packages/llm-model-catalog/src/defaultPrices.ts", + "internal-packages/llm-model-catalog/src/modelCatalog.ts", + // Maybe turn these on in the future + "**/*.yaml", + "**/*.mdx", + "**/*.md" + ] +} diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 0000000000..c51e04730c --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,77 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "categories": { + "correctness": "error" + }, + "plugins": ["typescript", "import", "react"], + "jsPlugins": [ + "./oxlint-plugins/no-thrown-unawaited-redirect.mjs", + "./oxlint-plugins/runops-residency.mjs", + "./oxlint-plugins/prisma-in-filter.mjs" + ], + "ignorePatterns": [ + "**/dist/**", + "**/build/**", + "**/.worktrees/**", + "**/*.d.ts", + "**/seed.js", + "**/seedCloud.ts", + "**/populate.js", + "internal-packages/tsql/src/grammar/" + ], + "rules": { + "no-unused-vars": [ + "error", + { + "args": "none", + "argsIgnorePattern": "^_", + "varsIgnorePattern": "^_", + "caughtErrors": "all", + "caughtErrorsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], + "no-empty-pattern": "off", + "no-control-regex": "off", + "typescript/no-non-null-asserted-optional-chain": "off", + "no-unused-expressions": [ + "error", + { + "allowShortCircuit": true, + "allowTernary": true + } + ], + "typescript/consistent-type-imports": "error", + "import/no-duplicates": "error", + "import/namespace": "off", + "react-hooks/exhaustive-deps": "off", + "react-hooks/rules-of-hooks": "off", + "trigger/no-thrown-unawaited-redirect": "error", + "trigger-prisma/no-unbounded-list-filter": "error", + "trigger-prisma/no-unbounded-list-filter-in-args-helper": "error" + }, + "overrides": [ + { + "files": ["apps/webapp/app/**/*.ts", "apps/webapp/app/**/*.tsx"], + "rules": { + "trigger-runops/no-control-plane-run-graph-access": "error", + "trigger-runops/no-control-plane-in-runops-slot": "error" + } + }, + { + "files": ["apps/webapp/app/**/*.test.ts", "apps/webapp/app/**/*.test.tsx"], + "rules": { + "trigger-runops/no-control-plane-run-graph-access": "off", + "trigger-runops/no-control-plane-in-runops-slot": "off" + } + }, + { + "files": ["**/*.test.ts", "**/*.test.tsx", "**/test/**", "**/tests/**", "**/e2e/**"], + "rules": { + "trigger-prisma/no-unbounded-list-filter": "off", + "trigger-prisma/no-unbounded-list-filter-in-args-helper": "off" + } + } + ] +} diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index a34447dd45..0000000000 --- a/.prettierignore +++ /dev/null @@ -1,10 +0,0 @@ -node_modules -.env -.env.local -pnpm-lock.yaml -tailwind.css -.babelrc.json -**/.react-email/ -**/storybook-static/ -**/.changeset/ -**/dist/ \ No newline at end of file diff --git a/.server-changes/.gitkeep b/.server-changes/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.server-changes/README.md b/.server-changes/README.md new file mode 100644 index 0000000000..e13ebec4a7 --- /dev/null +++ b/.server-changes/README.md @@ -0,0 +1,97 @@ +# Server Changes + +This directory tracks changes to server-only components (webapp, supervisor, etc.) that are not captured by changesets. Changesets only track published npm packages โ€” server changes would otherwise go undocumented. + +## When to add a file + +These entries are **user-facing release notes**, not a catalog of every change. The test is "would a user or customer care about this change?", not "did I touch a server app?". Add one only when a server-only change is something a user would notice, act on, or want to hear about (a feature, a bug fix they could have hit, a behavior or performance change they would feel). Skip it for internal-only or admin-only changes, refactors, test-only changes, chores, and performance tuning with no user-visible effect. Anyone who wants the exact history reads the commits. When in doubt, ask a maintainer rather than adding a note by default. + +**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, or other server components (and does NOT change anything in `packages/`) AND the change is user-facing, add a `.server-changes/` file. + +**Mixed PRs** (both packages and server): the changeset covers it, so no `.server-changes/` file is needed. If the package change is internal and needs no changeset but the server change is user-facing, add a `.server-changes/` file for it. + +**Package-only PRs**: Just add a changeset as usual, when the change is user-facing. + +## File format + +Create a markdown file with a descriptive name: + +``` +.server-changes/fix-batch-queue-stalls.md +``` + +With this format: + +```markdown +--- +area: webapp +type: fix +--- + +Speed up batch queue processing by removing stalls and fixing retry race +``` + +### Fields + +- **area** (required): `webapp` | `supervisor` +- **type** (required): `feature` | `fix` | `improvement` | `breaking` + +### Description + +The body text (below the frontmatter) is a one-line description of the change. Keep it concise โ€” it will appear in release notes. + +### Writing guidance + +These entries are public-facing - they ship verbatim in user-visible release notes. A few rules to keep them clean: + +- **Write for the user, not the reviewer.** Lead with what the user notices or has to do. If a reader who doesn't know the codebase can't tell what changed for them, rewrite it. +- **One sentence is usually enough.** The body is the bullet in the changelog. If you need a paragraph, you're probably describing the implementation rather than the change. +- **Describe behavior, not implementation.** Skip internal scopes, middleware names, library specifics, framework internals. Users care about what's different for them, not how it's wired. +- **Never name internal tools or infra.** Observability stacks, internal services, infra components, monitoring backends, CI surfaces, AWS specifics - none of these belong in user-facing notes. + +Before / after: + +- โŒ _"The image verification step now parses the manifest's layer media types and returns a new result the finalizer rejects."_ (describes the wiring; a user can't act on it) +- โœ… _"Deploying with an outdated CLI could produce an image that fails to start on every run. These deploys are now stopped before going live, with a message asking you to upgrade the CLI and re-deploy."_ (what the user sees and does) + +## Lifecycle + +1. Engineer adds a `.server-changes/` file in their PR +2. Files accumulate on `main` as PRs merge +3. The changeset release PR includes these in its summary +4. After the release merges, CI cleans up the consumed files + +## Examples + +**New feature:** + +```markdown +--- +area: webapp +type: feature +--- + +TRQL query language and the Query page +``` + +**Bug fix:** + +```markdown +--- +area: webapp +type: fix +--- + +Fix schedule limit counting for orgs with custom limits +``` + +**Improvement:** + +```markdown +--- +area: webapp +type: improvement +--- + +Use the replica for API auth queries to reduce primary load +``` diff --git a/.server-changes/bound-env-layout-environment-load.md b/.server-changes/bound-env-layout-environment-load.md new file mode 100644 index 0000000000..5c658a99e5 --- /dev/null +++ b/.server-changes/bound-env-layout-environment-load.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Dashboard pages load faster on projects with many preview branches by no longer loading every environment on each page. diff --git a/.server-changes/dequeue-worker-version-freshness.md b/.server-changes/dequeue-worker-version-freshness.md new file mode 100644 index 0000000000..d584ef2c56 --- /dev/null +++ b/.server-changes/dequeue-worker-version-freshness.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed a brief window after promoting or rolling back a deployment where newly triggered runs could still execute on the previous version. New runs now pick up the current version immediately. diff --git a/.server-changes/hide-root-api-key-creation-date.md b/.server-changes/hide-root-api-key-creation-date.md new file mode 100644 index 0000000000..311febd2fb --- /dev/null +++ b/.server-changes/hide-root-api-key-creation-date.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Root API keys no longer show an environment creation timestamp as their creation date. diff --git a/.server-changes/org-settings-app-version.md b/.server-changes/org-settings-app-version.md new file mode 100644 index 0000000000..43ae2cc5c3 --- /dev/null +++ b/.server-changes/org-settings-app-version.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The app version shown on the organization settings page now reports the real version instead of v0.0.0. diff --git a/.server-changes/paused-environment-stays-paused-after-deploy.md b/.server-changes/paused-environment-stays-paused-after-deploy.md new file mode 100644 index 0000000000..4c35b6e215 --- /dev/null +++ b/.server-changes/paused-environment-stays-paused-after-deploy.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fix paused environments starting to run work again after a deploy: a paused environment now stays paused until you resume it. diff --git a/.server-changes/settings-back-to-app-org.md b/.server-changes/settings-back-to-app-org.md new file mode 100644 index 0000000000..64464505ee --- /dev/null +++ b/.server-changes/settings-back-to-app-org.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The "Back to app" button in organization settings now returns you to that organization instead of your most recently used one. diff --git a/.server-changes/transaction-resilience-during-db-interruptions.md b/.server-changes/transaction-resilience-during-db-interruptions.md new file mode 100644 index 0000000000..c05bfd5f22 --- /dev/null +++ b/.server-changes/transaction-resilience-during-db-interruptions.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Triggering tasks is now more resilient to brief, transient service interruptions, so short stalls are less likely to surface as errors. diff --git a/.vouch.yml b/.vouch.yml new file mode 100644 index 0000000000..ec6e85aa70 --- /dev/null +++ b/.vouch.yml @@ -0,0 +1,4 @@ +vouch: + - github: edosrecki + - github: GautamBytes + - github: ConProgramming diff --git a/.vscode/launch.json b/.vscode/launch.json index d135aa70a2..1044443e19 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,84 +31,21 @@ "cwd": "${workspaceFolder}/apps/webapp", "sourceMaps": true }, - { - "type": "chrome", - "request": "launch", - "name": "Chrome webapp", - "url": "http://localhost:3030", - "webRoot": "${workspaceFolder}/apps/webapp/app" - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug V3 init CLI", - "command": "pnpm exec trigger init", - "cwd": "${workspaceFolder}/references/init-shell", - "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug V3 init dev CLI", - "command": "pnpm exec trigger dev", - "cwd": "${workspaceFolder}/references/init-shell", - "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug V3 Dev CLI", - "command": "pnpm exec trigger dev", - "cwd": "${workspaceFolder}/references/hello-world", - "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug Dev Next.js Realtime", - "command": "pnpm exec trigger dev", - "cwd": "${workspaceFolder}/references/nextjs-realtime", - "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug prisma-catalog deploy CLI", - "command": "pnpm exec trigger deploy --self-hosted --load-image", - "cwd": "${workspaceFolder}/references/prisma-catalog", - "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug V3 Deploy CLI", - "command": "pnpm exec trigger deploy --self-hosted --load-image", - "cwd": "${workspaceFolder}/references/hello-world", - "sourceMaps": true - }, { "type": "node-terminal", "request": "launch", - "name": "Debug V3 list-profiles CLI", - "command": "pnpm exec trigger list-profiles --log-level debug", - "cwd": "${workspaceFolder}/references/hello-world", - "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug V3 update CLI", - "command": "pnpm exec trigger update", - "cwd": "${workspaceFolder}/references/hello-world", + "name": "Debug opened test file", + "command": "pnpm run test -- ./${relativeFile}", + "envFile": "${workspaceFolder}/.env", + "cwd": "${workspaceFolder}", "sourceMaps": true }, { - "type": "node-terminal", + "type": "chrome", "request": "launch", - "name": "Debug V3 Management", - "command": "pnpm run management", - "cwd": "${workspaceFolder}/references/hello-world", - "sourceMaps": true + "name": "Chrome webapp", + "url": "http://localhost:3030", + "webRoot": "${workspaceFolder}/apps/webapp/app" }, { "type": "node", @@ -126,14 +63,6 @@ "cwd": "${workspaceFolder}/packages/cli-v3", "sourceMaps": true }, - { - "type": "node-terminal", - "request": "launch", - "name": "debug v3 hello-world dev", - "command": "pnpm exec trigger dev", - "cwd": "${workspaceFolder}/references/hello-world", - "sourceMaps": true - }, { "type": "node-terminal", "request": "launch", @@ -149,14 +78,6 @@ "command": "pnpm run test ./src/run-queue/index.test.ts --run", "cwd": "${workspaceFolder}/internal-packages/run-engine", "sourceMaps": true - }, - { - "type": "node-terminal", - "request": "launch", - "name": "Debug d3-demo", - "command": "pnpm exec trigger dev", - "cwd": "${workspaceFolder}/references/d3-demo", - "sourceMaps": true } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 12aefeb358..f969bb6d5d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"], + "deno.enablePaths": ["runtime_tests/tests/deno"], "debug.toolBarLocation": "commandCenter", "typescript.tsdk": "node_modules/typescript/lib", "search.exclude": { @@ -7,5 +7,5 @@ "packages/cli-v3/e2e": true }, "vitest.disableWorkspaceWarning": true, - "typescript.experimental.useTsgo": false + "chat.agent.maxRequests": 10000 } diff --git a/AGENTS.md b/AGENTS.md index 846c6d827c..2a2dea78b5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,68 +1,289 @@ -# Guidance for Coding Agents - -This repository is a pnpm monorepo managed with Turbo. It contains multiple apps and packages that make up the Trigger.dev platform and SDK. - -## Repository layout -- `apps/webapp` โ€“ Remix application that serves as the main API and dashboard. -- `apps/supervisor` โ€“ Node application for executing built tasks. -- `packages/*` โ€“ Published packages such as `@trigger.dev/sdk`, the CLI (`trigger.dev`), and shared libraries. -- `internal-packages/*` โ€“ Internal-only packages used by the webapp and other apps. -- `references/*` โ€“ Example projects for manual testing and development of new features. -- `ai/references` โ€“ Contains additional documentation including an overview (`repo.md`) and testing guidelines (`tests.md`). - -See `ai/references/repo.md` for a more complete explanation of the workspaces. - -## Development setup -1. Install dependencies with `pnpm i` (pnpm `10.23.0` and Node.js `20.11.1` are required). -2. Copy `.env.example` to `.env` and generate a random 16 byte hex string for `ENCRYPTION_KEY` (`openssl rand -hex 16`). Update other secrets if needed. -3. Start the local services with Docker: - ```bash - pnpm run docker - ``` -4. Run database migrations: - ```bash - pnpm run db:migrate - ``` -5. Build the webapp, CLI and SDK packages: - ```bash - pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk - ``` -6. Launch the development server: - ```bash - pnpm run dev --filter webapp - ``` - The webapp runs on . - -For full setup instructions see `CONTRIBUTING.md`. - -## Running tests -- Unit tests use **vitest**. Run all tests: - ```bash - pnpm run test - ``` -- Run tests for a specific workspace (example for `webapp`): - ```bash - pnpm run test --filter webapp - ``` -- Prefer running a single test file from within its directory: - ```bash - cd apps/webapp - pnpm run test ./src/components/Button.test.ts - ``` - If packages in that workspace need to be built first, run `pnpm run build --filter webapp`. - -Refer to `ai/references/tests.md` for details on writing tests. Tests should avoid mocks or stubs and use the helpers from `@internal/testcontainers` when Redis or Postgres are needed. - -## Coding style -- Formatting is enforced using Prettier. Run `pnpm run format` before committing. -- Follow the existing project conventions. Test files live beside the files under test and use descriptive `describe` and `it` blocks. -- Do not commit directly to the `main` branch. All changes should be made in a separate branch and go through a pull request. - -## Additional docs -- The root `README.md` describes Trigger.dev and links to documentation. -- The `docs` workspace contains our documentation site, which can be run locally with: - ```bash - pnpm run dev --filter docs - ``` -- `references/README.md` explains how to create new reference projects for manual testing. +# AGENTS.md +This file provides guidance to Claude Code when working with this repository. Subdirectory CLAUDE.md files provide deeper context when you navigate into specific areas. + +## Build and Development Commands + +This is a pnpm 10.33.2 monorepo using Turborepo. Run commands from root with `pnpm run`. + +**Adding dependencies:** Edit `package.json` directly instead of using `pnpm add`, then run `pnpm i` from the repo root. See `.claude/rules/package-installation.md` for the full process. + +```bash +pnpm run docker # Core dev services (Postgres, Redis, Electric, MinIO, ClickHouse, s2-lite) +# pnpm run docker:full # Same + observability stack (Prometheus, Grafana, OTEL) and chaos tooling +pnpm run db:migrate # Run database migrations +pnpm run db:seed # Seed the database (required for reference projects) + +# Build packages (required before running) +pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk + +pnpm run dev --filter webapp # Run webapp (http://localhost:3030) +pnpm run dev --filter trigger.dev --filter "@trigger.dev/*" # Watch CLI and packages +``` + +### Verifying Changes + +The verification command depends on where the change lives: + +- **Apps and internal packages** (`apps/*`, `internal-packages/*`): Use `typecheck`. **Never use `build`** for these โ€” building proves almost nothing about correctness. +- **Public packages** (`packages/*`): Use `build`. + +```bash +# Apps and internal packages โ€” use typecheck +pnpm run typecheck --filter webapp # ~1-2 minutes +pnpm run typecheck --filter @internal/run-engine + +# Public packages โ€” use build +pnpm run build --filter @trigger.dev/sdk +pnpm run build --filter @trigger.dev/core +``` + +Only run typecheck/build after major changes (new files, significant refactors, schema changes). For small edits, trust the types and let CI catch issues. + +## Testing + +We use vitest exclusively. **Never mock anything** - use testcontainers instead. + +```bash +pnpm run test --filter webapp # All tests for a package +cd internal-packages/run-engine +pnpm run test ./src/engine/tests/ttl.test.ts --run # Single test file +pnpm run build --filter @internal/run-engine # May need to build deps first +``` + +Test files go next to source files (e.g., `MyService.ts` -> `MyService.test.ts`). + +### Testcontainers for Redis/PostgreSQL + +```typescript +import { redisTest, postgresTest, containerTest } from "@internal/testcontainers"; + +redisTest("should use redis", async ({ redisOptions }) => { + /* ... */ +}); +postgresTest("should use postgres", async ({ prisma }) => { + /* ... */ +}); +containerTest("should use both", async ({ prisma, redisOptions }) => { + /* ... */ +}); +``` + +## Code Style + +### Formatting and linting + +Format and lint are enforced by CI (`code-quality` check). Run before committing: + +```bash +pnpm run format # oxfmt โ€” auto-fixes formatting +pnpm run lint:fix # oxlint โ€” auto-fixes lint violations +pnpm run lint # oxlint โ€” check only (no fixes) +``` + +### Imports + +**Prefer static imports over dynamic imports.** Only use dynamic `import()` when: +- Circular dependencies cannot be resolved otherwise +- Code splitting is genuinely needed for performance +- The module must be loaded conditionally at runtime + +Dynamic imports add unnecessary overhead in hot paths and make code harder to analyze. If you find yourself using `await import()`, ask if a regular `import` statement would work instead. + +## Changesets and Server Changes + +Changesets and `.server-changes/` files are **user-facing release notes**. They ship verbatim into the changelog that customers read to decide what to upgrade for or pay attention to. They are not a catalog of every change: anyone who wants the exact history reads the commits. So the question is not "did I touch a public package or a server app?" but **"would a user or customer care about this change?"** + +**Add one** when the change is something a user would notice, act on, or want to hear about: a new feature, a bug fix they could have hit, a behavior or performance change they would feel, a breaking change. + +**Skip it** (no changeset, no `.server-changes/` file) when the change is not worth communicating to users, even if it touches a public package or a server app. For example: + +- internal-only or admin-only changes, refactors, test-only changes, chores +- performance or query tuning with no user-visible behavior change +- changes to a public package that is not consumed independently (e.g. `@trigger.dev/redis-worker`), where a version bump means nothing to a user + +When in doubt, ask a maintainer rather than adding a note by default. An unnecessary entry is noise in the changelog, not a safe default. + +### How to add one + +When a **public package** (`packages/*` or `integrations/*`) change is user-facing, add a changeset: + +```bash +pnpm run changeset:add +``` + +- Default to **patch** for bug fixes and minor changes +- Confirm with maintainers before selecting **minor** (new features) +- **Never** select major without explicit approval + +When a **server-only** change (`apps/webapp/`, `apps/supervisor/`, etc., with no package changes) is user-facing, add a `.server-changes/` file instead. See `.server-changes/README.md` for format and documentation. + +**Write the description for users, not maintainers.** Both changesets and `.server-changes/` notes ship verbatim in user-visible release notes. Lead with what changed *for the user*: one plain sentence describing behavior, not implementation, and never naming internal tools or infra. The full writing guidance in `.server-changes/README.md` applies to changesets too. + +## Dependency Pinning + +Zod is pinned to a single version across the entire monorepo (currently `3.25.76`). When adding zod to a new or existing package, use the **exact same version** as the rest of the repo - never a different version or a range. Mismatched zod versions cause runtime type incompatibilities (e.g., schemas from one package can't be used as body validators in another). + +## Architecture Overview + +### Request Flow + +User API call -> Webapp routes -> Services -> RunEngine -> Redis Queue -> Supervisor -> Container execution -> Results back through RunEngine -> ClickHouse (analytics) + PostgreSQL (state) + +### Apps + +- **apps/webapp**: Remix 2.17.4 app - main API, dashboard, orchestration. Uses Express server. +- **apps/supervisor**: Manages task execution containers (Docker/Kubernetes). + +### Public Packages + +- **packages/trigger-sdk** (`@trigger.dev/sdk`): Main SDK for writing tasks +- **packages/cli-v3** (`trigger.dev`): CLI - also bundles code that goes into customer task images +- **packages/core** (`@trigger.dev/core`): Shared types. **Import subpaths only** (never root). +- **packages/build** (`@trigger.dev/build`): Build extensions and types +- **packages/react-hooks**: React hooks for realtime and triggering +- **packages/redis-worker** (`@trigger.dev/redis-worker`): Redis-based background job system + +### Internal Packages + +- **internal-packages/database**: Prisma 6.14.0 client and schema (PostgreSQL) +- **internal-packages/clickhouse**: ClickHouse client, schema migrations, analytics queries +- **internal-packages/run-engine**: "Run Engine 2.0" - core run lifecycle management +- **internal-packages/redis**: Redis client creation utilities (ioredis) +- **internal-packages/testcontainers**: Test helpers for Redis/PostgreSQL containers +- **internal-packages/schedule-engine**: Durable cron scheduling + +### v3 (engine V1) removed + +v3 (engine V1: MarQS + Graphile worker) is end-of-life and its execution code has been removed. The `apps/webapp/app/v3/` directory name is historical - everything there now serves V2 (Run Engine 2.0, `@internal/run-engine` + redis-worker). There is no V1 execution path: a `RunEngineVersion` `V1` branch only rejects or finalizes gracefully so v3 clients get a clean 4xx, never a 5xx. Do not reintroduce V1. See `apps/webapp/CLAUDE.md` and `.claude/rules/legacy-v3-code.md`. + +### Documentation + +Docs live in `docs/` as a Mintlify site (MDX format). See `docs/CLAUDE.md` for conventions. + +### Reference Projects + +Reference/example projects for testing SDK and platform features live in a separate repo: [`triggerdotdev/references`](https://github.com/triggerdotdev/references). Clone it alongside this repo and use its `projects/hello-world` to manually test changes before submitting PRs. See that repo's README for setup and linking to a local monorepo build. + +## Docker Image Guidelines + +When updating Docker image references: + +- **Always use multiplatform/index digests**, not architecture-specific digests +- Architecture-specific digests cause CI failures on different build environments +- Use the digest from the main Docker Hub page, not from a specific OS/ARCH variant + +## Writing Trigger.dev Tasks + +Always import from `@trigger.dev/sdk`. Never use `@trigger.dev/sdk/v3` or deprecated `client.defineJob`. + +```typescript +import { task } from "@trigger.dev/sdk"; + +export const myTask = task({ + id: "my-task", + run: async (payload: { message: string }) => { + // Task logic + }, +}); +``` + +### SDK Documentation Rules + +The `rules/` directory contains versioned SDK documentation distributed via the SDK installer. Current version: `rules/manifest.json`. Do NOT update `rules/` or `.claude/skills/trigger-dev-tasks/` unless explicitly asked - these are maintained in separate dedicated passes. + +## Testing with the hello-world Reference Project + +The reference projects live in the separate [`triggerdotdev/references`](https://github.com/triggerdotdev/references) repo - clone it alongside this repo. + +First-time setup: + +1. `pnpm run db:seed` to seed the database (creates the References org + hello-world project) +2. Build the CLI/packages you want to test: `pnpm run build --filter trigger.dev` +3. In your `references` clone, follow its README to link to your local monorepo build, then authorize: `cd projects/hello-world && pnpm exec trigger login -a http://localhost:3030` + +Running (from your `references` clone): `cd projects/hello-world && pnpm exec trigger dev` + +## Local Task Testing Workflow + +### Step 1: Start Webapp in Background + +```bash +# Run from repo root with run_in_background: true +pnpm run dev --filter webapp +curl -s http://localhost:3030/healthcheck # Verify running +``` + +### Step 2: Start Trigger Dev in Background + +```bash +# in your triggerdotdev/references clone +cd projects/hello-world && pnpm exec trigger dev +# Wait for "Local worker ready [node]" +``` + +### Step 3: Trigger and Monitor Tasks via MCP + +``` +mcp__trigger__get_current_worker(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev") +mcp__trigger__trigger_task(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev", taskId: "hello-world", payload: {"message": "Hello"}) +mcp__trigger__list_runs(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev", taskIdentifier: "hello-world", limit: 5) +``` + +Dashboard: http://localhost:3030/orgs/references-9dfd/projects/hello-world-97DT/env/dev/runs + + + +# Skill mappings โ€” when working in these areas, load the linked skill file into context. + +skills: + +- task: "Using agentcrumbs for debug tracing, adding crumbs, trails, markers, querying traces, or stripping debug code before merge" + load: "node_modules/agentcrumbs/skills/agentcrumbs/SKILL.md" +- task: "Setting up agentcrumbs in the project, initializing namespace catalog, running crumbs init" + load: "node_modules/agentcrumbs/skills/agentcrumbs/init/SKILL.md" + + +## agentcrumbs + +Add crumbs as you write code โ€” not just when debugging. Mark lines with +`// @crumbs` or wrap blocks in `// #region @crumbs`. They stay on the +branch throughout development and are stripped by `agentcrumbs strip` +before merge. + +### Namespaces + +| Namespace | Description | Path | +| ----------------- | ---------------------------------------------- | ----------------------------------- | +| `webapp` | Main Remix app โ€” API, dashboard, orchestration | `apps/webapp` | +| `supervisor` | Task execution container manager | `apps/supervisor` | +| `sdk` | Main SDK for writing tasks | `packages/trigger-sdk` | +| `cli` | CLI and task image bundling | `packages/cli-v3` | +| `core` | Shared types and subpath exports | `packages/core` | +| `build` | Build extensions and types | `packages/build` | +| `react-hooks` | React hooks for realtime/triggering | `packages/react-hooks` | +| `redis-worker` | Redis-based background job system | `packages/redis-worker` | +| `run-engine` | Run Engine 2.0 โ€” core run lifecycle | `internal-packages/run-engine` | +| `database` | Prisma client and schema | `internal-packages/database` | +| `clickhouse` | ClickHouse client, migrations, analytics | `internal-packages/clickhouse` | +| `schedule-engine` | Durable cron scheduling | `internal-packages/schedule-engine` | +| `run-queue` | Run queue management | `internal-packages/run-queue` | + +Do not invent new namespaces โ€” pick from this table or ask first. + +### For PR reviewers + +Do not flag `// @crumbs` markers, `#region @crumbs` blocks, or agentcrumbs +imports in reviews. These are temporary debug instrumentation stripped +before merge. Data logged in crumbs (IDs, names, values) never reaches +production. + +### CLI + +```bash +pnpm exec agentcrumbs collect # start collector (multi-service) +pnpm exec agentcrumbs tail --app trigger # live tail +pnpm exec agentcrumbs clear --app trigger # remove crumbs before merge +``` + +The preferred way to query for crumbs is to use `pnpm exec agentcrumbs query --app trigger` with the `--limit` option and cursor pagination, and clear existing crumbs before reproducing a bug via `pnpm exec agentcrumbs clear --app trigger`. diff --git a/CHANGESETS.md b/CHANGESETS.md index 722fe64eb4..74f8adda6c 100644 --- a/CHANGESETS.md +++ b/CHANGESETS.md @@ -1,24 +1,53 @@ -# Changesets +# Changesets and Server Changes -Trigger.dev uses [changesets](https://github.com/changesets/changesets) to manage updated our packages and releasing them to npm. +Trigger.dev uses [changesets](https://github.com/changesets/changesets) to manage package versions and releasing them to npm. For server-only changes, we use a lightweight `.server-changes/` convention. -## Adding a changeset +## Adding a changeset (package changes) -To add a changeset, use `pnpm run changeset:add` and follow the instructions [here](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md). Please only ever select one of our public packages when adding a changeset. +Changesets and `.server-changes/` files are user-facing release notes, not a catalog of every change. Add one only when the change is something a user would notice or act on. Skip it for internal-only changes, refactors, chores, and packages that are not consumed independently (e.g. `@trigger.dev/redis-worker`). Anyone who wants the exact history reads the commits. -## Release instructions (local only) +To add a changeset, use `pnpm run changeset:add` and follow the [Changesets adding-a-changeset guide](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md). Please only ever select one of our public packages when adding a changeset. -Based on the instructions [here](https://github.com/changesets/changesets/blob/main/docs/intro-to-using-changesets.md) +## Adding a server change (server-only changes) -1. Run `pnpm run changeset:version` -2. Run `pnpm run changeset:release` +If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, etc.), does NOT change any published packages, AND the change is user-facing, add a `.server-changes/` file instead of a changeset: + +```sh +cat > .server-changes/fix-batch-queue-stalls.md << 'EOF' +--- +area: webapp +type: fix +--- + +Speed up batch queue processing by removing stalls and fixing retry race +EOF +``` + +- `area`: `webapp` | `supervisor` +- `type`: `feature` | `fix` | `improvement` | `breaking` + +For **mixed PRs** (both packages and server): the changeset covers it, so no `.server-changes/` file is needed. If the package change is internal and needs no changeset but the server change is user-facing, add a `.server-changes/` file for it. + +See `.server-changes/README.md` for full documentation. + +## When to add which + +Only for user-facing changes. Skip the note entirely for internal-only or admin-only changes, refactors, and chores. + +| PR changes | What to add | +|---|---| +| Only packages (`packages/` or `integrations/`) | Changeset (`pnpm run changeset:add`), if the package change is user-facing | +| Only server (`apps/`) | `.server-changes/` file, if the server change is user-facing | +| Both packages and server | The changeset covers it; if the package change needs no changeset but the server change is user-facing, add a `.server-changes/` file | ## Release instructions (CI) Please follow the best-practice of adding changesets in the same commit as the code making the change with `pnpm run changeset:add`, as it will allow our release.yml CI workflow to function properly: -- Anytime new changesets are added in a commit in the `main` branch, the [release.yml](./.github/workflows/release.yml) workflow will run and will automatically create/update a PR with a fresh run of `pnpm run changeset:version`. -- When the version PR is merged into `main`, the release.yml workflow will automatically run `pnpm run changeset:release` to build and release packages to npm. +- Anytime new changesets are added in a commit in the `main` branch, the [changesets-pr.yml](./.github/workflows/changesets-pr.yml) workflow will run and will automatically create/update a PR with a fresh run of `pnpm run changeset:version`. +- The release PR body is automatically enhanced with a clean, deduplicated summary that includes both package changes and `.server-changes/` entries. +- Consumed `.server-changes/` files are removed on the `changeset-release/main` branch โ€” the same way changesets deletes `.changeset/*.md` files. When the release PR merges, they're gone from main. +- When the version PR is merged into `main`, the [release.yml](./.github/workflows/release.yml) workflow will automatically build, release packages to npm, and create a single unified GitHub release. ## Pre-release instructions diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..43c994c2d3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5924f89da3..591828c1f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,25 @@ Thank you for taking the time to contribute to Trigger.dev. Your involvement is not just welcomed, but we encourage it! ๐Ÿš€ -Please take some time to read this guide to understand contributing best practices for Trigger.dev. +Please take some time to read this guide to understand contributing best practices for Trigger.dev. Note that we use [vouch](https://github.com/mitchellh/vouch) to manage contributor trust, so you'll need to be vouched before opening a PR. Thank you for helping us make Trigger.dev even better! ๐Ÿคฉ +> **Important:** We only accept PRs that address a single issue. Please do not submit PRs containing multiple unrelated fixes or features. If you have multiple contributions, open a separate PR for each one. + +## Getting vouched (required before opening a PR) + +We use [vouch](https://github.com/mitchellh/vouch) to manage contributor trust. **PRs from unvouched users are automatically closed.** + +Before you open your first pull request, you need to be vouched by a maintainer. Here's how: + +1. Open a [Vouch Request](https://github.com/triggerdotdev/trigger.dev/issues/new?template=vouch-request.yml) issue. +2. Tell us what you'd like to work on and share any relevant background. +3. A maintainer will review your request and vouch for you by commenting on the issue. +4. Once vouched, your PRs will be accepted normally. + +If you're unsure whether you're already vouched, go ahead and open a PR โ€” the check will tell you. + ## Developing The development branch is `main`. This is the branch that all pull @@ -14,8 +29,8 @@ branch are tagged into a release periodically. ### Prerequisites -- [Node.js](https://nodejs.org/en) version 20.11.1 -- [pnpm package manager](https://pnpm.io/installation) version 10.23.0 +- [Node.js](https://nodejs.org/en) version 24.18.0 +- [pnpm package manager](https://pnpm.io/installation) version 10.33.2 - [Docker](https://www.docker.com/get-started/) - [protobuf](https://github.com/protocolbuffers/protobuf) @@ -34,9 +49,9 @@ branch are tagged into a release periodically. ``` cd trigger.dev ``` -3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository. +3. Ensure you are on the correct version of Node.js (24.18.0). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository. -4. Run `corepack enable` to use the correct version of pnpm (`10.23.0`) as specified in the root `package.json` file. +4. Run `corepack enable` to use the correct version of pnpm (`10.33.2`) as specified in the root `package.json` file. 5. Install the required packages using pnpm. ``` @@ -56,21 +71,27 @@ branch are tagged into a release periodically. Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method. -8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md) +8. Start Docker. This starts the core dev services (Postgres, Redis, Electric, MinIO, ClickHouse, s2-lite) and runs the ClickHouse migrator once on first start. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md). ``` pnpm run docker ``` + For the observability stack (Prometheus, Grafana, OTEL collector) and other optional tooling (Toxiproxy, nginx-h2, ch-ui, extra electric shard), use `pnpm run docker:full` instead. See `docker/docker-compose.extras.yml` for the full list. + 9. Migrate the database ``` pnpm run db:migrate ``` -10. Build everything +10. Build the webapp, CLI, and SDK + ``` + pnpm run build --filter webapp --filter trigger.dev --filter @trigger.dev/sdk ``` - pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk +11. Seed the database. This creates a local user, a `References` org, and the reference projects (including `hello-world`) with stable IDs. ``` -11. Run the app. See the section below. + pnpm run db:seed + ``` +12. Run the app. See the section below. ## Running @@ -86,29 +107,32 @@ branch are tagged into a release periodically. ## Manual testing using hello-world -We use the `/references/hello-world` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `packages/core`), the CLI (`trigger.dev` at `/packages/cli-v3`) and the platform (The remix app at `/apps/webapp`). The instructions below will get you started on using the `hello-world` for local development of Trigger.dev. - -### First-time setup +The `hello-world` reference project (and the others) live in a separate repo: +[`triggerdotdev/references`](https://github.com/triggerdotdev/references). Clone it +alongside this repo. It's the staging ground for testing changes to the SDK +(`@trigger.dev/sdk` at `/packages/trigger-sdk`), the Core package +(`@trigger.dev/core` at `/packages/core`), the CLI (`trigger.dev` at +`/packages/cli-v3`) and the platform (the Remix app at `/apps/webapp`). +To exercise your local monorepo changes, the reference project links to your local +build โ€” see the references repo's README for the `pnpm run link` flow. -First, make sure you are running the webapp according to the instructions above. Then: +> Paths below such as `projects/hello-world` are relative to your `references` +> clone, not this repo. -1. Visit http://localhost:3030 in your browser and create a new V3 project called "hello-world". +### First-time setup -2. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `proj_rrkpdguyagvsoktglnod`. +First, make sure you are running the webapp according to the instructions above. The seed step from setup already created a `hello-world` project under the `References` org with the stable ref `proj_rrkpdguyagvsoktglnod` โ€” log in at http://localhost:3030 with any email to access it. Then: -3. Build the CLI +1. Build the CLI and packages (skip if you already ran the build step in setup) ```sh -# Build the CLI -pnpm run build --filter trigger.dev -# Make it accessible to `pnpm exec` -pnpm i +pnpm run build --filter trigger.dev --filter "@trigger.dev/*" ``` -4. Change into the `/references/hello-world` directory and authorize the CLI to the local server: +2. In your `references` clone, link to your local monorepo build (see its README), then change into `projects/hello-world` and authorize the CLI to the local server: ```sh -cd references/hello-world +cd projects/hello-world cp .env.example .env pnpm exec trigger login -a http://localhost:3030 ``` @@ -118,7 +142,7 @@ This will open a new browser window and authorize the CLI against your local use You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development: ```sh -cd references/hello-world +cd projects/hello-world pnpm exec trigger login -a http://localhost:3030 --profile local # later when you run the dev or deploy command: pnpm exec trigger dev --profile local @@ -127,50 +151,50 @@ pnpm exec trigger deploy --profile local ### Running -The following steps should be followed any time you start working on a new feature you want to test in v3: +The following steps should be followed any time you start working on a new feature you want to test: 1. Make sure the webapp is running on localhost:3030 -2. Open a terminal window and build the CLI and packages and watch for changes +2. In this repo, open a terminal window and build the CLI and packages and watch for changes (the reference project links against this build) ```sh pnpm run dev --filter trigger.dev --filter "@trigger.dev/*" ``` -3. Open another terminal window, and change into the `/references/hello-world` directory. +3. Open another terminal window, and change into `projects/hello-world` in your `references` clone. 4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution: ```sh -# in /references/hello-world +# in /projects/hello-world pnpm exec trigger dev ``` If you want additional debug logging, you can use the `--log-level debug` flag: ```sh -# in /references/hello-world +# in /projects/hello-world pnpm exec trigger dev --log-level debug ``` -6. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `hello-world/src/trigger` dir will automatically be rebuilt by the `dev` command. +5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside the reference project's `src/trigger` dir will automatically be rebuilt by the `dev` command. -7. Navigate to the `hello-world` project in your local dashboard at localhost:3030 and you should see the list of tasks. +6. Navigate to the `hello-world` project in your local dashboard at localhost:3030 and you should see the list of tasks. -8. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/hello-world/src/trigger` folder. Many of them accept an empty payload. +7. On the Tasks page, open a task and press the "Test" button to open its test page. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the reference project's `src/trigger` folder. Many of them accept an empty payload. -9. Feel free to add additional files in `hello-world/src/trigger` to test out specific aspects of the system, or add in edge cases. +8. Feel free to add additional files in the reference project's `src/trigger` dir to test out specific aspects of the system, or add in edge cases. ## Adding and running migrations -1. Modify internal-packages/database/prisma/schema.prisma file -2. Change directory to the packages/database folder +1. Modify `internal-packages/database/prisma/schema.prisma`. +2. Change directory to the database package: ```sh - cd packages/database + cd internal-packages/database ``` -3. Create a migration +3. Create a migration: ``` pnpm run db:migrate:dev:create @@ -178,65 +202,65 @@ pnpm exec trigger dev --log-level debug This creates a migration file. Check the migration file does only what you want. If you're adding any database indexes they must use `CONCURRENTLY`, otherwise they'll lock the table when executed. -4. Run the migration. +4. Run the migration: -``` -pnpm run db:migrate:deploy -pnpm run generate -``` + ``` + pnpm run db:migrate:deploy + pnpm run generate + ``` -This executes the migrations against your database and applies changes to the database schema(s), and then regenerates the Prisma client. + This executes the migrations against your database and applies changes to the database schema(s), and then regenerates the Prisma client. -4. Commit generated migrations as well as changes to the schema.prisma file -5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`. +5. Commit the generated migration files as well as the changes to `schema.prisma`. +6. If you're using VSCode you may need to restart the TypeScript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`. -## Add sample jobs +## Git hooks (lefthook) -The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with. +We use [lefthook](https://lefthook.dev) for local git hooks, configured in `lefthook.yml` (the source of truth for what runs and when). Today that's a pre-push hook mirroring the CI `code-quality` checks; the set may grow, so check `lefthook.yml` rather than this guide. -1. `cd` into `references/job-catalog` -2. Create a `.env` file with the following content, - replacing `` with an actual key: +Hooks install automatically on `pnpm install`. A failing hook prints exactly what to run to fix it. -```env -TRIGGER_API_KEY=[TRIGGER_DEV_API_KEY] -TRIGGER_API_URL=http://localhost:3030 -``` +**Opting out** -`TRIGGER_API_URL` is used to configure the URL for your Trigger.dev instance, -where the jobs will be registered. +- GitButler skips hooks on `but push` unless you enable **Run hooks** in the project settings (off by default). +- Plain git: `LEFTHOOK=0 git push` / `--no-verify` to skip once; `pnpm exec lefthook uninstall` to remove. -3. Run one of the the `job-catalog` files: +This never affects correctness โ€” CI enforces the same checks on every PR; the hooks just give you faster feedback. -```sh -pnpm run events -``` +## Making a pull request -This will open up a local server using `express` on port 8080. Then in a new terminal window you can run the trigger-cli dev command: +**If you get errors, be sure to fix them before committing.** -```sh -pnpm run dev:trigger -``` +> **Note:** We may close PRs if we decide that the cost of integrating the change outweighs the benefits. To improve the chances of your PR getting accepted, follow the guidelines below. -See the [Job Catalog](./references/job-catalog/README.md) file for more. +### PR workflow -4. Navigate to your trigger.dev instance ([http://localhost:3030](http://localhost:3030/)), to see the jobs. - You can use the test feature to trigger them. +1. **Always open your PR in draft status first.** Do not mark it as "Ready for Review" until the steps below are complete. +2. **Run format and lint locally before pushing:** + ```bash + pnpm run format # auto-fixes formatting (oxfmt) + pnpm run lint:fix # auto-fixes lint violations (oxlint) + ``` + Both are enforced by CI โ€” the `code-quality` check will fail if either produces a diff or errors. +3. **Address all CodeRabbit code review comments.** Our CI runs an automated code review via CodeRabbit. Go through each comment and either fix the issue or resolve it with a comment explaining why no change is needed. +4. **Wait for all CI checks to pass.** Do not mark the PR as "Ready for Review" until every check is green. +5. **Then mark the PR as "Ready for Review"** so a maintainer can take a look. -## Making a pull request +### Cost/benefit analysis for risky changes -**If you get errors, be sure to fix them before committing.** +If your change touches core infrastructure, modifies widely-used code paths, or could introduce regressions, consider doing a brief cost/benefit analysis and including it in the PR description. Explain what the benefit is to users and why the risk is worth it. This goes a long way toward helping maintainers evaluate your contribution. + +### General guidelines -- Be sure to [check the "Allow edits from maintainers" option](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork) while creating you PR. -- If your PR refers to or fixes an issue, be sure to add `refs #XXX` or `fixes #XXX` to the PR description. Replacing `XXX` with the respective issue number. See more about [Linking a pull request to an issue - ](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue). +- Be sure to [check the "Allow edits from maintainers" option](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork) while creating your PR. +- If your PR refers to or fixes an issue, be sure to add `refs #XXX` or `fixes #XXX` to the PR description. Replacing `XXX` with the respective issue number. See more about [Linking a pull request to an issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue). - Be sure to fill the PR Template accordingly. ## Adding changesets We use [changesets](https://github.com/changesets/changesets) to manage our package versions and changelogs. If you've never used changesets before, first read [their guide here](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md). -If you are contributing a change to any packages in this monorepo (anything in either the `/packages` or `/integrations` directories), then you will need to add a changeset to your Pull Requests before they can be merged. +Changesets are user-facing release notes, not a catalog of every change. If you are contributing a **user-facing** change to a package in this monorepo (anything in `/packages` or `/integrations` that a user would notice or act on), add a changeset to your Pull Request before it can be merged. Skip the changeset for internal-only changes, refactors, chores, and packages that are not consumed independently (e.g. `@trigger.dev/redis-worker`), where a version bump means nothing to a user. To add a changeset, run the following command in the root of the repo @@ -252,6 +276,39 @@ You will be prompted to select which packages to include in the changeset. Only Most of the time the changes you'll make are likely to be categorized as patch releases. If you feel like there is the need for a minor or major release of the package based on the changes being made, add the changeset as such and it will be discussed during PR review. +## Adding server changes + +Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, etc.) with no package changes AND the change is user-facing, add a `.server-changes/` file so the change appears in release notes. Skip it for internal-only or admin-only changes, refactors, and chores. + +Create a markdown file with a descriptive name: + +```sh +cat > .server-changes/fix-batch-queue-stalls.md << 'EOF' +--- +area: webapp +type: fix +--- + +Speed up batch queue processing by removing stalls and fixing retry race +EOF +``` + +**Fields:** +- `area` (required): `webapp` | `supervisor` +- `type` (required): `feature` | `fix` | `improvement` | `breaking` + +The body text (below the frontmatter) is a one-line description of the change. Keep it concise โ€” it will appear in release notes. + +**When to add which** (only for user-facing changes; skip the note entirely for internal-only or admin-only changes, refactors, and chores): + +| PR changes | What to add | +|---|---| +| Only packages (`packages/` or `integrations/`) | Changeset (if the package change is user-facing) | +| Only server (`apps/`) | `.server-changes/` file (if the server change is user-facing) | +| Both packages and server | The changeset covers it; if the package change needs no changeset but the server change is user-facing, add a `.server-changes/` file | + +See `.server-changes/README.md` for more details. + ## Troubleshooting ### EADDRINUSE: address already in use :::3030 @@ -272,3 +329,7 @@ The process running on port `3030` should be destroyed. ```sh sudo kill -9 ``` + +### Running two clones side by side (worktree, branch experiment) + +The default `pnpm run docker` uses the project name `triggerdotdev-docker` and the standard host ports (5432, 6379, 3060, 4566, 8123, 9000, 9005, 9006). To stand up a second instance in another clone without clashing, set a different `COMPOSE_PROJECT_NAME` and the offset host ports in that clone's `.env`. The "Running multiple instances side by side" block in `.env.example` lists every overridable env var with its default for reference; uncomment the lines you need and update `DATABASE_URL` / `CLICKHOUSE_URL` / `REDIS_PORT` / `APP_ORIGIN` / `LOGIN_ORIGIN` / `ELECTRIC_ORIGIN` / `REALTIME_STREAMS_S2_ENDPOINT` to match. diff --git a/ONBOARDING.md b/ONBOARDING.md new file mode 100644 index 0000000000..c7e1ebaf68 --- /dev/null +++ b/ONBOARDING.md @@ -0,0 +1,221 @@ +# Onboarding: taking over the hosted webhooks PR (#4344) for design + +You are picking up **PR #4344 "hosted webhooks, agent channels, and human-in-the-loop"** to own the UX and front-end. This doc gets you from a clean machine to a running dashboard with realistic webhook data you can screenshot, restyle, and iterate on. + +The feature is built and green (all backend plumbing, all four dashboard surfaces, the in-app test console). Your job is the visual and interaction design of the dashboard surfaces, not the backend. Everything below is oriented around that. + +--- + +## 1. What you are designing + +Hosted webhooks let a Trigger.dev user receive and verify a provider's webhooks (Stripe, GitHub, and so on) as a task, with no ingress or verification code of their own. A `webhook()` handler in their project gets a hosted URL; deliveries to that URL are verified, recorded, and routed to their `onEvent` handler. + +The dashboard has **four surfaces you own**, all under the "Webhooks" nav section (teal icon): + +| Surface | Route (under `/orgs/:org/projects/:project/env/:env`) | What it shows | +| --- | --- | --- | +| **Deliveries list** | `/webhooks` | Every delivery across all endpoints in the environment. Runs-style filter bar (Status, Webhook, Created, plus a More-filters menu for Delivery ID / Run ID), applied-filter pills, a Webhook column linking to the handler. This is the main screen. | +| **Delivery detail** | `/webhooks/deliveries/:deliveryParam` | One delivery. Main panel is a tabbed view (Event payload / Request headers) rendered as JSON. Sidebar property table (status badge, webhook + run links, external delivery id, idempotency key, timestamps, computed duration, error). Also has a friendly "not available / retained for N days" empty state for expired or bogus links. | +| **Handler detail + Console** | `/webhooks/:webhookParam` | The handler (the `webhook()` in the user's code). Tabs: Deliveries, Runs, Endpoints. This page also hosts the **Webhook Console / Composer** (see section 5), the tool you will lean on for data. | +| **Endpoint detail** | `/webhooks/endpoints/:endpointParam` | One endpoint. Left: scoped deliveries. Right: a **Connect** card (webhook URL, signing secret set/rotate/generate, provider setup rendered from the verifier config), Routing, Scope, Metadata. | + +The status vocabulary, badges, and colors live in `components/webhookDeliveries/v1/DeliveryStatus.tsx` and `components/webhookEndpoints/v1/EndpointStatus.tsx`. The nav accent color is a Tailwind token `--color-webhooks` (teal), used via `text-webhooks`. + +--- + +## 2. Get the code + +You need the PR branch, `feat/hosted-webhook-ingress`. + +```bash +git clone https://github.com/triggerdotdev/trigger.dev.git +cd trigger.dev +gh pr checkout 4344 # lands you on feat/hosted-webhook-ingress +``` + +If you plan to push design changes back to this branch, coordinate with Eric first: the branch is rebased and force-pushed periodically, so agree on timing or work on a child branch and open a follow-up. + +Toolchain: pnpm 10.33.2 via corepack, Node 22+. Use `corepack pnpm` (a bare `pnpm` can be an old global that wipes `node_modules`). + +```bash +corepack enable +corepack pnpm install +``` + +--- + +## 3. Bring the stack up + +Four services and the webapp. Run from the repo root. + +```bash +# 1. Core dev services: Postgres, Redis, Electric, MinIO, ClickHouse, s2-lite +corepack pnpm run docker + +# 2. Config +cp .env.example .env +``` + +Now edit `.env` and add the two webhook-delivery replication lines (they are NOT in `.env.example`, and without them the Deliveries list looks empty even after you send webhooks, see section 5): + +```bash +# webhook deliveries replication (required for the Deliveries list/detail to populate) +WEBHOOK_DELIVERIES_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123 +WEBHOOK_DELIVERIES_REPLICATION_ENABLED=1 +``` + +Then migrate, seed, build, and run: + +```bash +corepack pnpm run db:migrate +corepack pnpm run db:seed # creates the References org + hello-world project + +# Build the pieces you will run (do these sequentially, not with db:seed running) +corepack pnpm run build --filter webapp --filter trigger.dev --filter "@trigger.dev/sdk" + +# Run the webapp (http://localhost:3030) +corepack pnpm run dev --filter webapp +curl -s http://localhost:3030/healthcheck # verify +``` + +**Log in (dev):** open http://localhost:3030, submit the email `local@trigger.dev`. Dev auto-verifies the magic link (watch the webapp log for `/magic?token=`). That seeded user is an org admin, which matters for the next step. + +--- + +## 4. Turn the feature on + +The dashboard is gated by a feature flag, `hasWebhooksAccess` (default off). + +- The seeded dev user `local@trigger.dev` is an **admin**, and admins bypass the flag, so on a fresh seed the Webhooks nav section is already visible to you. Nothing to do. +- If you use a non-admin user, set `featureFlags.hasWebhooksAccess = true` on the `Organization` row to reveal the nav section. (A global `FeatureFlag` row with key `hasWebhooksAccess` makes the pages reachable by URL, but the left nav reads only org-level flags, so the section stays hidden for non-admins.) + +If the "Webhooks" section is missing from the left nav, this flag is why. + +--- + +## 5. Get nice data (the part that matters) + +Delivery rows are what make these screens interesting: a spread of providers, statuses, payloads, timestamps. Here is how the data flows and how to produce it. + +### The pipeline (why an empty list is usually a setup issue, not a bug) + +`ingest -> engine (verify, filter, route) -> Postgres WebhookDelivery rows -> replication -> ClickHouse`. The Deliveries **list orders and paginates from ClickHouse**, then hydrates every visible field from Postgres. So if replication is off (section 3), you can create deliveries and still see an empty list. Enable the two replication env vars and restart the webapp. + +One caveat baked into the design: replication starts streaming from the moment it is enabled, so deliveries written **before** you turned it on will not appear. Turn replication on first, then generate data. + +### Fastest path: the seed script + +There is a seed script that inserts a full, stable dataset directly into both stores (Postgres and ClickHouse), so you get realistic screens on a fresh DB with no workers, no `trigger dev`, and no signing secrets to set: + +```bash +corepack pnpm --filter webapp run db:seed:webhooks +# optional: deliveries per endpoint (default 45) +corepack pnpm --filter webapp run db:seed:webhooks -- 60 +``` + +It creates six endpoints across different providers and verifier schemes (Stripe, GitHub, Slack, Svix, Discord, and a custom shared-secret one, with a mix of active/inactive and secret-set/not-set), then a spread of deliveries over the last two weeks covering **every** delivery status (SUCCEEDED, FAILED, FILTERED, PENDING, PROCESSING), realistic per-provider payloads and headers, and a mix of test and live. It attaches to the first DEVELOPMENT environment your local user can see (set `WEBHOOK_SEED_PROJECT=""` to target a specific one), and prints the exact Deliveries URL when it finishes. Re-running clears and reseeds that environment, so you always get the same clean dataset. The script is `apps/webapp/seed-webhook-deliveries.ts`; edit the `ENDPOINTS` array or the status weights to shape the data to whatever you are designing. + +Because it writes the ClickHouse rows directly, seeded data shows up **without** the replication setup in section 3. That replication env is only needed for the live and Composer paths below. (The seed uses `WEBHOOK_DELIVERIES_REPLICATION_CLICKHOUSE_URL` if set, otherwise `CLICKHOUSE_URL`, which is already in `.env.example`.) + +This is the recommended way to get data. The interactive paths below are for exercising the live pipeline (real verification, real routed runs) or the in-app test console. + +### Interactive: create an endpoint (one-time) + +The Composer sends to an endpoint, and endpoints only exist once a Trigger project that declares a `webhook()` has been dev-run or deployed. Quickest path: a tiny demo project. + +```ts +// demo/src/trigger/demo-webhook.ts +import { webhook, webhooks } from "@trigger.dev/sdk"; + +export const demoWebhook = webhook({ + id: "demo-webhook", + source: webhooks.custom<{ message: string }>({ /* generic HMAC */ }), + onEvent: async ({ event, headers, ctx }) => { + // event is the parsed body, headers is a Web Headers object + }, +}); + +// A real provider, for realistic payloads: +export const stripeWebhook = webhook({ + id: "stripe-webhook", + source: webhooks.stripe(), + onEvent: async ({ event }) => {}, +}); +``` + +Link that demo project to your local build and run `trigger dev` (see `AGENTS.md` "Testing with the hello-world Reference Project" for linking; the `triggerdotdev/references` repo has ready-made projects). Running `trigger dev` registers the `webhook()` handlers, which creates their endpoints. Set each endpoint's signing secret from the **endpoint detail Connect card** (Generate or paste). + +### Interactive: fire deliveries with the Webhook Console + +Open the handler detail page (`/webhooks/:webhookParam`). It hosts the **Composer** (`components/webhookConsole/WebhookComposer.tsx`). It has four source tabs and four signature modes, and it injects the delivery straight through the engine in-process, so it is fast and does not consume any real rate budget: + +- **Sample tab**: pick a real provider event from the built-in catalog (`@internal/webhook-sources`, six first-class providers plus a large sample manifest). This is the fastest way to get realistic Stripe / GitHub / Svix / Square / Discord payloads with correct-looking headers. +- **Body tab**: hand-write any JSON. +- **Replay tab**: re-send a prior delivery. +- **AI tab**: generate a payload with a prompt. +- **Signature modes** `signed | unsigned | tampered | simulate`: this is how you produce a **spread of delivery statuses**. `signed` (with a secret set) verifies and routes to a SUCCEEDED delivery; `unsigned` and `tampered` produce failed/rejected deliveries. Send a mix to populate every status badge you need to design. + +To get SUCCEEDED deliveries whose **runs** also complete (nicest end-to-end data), keep the demo project's `trigger dev` running so the routed task actually executes. + +### Interactive: a real provider (most realistic) + +For genuine payloads and headers, point the Stripe CLI at an endpoint: `stripe listen --forward-to http://localhost:3030/webhooks/v1/ingest/`, set that endpoint's `whsec` via the Connect card, then `stripe trigger payment_intent.succeeded`. + +--- + +## 6. Where the front-end code lives + +| Area | Path | +| --- | --- | +| Routes (pages) | `apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.webhooks*` | +| Deliveries list / detail components | `apps/webapp/app/components/webhookDeliveries/v1/` (`DeliveriesTable`, `DeliveryStatus`, `WebhookDeliveryFilters`, `DeliveryTimeline`, `useDeliveriesLiveReload`) | +| Endpoint components | `apps/webapp/app/components/webhookEndpoints/v1/` (`EndpointsTable`, `EndpointStatus`) | +| Console / Composer | `apps/webapp/app/components/webhookConsole/` (`WebhookComposer`, `SampleSourcePicker`, `ReplaySourcePicker`) | +| Data (presenters, read-only from your side) | `apps/webapp/app/presenters/v3/WebhookDeliveriesListPresenter.server.ts`, `WebhookDeliveryDetailPresenter.server.ts`, `WebhookDetailPresenter.server.ts`, `webhookComposerEndpoints.server.ts` | +| Nav entry | `apps/webapp/app/components/navigation/SideMenu.tsx` (the `staticSections` "webhooks" push) | +| Path builders | `apps/webapp/app/utils/pathBuilder.ts` (`v3WebhooksPath`, `v3WebhookDeliveryPath`, `v3WebhookEndpointPath`, `v3WebhookTaskPath`) | +| Accent color token | `apps/webapp/app/tailwind.css` (`--color-webhooks`, used as `text-webhooks`) | +| Data seed script | `apps/webapp/seed-webhook-deliveries.ts` (run via `db:seed:webhooks`) | + +**Styling:** the webapp is on Tailwind v4 (CSS-first `@theme` in `apps/webapp/app/tailwind.css`, there is no `tailwind.config.js`). Add or change design tokens there. + +**Design language to match:** these screens deliberately reuse the Runs page primitives (the filter bar is built from `RunFilters` / `SharedFilters`, the tables mirror the Runs table cells). Match the Runs and Sessions pages, not a new visual system. + +--- + +## 7. Iterating + +- **HMR vs restart:** editing a component (`.tsx`) hot-reloads. Editing a `.server.ts` file makes the Remix dev server restart the app (a brief connection refused, then it comes back). Editing Tailwind tokens hot-reloads. +- **Screenshots:** capture from the running dashboard at http://localhost:3030. Save shots outside the repo or to a scratch folder so they do not get committed. +- **Typecheck after non-trivial changes:** `corepack pnpm run typecheck --filter webapp` (about 1 to 2 minutes). For small style tweaks, trust it and let CI catch anything. +- **One boundary gotcha that the dev server will NOT catch:** route files must not leak server-only imports into the client bundle. The dev server tolerates it, but the production build fails. If you touch a route file and import anything server-only, run `corepack pnpm --filter webapp run build:remix` before pushing. Pure component and style edits are unaffected. + +--- + +## 8. Shipping your changes + +Follow the repo PR workflow: + +- Format and lint before committing: `corepack pnpm run format` (oxfmt) and `corepack pnpm run lint:fix` (oxlint). CI enforces both. +- Commit style is Conventional Commits, for example `feat(webapp): redesign webhook deliveries table`. No emoji, no attribution footer. +- The PR is a **draft** awaiting an AI review pass, then a human review, before it flips to ready. Do not flip it to ready yourself; push your commits and let Eric coordinate the review and any rebase onto `main`. +- CI to expect: `code-quality` (oxfmt + oxlint), `typecheck`, webapp unit shards, and the Playwright `e2e-webapp` job. Style-only changes usually only risk `code-quality`. + +--- + +## 9. Quick reference + +- **Webapp:** http://localhost:3030 (port comes from `REMIX_APP_PORT`, falling back to `PORT`/3030). +- **Default docker services:** Postgres 5432, Redis 6379, ClickHouse HTTP 8123 (`default:password`), MinIO, Electric, s2-lite. +- **Feature flag:** `hasWebhooksAccess` (admins bypass). +- **Seed data:** `corepack pnpm --filter webapp run db:seed:webhooks` (append `-- ` for deliveries per endpoint). +- **Must-set env for data to show:** `WEBHOOK_DELIVERIES_REPLICATION_ENABLED=1` and `WEBHOOK_DELIVERIES_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123`. +- **Login:** `local@trigger.dev`, magic link auto-verifies in dev. +- **Feature docs:** `docs/webhooks/` (overview, sources, connect, deliveries, channels, human-in-the-loop). Read `overview.mdx` and `deliveries.mdx` first for the mental model behind the screens. +- **PR:** https://github.com/triggerdotdev/trigger.dev/pull/4344 + +--- + +## 10. Mental model in one paragraph + +A user writes a `webhook()` in their project. On deploy (or `trigger dev`) that handler gets one or more hosted endpoints, each with a signing secret. A provider POSTs to the endpoint's URL; the engine verifies the signature, optionally filters, records a `WebhookDelivery`, and triggers the routed task run. The dashboard reads those deliveries: the list orders them out of ClickHouse and hydrates the rest from Postgres, the detail page reads Postgres directly (it holds the only copy of the event payload and headers). Everything you design sits on top of that delivery record and the endpoint that produced it. diff --git a/README.md b/README.md index 6838d45468..0d7f1ca293 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ### Build and deploy fullyโ€‘managed AI agents and workflows -[Website](https://trigger.dev) | [Docs](https://trigger.dev/docs) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Feature requests](https://triggerdev.featurebase.app/) | [Public roadmap](https://triggerdev.featurebase.app/roadmap) | [Self-hosting](https://trigger.dev/docs/self-hosting/overview) +[Website](https://trigger.dev) | [Docs](https://trigger.dev/docs) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Example projects](https://github.com/triggerdotdev/examples) | [Feature requests](https://triggerdev.featurebase.app/) | [Public roadmap](https://triggerdev.featurebase.app/roadmap) | [Self-hosting](https://trigger.dev/docs/self-hosting/overview) [![Open Source](https://img.shields.io/badge/Open%20Source-%E2%9D%A4-red.svg)](https://github.com/triggerdotdev/trigger.dev) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/triggerdotdev/trigger.dev/blob/main/LICENSE) diff --git a/RELEASE.md b/RELEASE.md index 1b9273cb14..8ba3ecb500 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,5 +1,28 @@ ## Guide on releasing a new version +### Automated release (v4+) + +Releases are fully automated via CI: + +1. PRs merge to `main` with changesets (for package changes) and/or `.server-changes/` files (for server-only changes). +2. The [changesets-pr.yml](./.github/workflows/changesets-pr.yml) workflow automatically creates/updates the `changeset-release/main` PR with version bumps and an enhanced summary of all changes. Consumed `.server-changes/` files are removed on the release branch (same approach changesets uses for `.changeset/` files โ€” they're deleted on the branch, so merging the PR cleans them up). +3. When ready to release, merge the changeset release PR into `main`. +4. The [release.yml](./.github/workflows/release.yml) workflow automatically: + - Publishes all packages to npm + - Creates a single unified GitHub release (e.g., "trigger.dev v4.3.4") + - Tags and triggers Docker image builds + - After Docker images are pushed, updates the GitHub release with the exact GHCR tag link + +### What engineers need to do + +- **Package changes**: Add a changeset with `pnpm run changeset:add` +- **Server-only changes**: Add a `.server-changes/` file (see `.server-changes/README.md`) +- **Mixed PRs**: Just the changeset is enough + +See `CHANGESETS.md` for full details on changesets and server changes. + +### Legacy release (v3) + 1. Merge in the changeset PR into main, making sure to cancel both the release and publish github actions from that merge. 2. Pull the changes locally into main 3. Run `pnpm i` which will update the pnpm lock file with the new versions diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..6dbe24a669 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,49 @@ +# Security Policy + +We take the security of Trigger.dev seriously โ€” for both our Cloud service and self-hosted deployments. This document explains how to report a vulnerability and what to expect from us. + +## Reporting a vulnerability + +**Please do not report security vulnerabilities through public GitHub issues, pull requests, or our Discord.** + +Use one of these private channels instead: + +1. **GitHub (preferred):** Open a private report from the repository's **Security** tab โ€” click **"Report a vulnerability"** ([direct link](https://github.com/triggerdotdev/trigger.dev/security/advisories/new)). +2. **Email:** `security-advisories@trigger.dev` + +Please include as much of the following as you can: + +- A description of the vulnerability and its impact +- Steps to reproduce, ideally with a proof of concept +- Affected version(s) and component(s) +- Any suggested remediation + +If you report by email, we will open a private GitHub Security Advisory to track the issue. All reports โ€” however they reach us โ€” are tracked there. + +## What to expect + +| Stage | Target | +| --- | --- | +| Acknowledgement of your report | within 3 business days | +| Validation and severity assessment (CVSS 3.1) | within 1 week | + +We assess severity using CVSS 3.1 and prioritise remediation accordingly: + +| Severity (CVSS 3.1) | Target time to resolve | +| --- | --- | +| Critical (9.0โ€“10.0) | 7 days | +| High (7.0โ€“8.9) | 30 days | +| Medium (4.0โ€“6.9) | 90 days | +| Low (0.1โ€“3.9) | As needed | + +These are best-effort targets, measured from the point we validate and accept a report โ€” not guarantees. Real-world exploitability may lead us to escalate an issue beyond its base score. + +## Coordinated disclosure + +We follow coordinated disclosure. Please give us a reasonable opportunity to investigate and ship a fix before any public disclosure. Our default disclosure window is 90 days from acceptance, though we aim to resolve issues sooner. + +Once a fix is released we publish a GitHub Security Advisory (and request a CVE where applicable), and we credit reporters unless you ask to remain anonymous. + +## Supported versions + +We patch the **latest released version line** only. Self-hosters should run the latest version-tagged release to receive security fixes. See the [self-hosting documentation](https://trigger.dev/docs/self-hosting/overview). diff --git a/ai/references/repo.md b/ai/references/repo.md index 4f67bde2b4..68286729ca 100644 --- a/ai/references/repo.md +++ b/ai/references/repo.md @@ -1,6 +1,6 @@ ## Repo Overview -This is a pnpm 10.23.0 monorepo that uses turborepo @turbo.json. The following workspaces are relevant +This is a pnpm 10.33.2 monorepo that uses turborepo @turbo.json. The following workspaces are relevant ## Apps @@ -23,7 +23,6 @@ This is a pnpm 10.23.0 monorepo that uses turborepo @turbo.json. The following w - /internal-packages/run-engine is the `@internal/run-engine` package that is "Run Engine 2.0" and handles moving a run all the way through it's lifecycle - /internal-packages/redis is the `@internal/redis` package that exports Redis types and the `createRedisClient` function to unify how we create redis clients in the repo. It's not used everywhere yet, but it's the preferred way to create redis clients from now on. - /internal-packages/testcontainers is the `@internal/testcontainers` package that exports a few useful functions for spinning up local testcontainers when writing vitest tests. See our [tests.md](./tests.md) file for more information. -- /internal-packages/zodworker is the `@internal/zodworker` package that implements a wrapper around graphile-worker that allows us to use zod to validate our background jobs. We are moving away from using graphile-worker as our background job system, replacing it with our own redis-worker package. ## References diff --git a/ailogger-output.log b/ailogger-output.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/coordinator/.env.example b/apps/coordinator/.env.example deleted file mode 100644 index 77377ab3cf..0000000000 --- a/apps/coordinator/.env.example +++ /dev/null @@ -1,4 +0,0 @@ -HTTP_SERVER_PORT=8020 -PLATFORM_ENABLED=true -PLATFORM_WS_PORT=3030 -SECURE_CONNECTION=false \ No newline at end of file diff --git a/apps/coordinator/.gitignore b/apps/coordinator/.gitignore deleted file mode 100644 index 5c84119d63..0000000000 --- a/apps/coordinator/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -dist/ -node_modules/ -.env \ No newline at end of file diff --git a/apps/coordinator/Containerfile b/apps/coordinator/Containerfile deleted file mode 100644 index 4e7b89e0af..0000000000 --- a/apps/coordinator/Containerfile +++ /dev/null @@ -1,60 +0,0 @@ -# syntax=docker/dockerfile:labs - -FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20 - -WORKDIR /app - -FROM node-20 AS pruner - -COPY --chown=node:node . . -RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker -RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - -FROM node-20 AS base - -RUN apt-get update \ - && apt-get install -y buildah ca-certificates dumb-init docker.io busybox \ - && rm -rf /var/lib/apt/lists/* - -COPY --chown=node:node .gitignore .gitignore -COPY --from=pruner --chown=node:node /app/out/json/ . -COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml - -FROM base AS dev-deps -RUN corepack enable -ENV NODE_ENV development - -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile - -FROM base AS builder -RUN corepack enable - -COPY --from=pruner --chown=node:node /app/out/full/ . -COPY --from=dev-deps --chown=node:node /app/ . -COPY --chown=node:node turbo.json turbo.json - -RUN pnpm run -r --filter coordinator build:bundle - -FROM alpine AS cri-tools - -WORKDIR /cri-tools - -ARG CRICTL_VERSION=v1.29.0 -ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb -ADD --checksum=${CRICTL_CHECKSUM} \ - https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz . -RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz - -FROM base AS runner - -RUN corepack enable -ENV NODE_ENV production - -COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin -COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs - -EXPOSE 8000 - -CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/coordinator/README.md b/apps/coordinator/README.md deleted file mode 100644 index fa6da2462b..0000000000 --- a/apps/coordinator/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Coordinator - -Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things. diff --git a/apps/coordinator/package.json b/apps/coordinator/package.json deleted file mode 100644 index 3b4240bd37..0000000000 --- a/apps/coordinator/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "coordinator", - "private": true, - "version": "0.0.1", - "description": "", - "main": "dist/index.cjs", - "scripts": { - "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", - "build:image": "docker build -f Containerfile . -t coordinator", - "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", - "start": "tsx src/index.ts", - "typecheck": "tsc --noEmit" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@trigger.dev/core": "workspace:*", - "nanoid": "^5.0.6", - "prom-client": "^15.1.0", - "socket.io": "4.7.4", - "tinyexec": "^0.3.0" - }, - "devDependencies": { - "dotenv": "^16.4.2", - "esbuild": "^0.19.11", - "tsx": "^4.7.0" - } -} \ No newline at end of file diff --git a/apps/coordinator/src/chaosMonkey.ts b/apps/coordinator/src/chaosMonkey.ts deleted file mode 100644 index e2bc147674..0000000000 --- a/apps/coordinator/src/chaosMonkey.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { setTimeout as timeout } from "node:timers/promises"; - -class ChaosMonkeyError extends Error { - constructor(message: string) { - super(message); - this.name = "ChaosMonkeyError"; - } -} - -export class ChaosMonkey { - private chaosEventRate = 0.2; - private delayInSeconds = 45; - - constructor( - private enabled = false, - private disableErrors = false, - private disableDelays = false - ) { - if (this.enabled) { - console.log("๐ŸŒ Chaos monkey enabled"); - } - } - - static Error = ChaosMonkeyError; - - enable() { - this.enabled = true; - console.log("๐ŸŒ Chaos monkey enabled"); - } - - disable() { - this.enabled = false; - console.log("๐ŸŒ Chaos monkey disabled"); - } - - async call({ - throwErrors = !this.disableErrors, - addDelays = !this.disableDelays, - }: { - throwErrors?: boolean; - addDelays?: boolean; - } = {}) { - if (!this.enabled) { - return; - } - - const random = Math.random(); - - if (random > this.chaosEventRate) { - // Don't interfere with normal operation - return; - } - - const chaosEvents: Array<() => Promise> = []; - - if (addDelays) { - chaosEvents.push(async () => { - console.log("๐ŸŒ Chaos monkey: Add delay"); - - await timeout(this.delayInSeconds * 1000); - }); - } - - if (throwErrors) { - chaosEvents.push(async () => { - console.log("๐ŸŒ Chaos monkey: Throw error"); - - throw new ChaosMonkey.Error("๐ŸŒ Chaos monkey: Throw error"); - }); - } - - if (chaosEvents.length === 0) { - console.error("๐ŸŒ Chaos monkey: No events selected"); - return; - } - - const randomIndex = Math.floor(Math.random() * chaosEvents.length); - - const chaosEvent = chaosEvents[randomIndex]; - - if (!chaosEvent) { - console.error("๐ŸŒ Chaos monkey: No event found"); - return; - } - - await chaosEvent(); - } -} diff --git a/apps/coordinator/src/checkpointer.ts b/apps/coordinator/src/checkpointer.ts deleted file mode 100644 index b5d4b52a25..0000000000 --- a/apps/coordinator/src/checkpointer.ts +++ /dev/null @@ -1,708 +0,0 @@ -import { ExponentialBackoff } from "@trigger.dev/core/v3/apps"; -import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly"; -import { nanoid } from "nanoid"; -import fs from "node:fs/promises"; -import { ChaosMonkey } from "./chaosMonkey"; -import { Buildah, Crictl, Exec } from "./exec"; -import { setTimeout } from "node:timers/promises"; -import { TempFileCleaner } from "./cleaner"; -import { numFromEnv, boolFromEnv } from "./util"; -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; - -type CheckpointerInitializeReturn = { - canCheckpoint: boolean; - willSimulate: boolean; -}; - -type CheckpointAndPushOptions = { - runId: string; - leaveRunning?: boolean; - projectRef: string; - deploymentVersion: string; - shouldHeartbeat?: boolean; - attemptNumber?: number; -}; - -type CheckpointAndPushResult = - | { success: true; checkpoint: CheckpointData } - | { - success: false; - reason?: "CANCELED" | "ERROR" | "SKIP_RETRYING"; - }; - -type CheckpointData = { - location: string; - docker: boolean; -}; - -type CheckpointerOptions = { - dockerMode: boolean; - forceSimulate: boolean; - heartbeat: (runId: string) => void; - registryHost?: string; - registryNamespace?: string; - registryTlsVerify?: boolean; - disableCheckpointSupport?: boolean; - checkpointPath?: string; - simulateCheckpointFailure?: boolean; - simulateCheckpointFailureSeconds?: number; - simulatePushFailure?: boolean; - simulatePushFailureSeconds?: number; - chaosMonkey?: ChaosMonkey; -}; - -async function getFileSize(filePath: string): Promise { - try { - const stats = await fs.stat(filePath); - return stats.size; - } catch (error) { - console.error("Error getting file size:", error); - return -1; - } -} - -async function getParsedFileSize(filePath: string) { - const sizeInBytes = await getFileSize(filePath); - - let message = `Size in bytes: ${sizeInBytes}`; - - if (sizeInBytes > 1024 * 1024) { - const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2); - message = `Size in MB (rounded): ${sizeInMB}`; - } else if (sizeInBytes > 1024) { - const sizeInKB = (sizeInBytes / 1024).toFixed(2); - message = `Size in KB (rounded): ${sizeInKB}`; - } - - return { - path: filePath, - sizeInBytes, - message, - }; -} - -export class Checkpointer { - #initialized = false; - #canCheckpoint = false; - #dockerMode: boolean; - - #logger = new SimpleStructuredLogger("checkpointer"); - - #failedCheckpoints = new Map(); - - // Indexed by run ID - #runAbortControllers = new Map< - string, - { signal: AbortSignal; abort: AbortController["abort"] } - >(); - - private registryHost: string; - private registryNamespace: string; - private registryTlsVerify: boolean; - - private disableCheckpointSupport: boolean; - - private simulateCheckpointFailure: boolean; - private simulateCheckpointFailureSeconds: number; - private simulatePushFailure: boolean; - private simulatePushFailureSeconds: number; - - private chaosMonkey: ChaosMonkey; - private tmpCleaner?: TempFileCleaner; - - constructor(private opts: CheckpointerOptions) { - this.#dockerMode = opts.dockerMode; - - this.registryHost = opts.registryHost ?? "localhost:5000"; - this.registryNamespace = opts.registryNamespace ?? "trigger"; - this.registryTlsVerify = opts.registryTlsVerify ?? true; - - this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false; - - this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false; - this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300; - this.simulatePushFailure = opts.simulatePushFailure ?? false; - this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300; - - this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED); - this.tmpCleaner = this.#createTmpCleaner(); - } - - async init(): Promise { - if (this.#initialized) { - return this.#getInitReturn(this.#canCheckpoint); - } - - this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`); - - if (this.#dockerMode) { - const testCheckpoint = await testDockerCheckpoint(); - - if (testCheckpoint.ok) { - return this.#getInitReturn(true); - } - - this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error }); - return this.#getInitReturn(false); - } - - const canLogin = await Buildah.canLogin(this.registryHost); - - if (!canLogin) { - this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`); - } - - return this.#getInitReturn(canLogin); - } - - #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn { - this.#canCheckpoint = canCheckpoint; - - if (canCheckpoint) { - if (!this.#initialized) { - this.#logger.log("Full checkpoint support!"); - } - } - - this.#initialized = true; - - const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate); - - if (willSimulate) { - this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", { - forceSimulate: this.opts.forceSimulate, - }); - } - - return { - canCheckpoint, - willSimulate, - }; - } - - #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) { - return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`; - } - - #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) { - const basename = `${projectRef}-${deploymentVersion}-${shortCode}`; - - if (this.#dockerMode) { - return basename; - } else { - return Crictl.getExportLocation(basename); - } - } - - async checkpointAndPush( - opts: CheckpointAndPushOptions, - delayMs?: number - ): Promise { - const start = performance.now(); - this.#logger.log(`checkpointAndPush() start`, { start, opts }); - - const { runId } = opts; - - let interval: NodeJS.Timer | undefined; - if (opts.shouldHeartbeat) { - interval = setInterval(() => { - this.#logger.log("Sending heartbeat", { runId }); - this.opts.heartbeat(runId); - }, 20_000); - } - - const controller = new AbortController(); - const signal = controller.signal; - const abort = controller.abort.bind(controller); - - const onAbort = () => { - this.#logger.error("Checkpoint aborted", { runId, options: opts }); - }; - - signal.addEventListener("abort", onAbort, { once: true }); - - const removeCurrentAbortController = () => { - const controller = this.#runAbortControllers.get(runId); - - // Ensure only the current controller is removed - if (controller && controller.signal === signal) { - this.#runAbortControllers.delete(runId); - } - - // Remove the abort listener in case it hasn't fired - signal.removeEventListener("abort", onAbort); - }; - - if (!this.#dockerMode && !this.#canCheckpoint) { - this.#logger.error("No checkpoint support. Simulation requires docker."); - this.#failCheckpoint(runId, "NO_SUPPORT"); - return; - } - - if (this.#isRunCheckpointing(runId)) { - this.#logger.error("Checkpoint procedure already in progress", { options: opts }); - this.#failCheckpoint(runId, "IN_PROGRESS"); - return; - } - - // This is a new checkpoint, clear any last failure for this run - this.#clearFailedCheckpoint(runId); - - if (this.disableCheckpointSupport) { - this.#logger.error("Checkpoint support disabled", { options: opts }); - this.#failCheckpoint(runId, "DISABLED"); - return; - } - - this.#runAbortControllers.set(runId, { signal, abort }); - - try { - const result = await this.#checkpointAndPushWithBackoff(opts, { delayMs, signal }); - - const end = performance.now(); - this.#logger.log(`checkpointAndPush() end`, { - start, - end, - diff: end - start, - diffWithoutDelay: end - start - (delayMs ?? 0), - opts, - success: result.success, - delayMs, - }); - - if (!result.success) { - return; - } - - return result.checkpoint; - } finally { - if (opts.shouldHeartbeat) { - // @ts-ignore - Some kind of node incompatible type issue - clearInterval(interval); - } - removeCurrentAbortController(); - } - } - - #isRunCheckpointing(runId: string) { - return this.#runAbortControllers.has(runId); - } - - cancelAllCheckpointsForRun(runId: string): boolean { - this.#logger.log("cancelAllCheckpointsForRun: call", { runId }); - - // If the last checkpoint failed, pretend we canceled it - // This ensures tasks don't wait for external resume messages to continue - if (this.#hasFailedCheckpoint(runId)) { - this.#logger.log("cancelAllCheckpointsForRun: hasFailedCheckpoint", { runId }); - this.#clearFailedCheckpoint(runId); - return true; - } - - const controller = this.#runAbortControllers.get(runId); - - if (!controller) { - this.#logger.debug("cancelAllCheckpointsForRun: no abort controller", { runId }); - return false; - } - - const { abort, signal } = controller; - - if (signal.aborted) { - this.#logger.debug("cancelAllCheckpointsForRun: signal already aborted", { runId }); - return false; - } - - abort("cancelCheckpoint()"); - this.#runAbortControllers.delete(runId); - - return true; - } - - async #checkpointAndPushWithBackoff( - { - runId, - leaveRunning = true, // This mirrors kubernetes behaviour more accurately - projectRef, - deploymentVersion, - attemptNumber, - }: CheckpointAndPushOptions, - { delayMs, signal }: { delayMs?: number; signal: AbortSignal } - ): Promise { - if (delayMs && delayMs > 0) { - this.#logger.log("Delaying checkpoint", { runId, delayMs }); - - try { - await setTimeout(delayMs, undefined, { signal }); - } catch (error) { - this.#logger.log("Checkpoint canceled during initial delay", { runId }); - return { success: false, reason: "CANCELED" }; - } - } - - this.#logger.log("Checkpointing with backoff", { - runId, - leaveRunning, - projectRef, - deploymentVersion, - }); - - const backoff = new ExponentialBackoff() - .type("EqualJitter") - .base(3) - .max(3 * 3600) - .maxElapsed(48 * 3600); - - for await (const { delay, retry } of backoff) { - try { - if (retry > 0) { - this.#logger.error("Retrying checkpoint", { - runId, - retry, - delay, - }); - - try { - await setTimeout(delay.milliseconds, undefined, { signal }); - } catch (error) { - this.#logger.log("Checkpoint canceled during retry delay", { runId }); - return { success: false, reason: "CANCELED" }; - } - } - - const result = await this.#checkpointAndPush( - { - runId, - leaveRunning, - projectRef, - deploymentVersion, - attemptNumber, - }, - { signal } - ); - - if (result.success) { - return result; - } - - if (result.reason === "CANCELED") { - this.#logger.log("Checkpoint canceled, won't retry", { runId }); - // Don't fail the checkpoint, as it was canceled - return result; - } - - if (result.reason === "SKIP_RETRYING") { - this.#logger.log("Skipping retrying", { runId }); - return result; - } - - continue; - } catch (error) { - this.#logger.error("Checkpoint error", { - retry, - runId, - delay, - error: error instanceof Error ? error.message : error, - }); - } - } - - this.#logger.error(`Checkpoint failed after exponential backoff`, { - runId, - leaveRunning, - projectRef, - deploymentVersion, - }); - this.#failCheckpoint(runId, "ERROR"); - - return { success: false, reason: "ERROR" }; - } - - async #checkpointAndPush( - { - runId, - leaveRunning = true, // This mirrors kubernetes behaviour more accurately - projectRef, - deploymentVersion, - attemptNumber, - }: CheckpointAndPushOptions, - { signal }: { signal: AbortSignal } - ): Promise { - await this.init(); - - const options = { - runId, - leaveRunning, - projectRef, - deploymentVersion, - attemptNumber, - }; - - const shortCode = nanoid(8); - const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode); - const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode); - - const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: signal }); - const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: signal }); - - const cleanup = async () => { - const metadata = { - runId, - exportLocation, - imageRef, - }; - - if (this.#dockerMode) { - this.#logger.debug("Skipping cleanup in docker mode", metadata); - return; - } - - this.#logger.log("Cleaning up", metadata); - - try { - await buildah.cleanup(); - await crictl.cleanup(); - } catch (error) { - this.#logger.error("Error during cleanup", { ...metadata, error }); - } - }; - - try { - await this.chaosMonkey.call(); - - this.#logger.log("checkpointAndPush: checkpointing", { options }); - - const containterName = this.#getRunContainerName(runId); - - // Create checkpoint (docker) - if (this.#dockerMode) { - await this.#createDockerCheckpoint( - signal, - runId, - exportLocation, - leaveRunning, - attemptNumber - ); - - this.#logger.log("checkpointAndPush: checkpoint created", { - runId, - location: exportLocation, - }); - - return { - success: true, - checkpoint: { - location: exportLocation, - docker: true, - }, - }; - } - - // Create checkpoint (CRI) - if (!this.#canCheckpoint) { - this.#logger.error("No checkpoint support in kubernetes mode."); - return { success: false, reason: "SKIP_RETRYING" }; - } - - const containerId = await crictl.ps(containterName, true); - - if (!containerId.stdout) { - this.#logger.error("could not find container id", { options, containterName }); - return { success: false, reason: "SKIP_RETRYING" }; - } - - const start = performance.now(); - - if (this.simulateCheckpointFailure) { - if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) { - this.#logger.error("Simulating checkpoint failure", { options }); - throw new Error("SIMULATE_CHECKPOINT_FAILURE"); - } - } - - // Create checkpoint - await crictl.checkpoint(containerId.stdout, exportLocation); - const postCheckpoint = performance.now(); - - // Print checkpoint size - const size = await getParsedFileSize(exportLocation); - this.#logger.log("checkpoint archive created", { size, options }); - - // Create image from checkpoint - const workingContainer = await buildah.from("scratch"); - const postFrom = performance.now(); - - await buildah.add(workingContainer.stdout, exportLocation, "/"); - const postAdd = performance.now(); - - await buildah.config(workingContainer.stdout, [ - `io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`, - ]); - const postConfig = performance.now(); - - await buildah.commit(workingContainer.stdout, imageRef); - const postCommit = performance.now(); - - if (this.simulatePushFailure) { - if (performance.now() < this.simulatePushFailureSeconds * 1000) { - this.#logger.error("Simulating push failure", { options }); - throw new Error("SIMULATE_PUSH_FAILURE"); - } - } - - // Push checkpoint image - await buildah.push(imageRef, this.registryTlsVerify); - const postPush = performance.now(); - - const perf = { - "crictl checkpoint": postCheckpoint - start, - "buildah from": postFrom - postCheckpoint, - "buildah add": postAdd - postFrom, - "buildah config": postConfig - postAdd, - "buildah commit": postCommit - postConfig, - "buildah push": postPush - postCommit, - }; - - this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf }); - - return { - success: true, - checkpoint: { - location: imageRef, - docker: false, - }, - }; - } catch (error) { - if (error instanceof Exec.Result) { - if (error.aborted) { - this.#logger.error("Checkpoint canceled: Exec", { options }); - - return { success: false, reason: "CANCELED" }; - } else { - this.#logger.error("Checkpoint command error", { options, error }); - - return { success: false, reason: "ERROR" }; - } - } - - this.#logger.error("Unhandled checkpoint error", { - options, - error: error instanceof Error ? error.message : error, - }); - - return { success: false, reason: "ERROR" }; - } finally { - await cleanup(); - - if (signal.aborted) { - this.#logger.error("Checkpoint canceled: Cleanup", { options }); - - // Overrides any prior return value - return { success: false, reason: "CANCELED" }; - } - } - } - - async unpause(runId: string, attemptNumber?: number): Promise { - try { - const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber); - const exec = new Exec({ logger: this.#logger }); - await exec.x("docker", ["unpause", containterNameWithAttempt]); - } catch (error) { - this.#logger.error("[Docker] Error during unpause", { runId, attemptNumber, error }); - } - } - - async #createDockerCheckpoint( - abortSignal: AbortSignal, - runId: string, - exportLocation: string, - leaveRunning: boolean, - attemptNumber?: number - ) { - const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber); - const exec = new Exec({ logger: this.#logger, abortSignal }); - - try { - if (this.opts.forceSimulate || !this.#canCheckpoint) { - this.#logger.log("Simulating checkpoint"); - - await exec.x("docker", ["pause", containterNameWithAttempt]); - - return; - } - - if (this.simulateCheckpointFailure) { - if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) { - this.#logger.error("Simulating checkpoint failure", { - runId, - exportLocation, - leaveRunning, - attemptNumber, - }); - - throw new Error("SIMULATE_CHECKPOINT_FAILURE"); - } - } - - const args = ["checkpoint", "create"]; - - if (leaveRunning) { - args.push("--leave-running"); - } - - args.push(containterNameWithAttempt, exportLocation); - - await exec.x("docker", args); - } catch (error) { - this.#logger.error("Failed while creating docker checkpoint", { exportLocation }); - throw error; - } - } - - #failCheckpoint(runId: string, error: unknown) { - this.#failedCheckpoints.set(runId, error); - } - - #clearFailedCheckpoint(runId: string) { - this.#failedCheckpoints.delete(runId); - } - - #hasFailedCheckpoint(runId: string) { - return this.#failedCheckpoints.has(runId); - } - - #getRunContainerName(suffix: string, attemptNumber?: number) { - return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`; - } - - #createTmpCleaner() { - if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) { - return; - } - - const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean); - const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? []; - const paths = pathsOverride.length ? pathsOverride : defaultPaths; - - if (paths.length === 0) { - this.#logger.error("TempFileCleaner enabled but no paths to clean", { - defaultPaths, - pathsOverride, - TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE, - }); - - return; - } - const cleaner = new TempFileCleaner({ - paths, - maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60), - intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300), - leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false), - }); - - cleaner.start(); - - return cleaner; - } -} diff --git a/apps/coordinator/src/cleaner.ts b/apps/coordinator/src/cleaner.ts deleted file mode 100644 index 58cfd24bb7..0000000000 --- a/apps/coordinator/src/cleaner.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import { Exec } from "./exec"; -import { setTimeout } from "timers/promises"; - -interface TempFileCleanerOptions { - paths: string[]; - maxAgeMinutes: number; - intervalSeconds: number; - leadingEdge?: boolean; -} - -export class TempFileCleaner { - private enabled = false; - - private logger: SimpleStructuredLogger; - private exec: Exec; - - constructor(private opts: TempFileCleanerOptions) { - this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts }); - this.exec = new Exec({ logger: this.logger }); - } - - async start() { - this.logger.log("TempFileCleaner.start"); - this.enabled = true; - - if (!this.opts.leadingEdge) { - await this.wait(); - } - - while (this.enabled) { - try { - await this.clean(); - } catch (error) { - this.logger.error("error during tick", { error }); - } - - await this.wait(); - } - } - - stop() { - this.logger.log("TempFileCleaner.stop"); - this.enabled = false; - } - - private wait() { - return setTimeout(this.opts.intervalSeconds * 1000); - } - - private async clean() { - for (const path of this.opts.paths) { - try { - await this.cleanSingle(path); - } catch (error) { - this.logger.error("error while cleaning", { path, error }); - } - } - } - - private async cleanSingle(startingPoint: string) { - const maxAgeMinutes = this.opts.maxAgeMinutes; - - const ignoreStartingPoint = ["!", "-path", startingPoint]; - const onlyDirectDescendants = ["-maxdepth", "1"]; - const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`]; - - const baseArgs = [ - startingPoint, - ...ignoreStartingPoint, - ...onlyDirectDescendants, - ...onlyOldFiles, - ]; - - const duArgs = ["-exec", "du", "-ch", "{}", "+"]; - const rmArgs = ["-exec", "rm", "-rf", "{}", "+"]; - - const du = this.x("find", [...baseArgs, ...duArgs]); - const duOutput = await du; - - const duLines = duOutput.stdout.trim().split("\n"); - const fileCount = duLines.length - 1; // last line is the total - const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0]; - - if (fileCount === 0) { - this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes }); - return; - } - - this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes }); - - const rm = this.x("find", [...baseArgs, ...rmArgs]); - const rmOutput = await rm; - - if (rmOutput.stderr.length > 0) { - this.logger.error("delete unsuccessful", { rmOutput }); - return; - } - - this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes }); - } - - private get x() { - return this.exec.x.bind(this.exec); - } -} diff --git a/apps/coordinator/src/exec.ts b/apps/coordinator/src/exec.ts deleted file mode 100644 index b905723c0f..0000000000 --- a/apps/coordinator/src/exec.ts +++ /dev/null @@ -1,293 +0,0 @@ -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import { randomUUID } from "crypto"; -import { homedir } from "os"; -import { type Result, x } from "tinyexec"; - -class TinyResult { - pid?: number; - exitCode?: number; - aborted: boolean; - killed: boolean; - - constructor(result: Result) { - this.pid = result.pid; - this.exitCode = result.exitCode; - this.aborted = result.aborted; - this.killed = result.killed; - } -} - -interface ExecOptions { - logger?: SimpleStructuredLogger; - abortSignal?: AbortSignal; - logOutput?: boolean; - trimArgs?: boolean; - neverThrow?: boolean; -} - -export class Exec { - private logger: SimpleStructuredLogger; - private abortSignal: AbortSignal | undefined; - - private logOutput: boolean; - private trimArgs: boolean; - private neverThrow: boolean; - - constructor(opts: ExecOptions) { - this.logger = opts.logger ?? new SimpleStructuredLogger("exec"); - this.abortSignal = opts.abortSignal; - - this.logOutput = opts.logOutput ?? true; - this.trimArgs = opts.trimArgs ?? true; - this.neverThrow = opts.neverThrow ?? false; - } - - async x( - command: string, - args?: string[], - opts?: { neverThrow?: boolean; ignoreAbort?: boolean } - ) { - const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args; - - const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`; - this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed }); - - const result = x(command, argsTrimmed, { - signal: opts?.ignoreAbort ? undefined : this.abortSignal, - // We don't use this as it doesn't cover killed and aborted processes - // throwOnError: true, - }); - - const output = await result; - - const metadata = { - command, - argsRaw: args, - argsTrimmed, - globalOpts: { - trimArgs: this.trimArgs, - neverThrow: this.neverThrow, - hasAbortSignal: !!this.abortSignal, - }, - localOpts: opts, - stdout: output.stdout, - stderr: output.stderr, - pid: result.pid, - exitCode: result.exitCode, - aborted: result.aborted, - killed: result.killed, - }; - - if (this.logOutput) { - this.logger.debug(`output: ${commandWithFirstArg}`, metadata); - } - - if (this.neverThrow || opts?.neverThrow) { - return output; - } - - if (result.aborted) { - this.logger.error(`aborted: ${commandWithFirstArg}`, metadata); - throw new TinyResult(result); - } - - if (result.killed) { - this.logger.error(`killed: ${commandWithFirstArg}`, metadata); - throw new TinyResult(result); - } - - if (result.exitCode !== 0) { - this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata); - throw new TinyResult(result); - } - - return output; - } - - static Result = TinyResult; -} - -interface BuildahOptions { - id?: string; - abortSignal?: AbortSignal; -} - -export class Buildah { - private id: string; - private logger: SimpleStructuredLogger; - private exec: Exec; - - private containers = new Set(); - private images = new Set(); - - constructor(opts: BuildahOptions) { - this.id = opts.id ?? randomUUID(); - this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id }); - - this.exec = new Exec({ - logger: this.logger, - abortSignal: opts.abortSignal, - }); - - this.logger.log("initiaized", { opts }); - } - - private get x() { - return this.exec.x.bind(this.exec); - } - - async from(baseImage: string) { - const output = await this.x("buildah", ["from", baseImage]); - this.containers.add(output.stdout); - return output; - } - - async add(container: string, src: string, dest: string) { - return await this.x("buildah", ["add", container, src, dest]); - } - - async config(container: string, annotations: string[]) { - const args = ["config"]; - - for (const annotation of annotations) { - args.push(`--annotation=${annotation}`); - } - - args.push(container); - - return await this.x("buildah", args); - } - - async commit(container: string, imageRef: string) { - const output = await this.x("buildah", ["commit", container, imageRef]); - this.images.add(output.stdout); - return output; - } - - async push(imageRef: string, registryTlsVerify?: boolean) { - return await this.x("buildah", [ - "push", - `--tls-verify=${String(!!registryTlsVerify)}`, - imageRef, - ]); - } - - async cleanup() { - if (this.containers.size > 0) { - try { - const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true }); - this.containers.clear(); - - if (output.stderr.length > 0) { - this.logger.error("failed to remove some containers", { output }); - } - } catch (error) { - this.logger.error("failed to clean up containers", { error, containers: this.containers }); - } - } else { - this.logger.debug("no containers to clean up"); - } - - if (this.images.size > 0) { - try { - const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true }); - this.images.clear(); - - if (output.stderr.length > 0) { - this.logger.error("failed to remove some images", { output }); - } - } catch (error) { - this.logger.error("failed to clean up images", { error, images: this.images }); - } - } else { - this.logger.debug("no images to clean up"); - } - } - - static async canLogin(registryHost: string) { - try { - await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true }); - return true; - } catch (error) { - return false; - } - } - - static get tmpDir() { - return process.env.TMPDIR ?? "/var/tmp"; - } - - static get storageRootDir() { - return process.getuid?.() === 0 - ? "/var/lib/containers/storage" - : `${homedir()}/.local/share/containers/storage`; - } -} - -interface CrictlOptions { - id?: string; - abortSignal?: AbortSignal; -} - -export class Crictl { - private id: string; - private logger: SimpleStructuredLogger; - private exec: Exec; - - private archives = new Set(); - - constructor(opts: CrictlOptions) { - this.id = opts.id ?? randomUUID(); - this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id }); - - this.exec = new Exec({ - logger: this.logger, - abortSignal: opts.abortSignal, - }); - - this.logger.log("initiaized", { opts }); - } - - private get x() { - return this.exec.x.bind(this.exec); - } - - async ps(containerName: string, quiet?: boolean) { - return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]); - } - - async checkpoint(containerId: string, exportLocation: string) { - const output = await this.x("crictl", [ - "checkpoint", - `--export=${exportLocation}`, - containerId, - ]); - this.archives.add(exportLocation); - return output; - } - - async cleanup() { - if (this.archives.size > 0) { - try { - const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true }); - this.archives.clear(); - - if (output.stderr.length > 0) { - this.logger.error("failed to remove some archives", { output }); - } - } catch (error) { - this.logger.error("failed to clean up archives", { error, archives: this.archives }); - } - } else { - this.logger.debug("no archives to clean up"); - } - } - - static getExportLocation(identifier: string) { - return `${this.checkpointDir}/${identifier}.tar`; - } - - static get checkpointDir() { - return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints"; - } -} diff --git a/apps/coordinator/src/index.ts b/apps/coordinator/src/index.ts deleted file mode 100644 index 815012fe04..0000000000 --- a/apps/coordinator/src/index.ts +++ /dev/null @@ -1,1781 +0,0 @@ -import { createServer } from "node:http"; -import { Server } from "socket.io"; -import { - CoordinatorToPlatformMessages, - CoordinatorToProdWorkerMessages, - omit, - PlatformToCoordinatorMessages, - ProdWorkerSocketData, - ProdWorkerToCoordinatorMessages, - WaitReason, -} from "@trigger.dev/core/v3"; -import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace"; -import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket"; -import { ExponentialBackoff, HttpReply, getTextBody } from "@trigger.dev/core/v3/apps"; -import { ChaosMonkey } from "./chaosMonkey"; -import { Checkpointer } from "./checkpointer"; -import { boolFromEnv, numFromEnv, safeJsonParse } from "./util"; - -import { collectDefaultMetrics, register, Gauge } from "prom-client"; -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -collectDefaultMetrics(); - -const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020); -const NODE_NAME = process.env.NODE_NAME || "coordinator"; -const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000; - -const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ?? "true"); -const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1"; -const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030; -const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret"; -const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false"); - -const TASK_RUN_COMPLETED_WITH_ACK_TIMEOUT_MS = - parseInt(process.env.TASK_RUN_COMPLETED_WITH_ACK_TIMEOUT_MS || "") || 30_000; -const TASK_RUN_COMPLETED_WITH_ACK_MAX_RETRIES = - parseInt(process.env.TASK_RUN_COMPLETED_WITH_ACK_MAX_RETRIES || "") || 7; - -const WAIT_FOR_TASK_CHECKPOINT_DELAY_MS = - parseInt(process.env.WAIT_FOR_TASK_CHECKPOINT_DELAY_MS || "") || 0; -const WAIT_FOR_BATCH_CHECKPOINT_DELAY_MS = - parseInt(process.env.WAIT_FOR_BATCH_CHECKPOINT_DELAY_MS || "") || 0; - -const logger = new SimpleStructuredLogger("coordinator", undefined, { nodeName: NODE_NAME }); -const chaosMonkey = new ChaosMonkey( - !!process.env.CHAOS_MONKEY_ENABLED, - !!process.env.CHAOS_MONKEY_DISABLE_ERRORS, - !!process.env.CHAOS_MONKEY_DISABLE_DELAYS -); - -class CheckpointReadinessTimeoutError extends Error {} -class CheckpointCancelError extends Error {} - -class TaskCoordinator { - #httpServer: ReturnType; - #internalHttpServer: ReturnType; - - #checkpointer = new Checkpointer({ - dockerMode: !process.env.KUBERNETES_PORT, - forceSimulate: boolFromEnv("FORCE_CHECKPOINT_SIMULATION", false), - heartbeat: this.#sendRunHeartbeat.bind(this), - registryHost: process.env.REGISTRY_HOST, - registryNamespace: process.env.REGISTRY_NAMESPACE, - registryTlsVerify: boolFromEnv("REGISTRY_TLS_VERIFY", true), - disableCheckpointSupport: boolFromEnv("DISABLE_CHECKPOINT_SUPPORT", false), - simulatePushFailure: boolFromEnv("SIMULATE_PUSH_FAILURE", false), - simulatePushFailureSeconds: numFromEnv("SIMULATE_PUSH_FAILURE_SECONDS", 300), - simulateCheckpointFailure: boolFromEnv("SIMULATE_CHECKPOINT_FAILURE", false), - simulateCheckpointFailureSeconds: numFromEnv("SIMULATE_CHECKPOINT_FAILURE_SECONDS", 300), - chaosMonkey, - }); - - #prodWorkerNamespace?: ZodNamespace< - typeof ProdWorkerToCoordinatorMessages, - typeof CoordinatorToProdWorkerMessages, - typeof ProdWorkerSocketData - >; - #platformSocket?: ZodSocketConnection< - typeof CoordinatorToPlatformMessages, - typeof PlatformToCoordinatorMessages - >; - - #checkpointableTasks = new Map< - string, - { resolve: (value: void) => void; reject: (err?: any) => void } - >(); - - #delayThresholdInMs: number = DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS; - - constructor( - private port: number, - private host = "0.0.0.0" - ) { - this.#httpServer = this.#createHttpServer(); - this.#internalHttpServer = this.#createInternalHttpServer(); - - this.#checkpointer.init(); - this.#platformSocket = this.#createPlatformSocket(); - - const connectedTasksTotal = new Gauge({ - name: "daemon_connected_tasks_total", // don't change this without updating dashboard config - help: "The number of tasks currently connected.", - collect: () => { - connectedTasksTotal.set(this.#prodWorkerNamespace?.namespace.sockets.size ?? 0); - }, - }); - register.registerMetric(connectedTasksTotal); - } - - #returnValidatedExtraHeaders(headers: Record) { - for (const [key, value] of Object.entries(headers)) { - if (value === undefined) { - throw new Error(`Extra header is undefined: ${key}`); - } - } - - return headers; - } - - // MARK: SOCKET: PLATFORM - #createPlatformSocket() { - if (!PLATFORM_ENABLED) { - logger.log("INFO: platform connection disabled"); - return; - } - - const extraHeaders = this.#returnValidatedExtraHeaders({ - "x-supports-dynamic-config": "yes", - }); - - const host = PLATFORM_HOST; - const port = Number(PLATFORM_WS_PORT); - - const platformLogger = new SimpleStructuredLogger("socket-platform", undefined, { - namespace: "coordinator", - }); - - platformLogger.log("connecting", { host, port }); - platformLogger.debug("connecting with extra headers", { extraHeaders }); - - const platformConnection = new ZodSocketConnection({ - namespace: "coordinator", - host, - port, - secure: SECURE_CONNECTION, - extraHeaders, - clientMessages: CoordinatorToPlatformMessages, - serverMessages: PlatformToCoordinatorMessages, - authToken: PLATFORM_SECRET, - logHandlerPayloads: false, - handlers: { - // This is used by resumeAttempt - RESUME_AFTER_DEPENDENCY: async (message) => { - const log = platformLogger.child({ - eventName: "RESUME_AFTER_DEPENDENCY", - ...omit(message, "completions", "executions"), - completions: message.completions.map((c) => ({ - id: c.id, - ok: c.ok, - })), - executions: message.executions.length, - }); - - log.log("Handling RESUME_AFTER_DEPENDENCY"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - log.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for RESUME_AFTER_DEPENDENCY"); - - await chaosMonkey.call(); - - // In case the task resumes before the checkpoint is created - this.#cancelCheckpoint(message.runId, { - event: "RESUME_AFTER_DEPENDENCY", - completions: message.completions.length, - }); - - taskSocket.emit("RESUME_AFTER_DEPENDENCY", message); - }, - // This is used by sharedQueueConsumer - RESUME_AFTER_DEPENDENCY_WITH_ACK: async (message) => { - const log = platformLogger.child({ - eventName: "RESUME_AFTER_DEPENDENCY_WITH_ACK", - ...omit(message, "completions", "executions"), - completions: message.completions.map((c) => ({ - id: c.id, - ok: c.ok, - })), - executions: message.executions.length, - }); - - log.log("Handling RESUME_AFTER_DEPENDENCY_WITH_ACK"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - log.debug("Socket for attempt not found"); - return { - success: false, - error: { - name: "SocketNotFoundError", - message: "Socket for attempt not found", - }, - }; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for RESUME_AFTER_DEPENDENCY_WITH_ACK"); - - //if this is set, we want to kill the process because it will be resumed with the checkpoint from the queue - if (taskSocket.data.requiresCheckpointResumeWithMessage) { - log.log("RESUME_AFTER_DEPENDENCY_WITH_ACK: Checkpoint is set so going to nack"); - - return { - success: false, - error: { - name: "CheckpointMessagePresentError", - message: - "Checkpoint message is present, so we need to kill the process and resume from the queue.", - }, - }; - } - - await chaosMonkey.call(); - - // In case the task resumes before the checkpoint is created - this.#cancelCheckpoint(message.runId, { - event: "RESUME_AFTER_DEPENDENCY_WITH_ACK", - completions: message.completions.length, - }); - - taskSocket.emit("RESUME_AFTER_DEPENDENCY", message); - - return { - success: true, - }; - }, - RESUME_AFTER_DURATION: async (message) => { - const log = platformLogger.child({ - eventName: "RESUME_AFTER_DURATION", - ...message, - }); - - log.log("Handling RESUME_AFTER_DURATION"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - log.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for RESUME_AFTER_DURATION"); - - await chaosMonkey.call(); - - taskSocket.emit("RESUME_AFTER_DURATION", message); - }, - REQUEST_ATTEMPT_CANCELLATION: async (message) => { - const log = platformLogger.child({ - eventName: "REQUEST_ATTEMPT_CANCELLATION", - ...message, - }); - - log.log("Handling REQUEST_ATTEMPT_CANCELLATION"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - logger.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for REQUEST_ATTEMPT_CANCELLATION"); - - taskSocket.emit("REQUEST_ATTEMPT_CANCELLATION", message); - }, - REQUEST_RUN_CANCELLATION: async (message) => { - const log = platformLogger.child({ - eventName: "REQUEST_RUN_CANCELLATION", - ...message, - }); - - log.log("Handling REQUEST_RUN_CANCELLATION"); - - const taskSocket = await this.#getRunSocket(message.runId); - - if (!taskSocket) { - logger.debug("Socket for run not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for REQUEST_RUN_CANCELLATION"); - - this.#cancelCheckpoint(message.runId, { event: "REQUEST_RUN_CANCELLATION", ...message }); - - if (message.delayInMs) { - taskSocket.emit("REQUEST_EXIT", { - version: "v2", - delayInMs: message.delayInMs, - }); - } else { - // If there's no delay, assume the worker doesn't support non-v1 messages - taskSocket.emit("REQUEST_EXIT", { - version: "v1", - }); - } - }, - READY_FOR_RETRY: async (message) => { - const log = platformLogger.child({ - eventName: "READY_FOR_RETRY", - ...message, - }); - - const taskSocket = await this.#getRunSocket(message.runId); - - if (!taskSocket) { - logger.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for READY_FOR_RETRY"); - - await chaosMonkey.call(); - - taskSocket.emit("READY_FOR_RETRY", message); - }, - DYNAMIC_CONFIG: async (message) => { - const log = platformLogger.child({ - eventName: "DYNAMIC_CONFIG", - ...message, - }); - - log.log("Handling DYNAMIC_CONFIG"); - - this.#delayThresholdInMs = message.checkpointThresholdInMs; - - // The first time we receive a dynamic config, the worker namespace will be created - if (!this.#prodWorkerNamespace) { - const io = new Server(this.#httpServer); - this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io); - } - }, - }, - }); - - return platformConnection; - } - - async #getRunSocket(runId: string) { - const sockets = (await this.#prodWorkerNamespace?.fetchSockets()) ?? []; - - for (const socket of sockets) { - if (socket.data.runId === runId) { - return socket; - } - } - } - - async #getAttemptSocket(attemptFriendlyId: string) { - const sockets = (await this.#prodWorkerNamespace?.fetchSockets()) ?? []; - - for (const socket of sockets) { - if (socket.data.attemptFriendlyId === attemptFriendlyId) { - return socket; - } - } - } - - // MARK: SOCKET: WORKERS - #createProdWorkerNamespace(io: Server) { - const provider = new ZodNamespace({ - io, - name: "prod-worker", - clientMessages: ProdWorkerToCoordinatorMessages, - serverMessages: CoordinatorToProdWorkerMessages, - socketData: ProdWorkerSocketData, - postAuth: async (socket, next, logger) => { - function setSocketDataFromHeader( - dataKey: keyof typeof socket.data, - headerName: string, - required: boolean = true - ) { - const value = socket.handshake.headers[headerName]; - - if (value) { - socket.data[dataKey] = Array.isArray(value) ? value[0] : value; - return; - } - - if (required) { - logger.error("missing required header", { headerName }); - throw new Error("missing header"); - } - } - - try { - setSocketDataFromHeader("podName", "x-pod-name"); - setSocketDataFromHeader("contentHash", "x-trigger-content-hash"); - setSocketDataFromHeader("projectRef", "x-trigger-project-ref"); - setSocketDataFromHeader("runId", "x-trigger-run-id"); - setSocketDataFromHeader("attemptFriendlyId", "x-trigger-attempt-friendly-id", false); - setSocketDataFromHeader("attemptNumber", "x-trigger-attempt-number", false); - setSocketDataFromHeader("envId", "x-trigger-env-id"); - setSocketDataFromHeader("deploymentId", "x-trigger-deployment-id"); - setSocketDataFromHeader("deploymentVersion", "x-trigger-deployment-version"); - } catch (error) { - logger.error("setSocketDataFromHeader error", { error }); - socket.disconnect(true); - return; - } - - logger.debug("success", socket.data); - - next(); - }, - onConnection: async (socket, handler, sender) => { - const logger = new SimpleStructuredLogger("ns-prod-worker", undefined, { - namespace: "prod-worker", - socketId: socket.id, - socketData: socket.data, - }); - - const getSocketMetadata = () => { - return { - attemptFriendlyId: socket.data.attemptFriendlyId, - attemptNumber: socket.data.attemptNumber, - requiresCheckpointResumeWithMessage: socket.data.requiresCheckpointResumeWithMessage, - }; - }; - - const getAttemptNumber = () => { - return socket.data.attemptNumber ? parseInt(socket.data.attemptNumber) : undefined; - }; - - const exitRun = () => { - logger.log("exitRun", getSocketMetadata()); - - socket.emit("REQUEST_EXIT", { - version: "v1", - }); - }; - - const crashRun = async (error: { name: string; message: string; stack?: string }) => { - logger.error("crashRun", { ...getSocketMetadata(), error }); - - try { - this.#platformSocket?.send("RUN_CRASHED", { - version: "v1", - runId: socket.data.runId, - error, - }); - } finally { - exitRun(); - } - }; - - const checkpointInProgress = () => { - return this.#checkpointableTasks.has(socket.data.runId); - }; - - const readyToCheckpoint = async ( - reason: WaitReason | "RETRY" - ): Promise< - | { - success: true; - } - | { - success: false; - reason?: string; - } - > => { - const log = logger.child(getSocketMetadata()); - - log.log("readyToCheckpoint", { runId: socket.data.runId, reason }); - - if (checkpointInProgress()) { - return { - success: false, - reason: "checkpoint in progress", - }; - } - - let timeout: NodeJS.Timeout | undefined = undefined; - - const CHECKPOINTABLE_TIMEOUT_SECONDS = 20; - - const isCheckpointable = new Promise((resolve, reject) => { - // We set a reasonable timeout to prevent waiting forever - timeout = setTimeout( - () => reject(new CheckpointReadinessTimeoutError()), - CHECKPOINTABLE_TIMEOUT_SECONDS * 1000 - ); - - this.#checkpointableTasks.set(socket.data.runId, { resolve, reject }); - }); - - try { - await isCheckpointable; - this.#checkpointableTasks.delete(socket.data.runId); - - return { - success: true, - }; - } catch (error) { - log.error("Error while waiting for checkpointable state", { error }); - - if (error instanceof CheckpointReadinessTimeoutError) { - logger.error( - `Failed to become checkpointable in ${CHECKPOINTABLE_TIMEOUT_SECONDS}s for ${reason}`, - { runId: socket.data.runId } - ); - - return { - success: false, - reason: "timeout", - }; - } - - if (error instanceof CheckpointCancelError) { - return { - success: false, - reason: "canceled", - }; - } - - return { - success: false, - reason: typeof error === "string" ? error : "unknown", - }; - } finally { - clearTimeout(timeout); - } - }; - - const updateAttemptFriendlyId = (attemptFriendlyId: string) => { - socket.data.attemptFriendlyId = attemptFriendlyId; - }; - - const updateAttemptNumber = (attemptNumber: string | number) => { - socket.data.attemptNumber = String(attemptNumber); - }; - - this.#platformSocket?.send("LOG", { - metadata: socket.data, - text: "connected", - }); - - socket.on("TEST", (message, callback) => { - logger.log("Handling TEST", { eventName: "TEST", ...getSocketMetadata(), ...message }); - - try { - callback(); - } catch (error) { - logger.error("TEST error", { error }); - } - }); - - // Deprecated: Only workers without support for lazy attempts use this - socket.on("READY_FOR_EXECUTION", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_EXECUTION", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_EXECUTION"); - - try { - const executionAck = await this.#platformSocket?.sendWithAck( - "READY_FOR_EXECUTION", - message - ); - - if (!executionAck) { - log.error("no execution ack"); - - await crashRun({ - name: "ReadyForExecutionError", - message: "No execution ack", - }); - - return; - } - - if (!executionAck.success) { - log.error("failed to get execution payload"); - - await crashRun({ - name: "ReadyForExecutionError", - message: "Failed to get execution payload", - }); - - return; - } - - socket.emit("EXECUTE_TASK_RUN", { - version: "v1", - executionPayload: executionAck.payload, - }); - - updateAttemptFriendlyId(executionAck.payload.execution.attempt.id); - updateAttemptNumber(executionAck.payload.execution.attempt.number); - } catch (error) { - log.error("READY_FOR_EXECUTION error", { error }); - - await crashRun({ - name: "ReadyForExecutionError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: LAZY ATTEMPT - socket.on("READY_FOR_LAZY_ATTEMPT", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_LAZY_ATTEMPT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_LAZY_ATTEMPT"); - - try { - const lazyAttempt = await this.#platformSocket?.sendWithAck("READY_FOR_LAZY_ATTEMPT", { - ...message, - envId: socket.data.envId, - }); - - if (!lazyAttempt) { - log.error("no lazy attempt ack"); - - await crashRun({ - name: "ReadyForLazyAttemptError", - message: "No lazy attempt ack", - }); - - return; - } - - if (!lazyAttempt.success) { - log.error("failed to get lazy attempt payload", { reason: lazyAttempt.reason }); - - await crashRun({ - name: "ReadyForLazyAttemptError", - message: "Failed to get lazy attempt payload", - }); - - return; - } - - await chaosMonkey.call(); - - const lazyPayload = { - ...lazyAttempt.lazyPayload, - metrics: [ - ...(message.startTime - ? [ - { - name: "start", - event: "lazy_payload", - timestamp: message.startTime, - duration: Date.now() - message.startTime, - }, - ] - : []), - ], - }; - - socket.emit("EXECUTE_TASK_RUN_LAZY_ATTEMPT", { - version: "v1", - lazyPayload, - }); - } catch (error) { - if (error instanceof ChaosMonkey.Error) { - log.error("ChaosMonkey error, won't crash run"); - return; - } - - log.error("READY_FOR_LAZY_ATTEMPT error", { error }); - - // await crashRun({ - // name: "ReadyForLazyAttemptError", - // message: - // error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - // }); - - return; - } - }); - - // MARK: RESUME READY - socket.on("READY_FOR_RESUME", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_RESUME", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_RESUME"); - - try { - updateAttemptFriendlyId(message.attemptFriendlyId); - - if (message.version === "v2") { - updateAttemptNumber(message.attemptNumber); - } - - this.#platformSocket?.send("READY_FOR_RESUME", { ...message, version: "v1" }); - } catch (error) { - log.error("READY_FOR_RESUME error", { error }); - - await crashRun({ - name: "ReadyForResumeError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: RUN COMPLETED - socket.on("TASK_RUN_COMPLETED", async (message, callback) => { - const log = logger.child({ - eventName: "TASK_RUN_COMPLETED", - ...getSocketMetadata(), - ...omit(message, "completion", "execution"), - completion: { - id: message.completion.id, - ok: message.completion.ok, - }, - }); - - log.log("Handling TASK_RUN_COMPLETED"); - - try { - const { completion, execution } = message; - - // Cancel all in-progress checkpoints (if any) - this.#cancelCheckpoint(socket.data.runId, { - event: "TASK_RUN_COMPLETED", - attemptNumber: execution.attempt.number, - }); - - await chaosMonkey.call({ throwErrors: false }); - - const sendCompletionWithAck = async (): Promise => { - try { - const response = await this.#platformSocket?.sendWithAck( - "TASK_RUN_COMPLETED_WITH_ACK", - { - version: "v2", - execution, - completion, - }, - TASK_RUN_COMPLETED_WITH_ACK_TIMEOUT_MS - ); - - if (!response) { - log.error("TASK_RUN_COMPLETED_WITH_ACK: no response"); - return false; - } - - if (!response.success) { - log.error("TASK_RUN_COMPLETED_WITH_ACK: error response", { - error: response.error, - }); - return false; - } - - log.log("TASK_RUN_COMPLETED_WITH_ACK: successful response"); - return true; - } catch (error) { - log.error("TASK_RUN_COMPLETED_WITH_ACK: threw error", { error }); - return false; - } - }; - - const completeWithoutCheckpoint = async (shouldExit: boolean) => { - const supportsRetryCheckpoints = message.version === "v1"; - - callback({ willCheckpointAndRestore: false, shouldExit }); - - if (supportsRetryCheckpoints) { - // This is only here for backwards compat - this.#platformSocket?.send("TASK_RUN_COMPLETED", { - version: "v1", - execution, - completion, - }); - } else { - // 99.99% of runs should end up here - - const completedWithAckBackoff = new ExponentialBackoff("FullJitter").maxRetries( - TASK_RUN_COMPLETED_WITH_ACK_MAX_RETRIES - ); - - const result = await completedWithAckBackoff.execute( - async ({ retry, delay, elapsedMs }) => { - logger.log("TASK_RUN_COMPLETED_WITH_ACK: sending with backoff", { - retry, - delay, - elapsedMs, - }); - - const success = await sendCompletionWithAck(); - - if (!success) { - throw new Error("Failed to send completion with ack"); - } - } - ); - - if (!result.success) { - logger.error("TASK_RUN_COMPLETED_WITH_ACK: failed to send with backoff", result); - return; - } - - logger.log("TASK_RUN_COMPLETED_WITH_ACK: sent with backoff", result); - } - }; - - if (completion.ok) { - await completeWithoutCheckpoint(true); - return; - } - - if ( - completion.error.type === "INTERNAL_ERROR" && - completion.error.code === "TASK_RUN_CANCELLED" - ) { - await completeWithoutCheckpoint(true); - return; - } - - if (completion.retry === undefined) { - await completeWithoutCheckpoint(true); - return; - } - - if (completion.retry.delay < this.#delayThresholdInMs) { - await completeWithoutCheckpoint(false); - - // Prevents runs that fail fast from never sending a heartbeat - this.#sendRunHeartbeat(socket.data.runId); - - return; - } - - if (message.version === "v2") { - await completeWithoutCheckpoint(true); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - if (!willCheckpointAndRestore) { - await completeWithoutCheckpoint(false); - return; - } - - // The worker will then put itself in a checkpointable state - callback({ willCheckpointAndRestore: true, shouldExit: false }); - - const ready = await readyToCheckpoint("RETRY"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - - return; - } - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId: socket.data.runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - shouldHeartbeat: true, - }); - - if (!checkpoint) { - log.error("Failed to checkpoint"); - await completeWithoutCheckpoint(false); - return; - } - - log.addFields({ checkpoint }); - - this.#platformSocket?.send("TASK_RUN_COMPLETED", { - version: "v1", - execution, - completion, - checkpoint, - }); - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("TASK_RUN_COMPLETED error", { error }); - - await crashRun({ - name: "TaskRunCompletedError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: TASK FAILED - socket.on("TASK_RUN_FAILED_TO_RUN", async ({ completion }) => { - const log = logger.child({ - eventName: "TASK_RUN_FAILED_TO_RUN", - ...getSocketMetadata(), - completion: { - id: completion.id, - ok: completion.ok, - }, - }); - - log.log("Handling TASK_RUN_FAILED_TO_RUN"); - - try { - // Cancel all in-progress checkpoints (if any) - this.#cancelCheckpoint(socket.data.runId, { - event: "TASK_RUN_FAILED_TO_RUN", - errorType: completion.error.type, - }); - - this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", { - version: "v1", - completion, - }); - - exitRun(); - } catch (error) { - log.error("TASK_RUN_FAILED_TO_RUN error", { error }); - - return; - } - }); - - // MARK: CHECKPOINT - socket.on("READY_FOR_CHECKPOINT", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_CHECKPOINT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_CHECKPOINT"); - - try { - const checkpointable = this.#checkpointableTasks.get(socket.data.runId); - - if (!checkpointable) { - log.error("No checkpoint scheduled"); - return; - } - - checkpointable.resolve(); - } catch (error) { - log.error("READY_FOR_CHECKPOINT error", { error }); - - return; - } - }); - - // MARK: CXX CHECKPOINT - socket.on("CANCEL_CHECKPOINT", async (message, callback) => { - const log = logger.child({ - eventName: "CANCEL_CHECKPOINT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling CANCEL_CHECKPOINT"); - - try { - if (message.version === "v1") { - this.#cancelCheckpoint(socket.data.runId, { event: "CANCEL_CHECKPOINT", ...message }); - // v1 has no callback - return; - } - - const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId, { - event: "CANCEL_CHECKPOINT", - ...message, - }); - - callback({ version: "v2", checkpointCanceled }); - } catch (error) { - log.error("CANCEL_CHECKPOINT error", { error }); - } - }); - - // MARK: DURATION WAIT - socket.on("WAIT_FOR_DURATION", async (message, callback) => { - const log = logger.child({ - eventName: "WAIT_FOR_DURATION", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling WAIT_FOR_DURATION"); - - try { - await chaosMonkey.call({ throwErrors: false }); - - if (checkpointInProgress()) { - log.error("Checkpoint already in progress"); - callback({ willCheckpointAndRestore: false }); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - callback({ willCheckpointAndRestore }); - - if (!willCheckpointAndRestore) { - return; - } - - const ready = await readyToCheckpoint("WAIT_FOR_DURATION"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - return; - } - - const runId = socket.data.runId; - const attemptNumber = getAttemptNumber(); - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - attemptNumber, - }); - - if (!checkpoint) { - // The task container will keep running until the wait duration has elapsed - log.error("Failed to checkpoint"); - return; - } - - log.addFields({ checkpoint }); - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId: socket.data.runId, - attemptFriendlyId: message.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_DURATION", - ms: message.ms, - now: message.now, - }, - }); - - if (ack?.keepRunAlive) { - log.log("keeping run alive after duration checkpoint"); - - if (checkpoint.docker && willSimulate) { - // The container is still paused so we need to unpause it - log.log("unpausing container after duration checkpoint"); - this.#checkpointer.unpause(runId, attemptNumber); - } - - return; - } - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("WAIT_FOR_DURATION error", { error }); - - await crashRun({ - name: "WaitForDurationError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: TASK WAIT - socket.on("WAIT_FOR_TASK", async (message, callback) => { - const log = logger.child({ - eventName: "WAIT_FOR_TASK", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling WAIT_FOR_TASK"); - - try { - await chaosMonkey.call({ throwErrors: false }); - - if (checkpointInProgress()) { - log.error("Checkpoint already in progress"); - callback({ willCheckpointAndRestore: false }); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - callback({ willCheckpointAndRestore }); - - if (!willCheckpointAndRestore) { - return; - } - - // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits - if (message.version === "v2") { - const ready = await readyToCheckpoint("WAIT_FOR_TASK"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - return; - } - } - - const runId = socket.data.runId; - const attemptNumber = getAttemptNumber(); - - const checkpoint = await this.#checkpointer.checkpointAndPush( - { - runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - attemptNumber, - }, - WAIT_FOR_TASK_CHECKPOINT_DELAY_MS - ); - - if (!checkpoint) { - log.error("Failed to checkpoint"); - return; - } - - log.addFields({ checkpoint }); - - log.log("WAIT_FOR_TASK checkpoint created"); - - //setting this means we can only resume from a checkpoint - socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`; - log.log("WAIT_FOR_TASK set requiresCheckpointResumeWithMessage"); - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId: socket.data.runId, - attemptFriendlyId: message.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_TASK", - friendlyId: message.friendlyId, - }, - }); - - if (ack?.keepRunAlive) { - socket.data.requiresCheckpointResumeWithMessage = undefined; - log.log("keeping run alive after task checkpoint"); - - if (checkpoint.docker && willSimulate) { - // The container is still paused so we need to unpause it - log.log("unpausing container after duration checkpoint"); - this.#checkpointer.unpause(runId, attemptNumber); - } - - return; - } - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("WAIT_FOR_TASK error", { error }); - - await crashRun({ - name: "WaitForTaskError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: BATCH WAIT - socket.on("WAIT_FOR_BATCH", async (message, callback) => { - const log = logger.child({ - eventName: "WAIT_FOR_BATCH", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling WAIT_FOR_BATCH", message); - - try { - await chaosMonkey.call({ throwErrors: false }); - - if (checkpointInProgress()) { - log.error("Checkpoint already in progress"); - callback({ willCheckpointAndRestore: false }); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - callback({ willCheckpointAndRestore }); - - if (!willCheckpointAndRestore) { - return; - } - - // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits - if (message.version === "v2") { - const ready = await readyToCheckpoint("WAIT_FOR_BATCH"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - return; - } - } - - const runId = socket.data.runId; - const attemptNumber = getAttemptNumber(); - - const checkpoint = await this.#checkpointer.checkpointAndPush( - { - runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - attemptNumber, - }, - WAIT_FOR_BATCH_CHECKPOINT_DELAY_MS - ); - - if (!checkpoint) { - log.error("Failed to checkpoint"); - return; - } - - log.addFields({ checkpoint }); - - log.log("WAIT_FOR_BATCH checkpoint created"); - - //setting this means we can only resume from a checkpoint - socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`; - log.log("WAIT_FOR_BATCH set checkpoint"); - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId: socket.data.runId, - attemptFriendlyId: message.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_BATCH", - batchFriendlyId: message.batchFriendlyId, - runFriendlyIds: message.runFriendlyIds, - }, - }); - - if (ack?.keepRunAlive) { - socket.data.requiresCheckpointResumeWithMessage = undefined; - log.log("keeping run alive after batch checkpoint"); - - if (checkpoint.docker && willSimulate) { - // The container is still paused so we need to unpause it - log.log("unpausing container after batch checkpoint"); - this.#checkpointer.unpause(runId, attemptNumber); - } - - return; - } - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("WAIT_FOR_BATCH error", { error }); - - await crashRun({ - name: "WaitForBatchError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: INDEX - socket.on("INDEX_TASKS", async (message, callback) => { - const log = logger.child({ - eventName: "INDEX_TASKS", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling INDEX_TASKS"); - - try { - const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", { - version: "v2", - projectRef: socket.data.projectRef, - envId: socket.data.envId, - deploymentId: message.deploymentId, - metadata: { - contentHash: socket.data.contentHash, - packageVersion: message.packageVersion, - tasks: message.tasks, - }, - supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts, - }); - - if (!workerAck) { - log.debug("no worker ack while indexing"); - } - - callback({ success: !!workerAck?.success }); - } catch (error) { - log.error("INDEX_TASKS error", { error }); - callback({ success: false }); - } - }); - - // MARK: INDEX FAILED - socket.on("INDEXING_FAILED", async (message) => { - const log = logger.child({ - eventName: "INDEXING_FAILED", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling INDEXING_FAILED"); - - try { - this.#platformSocket?.send("INDEXING_FAILED", { - version: "v1", - deploymentId: message.deploymentId, - error: message.error, - }); - } catch (error) { - log.error("INDEXING_FAILED error", { error }); - } - }); - - // MARK: CREATE ATTEMPT - socket.on("CREATE_TASK_RUN_ATTEMPT", async (message, callback) => { - const log = logger.child({ - eventName: "CREATE_TASK_RUN_ATTEMPT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling CREATE_TASK_RUN_ATTEMPT"); - - try { - await chaosMonkey.call({ throwErrors: false }); - - const createAttempt = await this.#platformSocket?.sendWithAck( - "CREATE_TASK_RUN_ATTEMPT", - { - runId: message.runId, - envId: socket.data.envId, - } - ); - - if (!createAttempt?.success) { - log.debug("no ack while creating attempt", { reason: createAttempt?.reason }); - callback({ success: false, reason: createAttempt?.reason }); - return; - } - - updateAttemptFriendlyId(createAttempt.executionPayload.execution.attempt.id); - updateAttemptNumber(createAttempt.executionPayload.execution.attempt.number); - - callback({ - success: true, - executionPayload: createAttempt.executionPayload, - }); - } catch (error) { - log.error("CREATE_TASK_RUN_ATTEMPT error", { error }); - callback({ - success: false, - reason: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - } - }); - - socket.on("UNRECOVERABLE_ERROR", async (message) => { - const log = logger.child({ - eventName: "UNRECOVERABLE_ERROR", - ...getSocketMetadata(), - error: message.error, - }); - - log.log("Handling UNRECOVERABLE_ERROR"); - - try { - await crashRun(message.error); - } catch (error) { - log.error("UNRECOVERABLE_ERROR error", { error }); - } - }); - - socket.on("SET_STATE", async (message) => { - const log = logger.child({ - eventName: "SET_STATE", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling SET_STATE"); - - try { - if (message.attemptFriendlyId) { - updateAttemptFriendlyId(message.attemptFriendlyId); - } - - if (message.attemptNumber) { - updateAttemptNumber(message.attemptNumber); - } - } catch (error) { - log.error("SET_STATE error", { error }); - } - }); - }, - onDisconnect: async (socket, handler, sender, logger) => { - try { - this.#platformSocket?.send("LOG", { - metadata: socket.data, - text: "disconnect", - }); - } catch (error) { - logger.error("onDisconnect error", { error }); - } - }, - handlers: { - TASK_HEARTBEAT: async (message) => { - this.#platformSocket?.send("TASK_HEARTBEAT", message); - }, - TASK_RUN_HEARTBEAT: async (message) => { - this.#sendRunHeartbeat(message.runId); - }, - }, - }); - - return provider; - } - - #sendRunHeartbeat(runId: string) { - this.#platformSocket?.send("TASK_RUN_HEARTBEAT", { - version: "v1", - runId, - }); - } - - #cancelCheckpoint(runId: string, reason?: any): boolean { - logger.log("cancelCheckpoint: call", { runId, reason }); - - const checkpointWait = this.#checkpointableTasks.get(runId); - - if (checkpointWait) { - // Stop waiting for task to reach checkpointable state - checkpointWait.reject(new CheckpointCancelError()); - } - - // Cancel checkpointing procedure - const checkpointCanceled = this.#checkpointer.cancelAllCheckpointsForRun(runId); - - logger.log("cancelCheckpoint: result", { - runId, - reason, - checkpointCanceled, - hadCheckpointWait: !!checkpointWait, - }); - - return checkpointCanceled; - } - - // MARK: HTTP SERVER - #createHttpServer() { - const httpServer = createServer(async (req, res) => { - logger.log(`[${req.method}]`, { url: req.url }); - - const reply = new HttpReply(res); - - switch (req.url) { - case "/health": { - return reply.text("ok"); - } - case "/metrics": { - return reply.text(await register.metrics(), 200, register.contentType); - } - default: { - return reply.empty(404); - } - } - }); - - httpServer.on("clientError", (err, socket) => { - socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); - }); - - httpServer.on("listening", () => { - logger.log("server listening on port", { port: HTTP_SERVER_PORT }); - }); - - return httpServer; - } - - #createInternalHttpServer() { - const httpServer = createServer(async (req, res) => { - logger.log(`[${req.method}]`, { url: req.url }); - - const reply = new HttpReply(res); - - switch (req.url) { - case "/whoami": { - return reply.text(NODE_NAME); - } - case "/checkpoint/duration": { - try { - const body = await getTextBody(req); - const json = safeJsonParse(body); - - if (typeof json !== "object" || !json) { - return reply.text("Invalid body", 400); - } - - if (!("runId" in json) || typeof json.runId !== "string") { - return reply.text("Missing or invalid: runId", 400); - } - - if (!("now" in json) || typeof json.now !== "number") { - return reply.text("Missing or invalid: now", 400); - } - - if (!("ms" in json) || typeof json.ms !== "number") { - return reply.text("Missing or invalid: ms", 400); - } - - let keepRunAlive = false; - if ("keepRunAlive" in json && typeof json.keepRunAlive === "boolean") { - keepRunAlive = json.keepRunAlive; - } - - let async = false; - if ("async" in json && typeof json.async === "boolean") { - async = json.async; - } - - const { runId, now, ms } = json; - - if (!runId) { - return reply.text("Missing runId", 400); - } - - const runSocket = await this.#getRunSocket(runId); - if (!runSocket) { - return reply.text("Run socket not found", 404); - } - - const { data } = runSocket; - - console.log("Manual duration checkpoint", data); - - if (async) { - reply.text("Creating checkpoint in the background", 202); - } - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId: data.runId, - projectRef: data.projectRef, - deploymentVersion: data.deploymentVersion, - attemptNumber: data.attemptNumber ? parseInt(data.attemptNumber) : undefined, - }); - - if (!checkpoint) { - return reply.text("Failed to checkpoint", 500); - } - - if (!data.attemptFriendlyId) { - return reply.text("Socket data missing attemptFriendlyId", 500); - } - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId, - attemptFriendlyId: data.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_DURATION", - ms, - now, - }, - }); - - if (ack?.keepRunAlive || keepRunAlive) { - return reply.json({ - message: `keeping run ${runId} alive after checkpoint`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } - - runSocket.emit("REQUEST_EXIT", { - version: "v1", - }); - - return reply.json({ - message: `checkpoint created for run ${runId}`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } catch (error) { - return reply.json({ - message: `error`, - error, - }); - } - } - case "/checkpoint/manual": { - try { - const body = await getTextBody(req); - const json = safeJsonParse(body); - - if (typeof json !== "object" || !json) { - return reply.text("Invalid body", 400); - } - - if (!("runId" in json) || typeof json.runId !== "string") { - return reply.text("Missing or invalid: runId", 400); - } - - let restoreAtUnixTimeMs: number | undefined; - if ("restoreAtUnixTimeMs" in json && typeof json.restoreAtUnixTimeMs === "number") { - restoreAtUnixTimeMs = json.restoreAtUnixTimeMs; - } - - let keepRunAlive = false; - if ("keepRunAlive" in json && typeof json.keepRunAlive === "boolean") { - keepRunAlive = json.keepRunAlive; - } - - let async = false; - if ("async" in json && typeof json.async === "boolean") { - async = json.async; - } - - const { runId } = json; - - if (!runId) { - return reply.text("Missing runId", 400); - } - - const runSocket = await this.#getRunSocket(runId); - if (!runSocket) { - return reply.text("Run socket not found", 404); - } - - const { data } = runSocket; - - console.log("Manual checkpoint", data); - - if (async) { - reply.text("Creating checkpoint in the background", 202); - } - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId: data.runId, - projectRef: data.projectRef, - deploymentVersion: data.deploymentVersion, - attemptNumber: data.attemptNumber ? parseInt(data.attemptNumber) : undefined, - }); - - if (!checkpoint) { - return reply.text("Failed to checkpoint", 500); - } - - if (!data.attemptFriendlyId) { - return reply.text("Socket data missing attemptFriendlyId", 500); - } - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId, - attemptFriendlyId: data.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "MANUAL", - restoreAtUnixTimeMs, - }, - }); - - if (ack?.keepRunAlive || keepRunAlive) { - return reply.json({ - message: `keeping run ${runId} alive after checkpoint`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } - - runSocket.emit("REQUEST_EXIT", { - version: "v1", - }); - - return reply.json({ - message: `checkpoint created for run ${runId}`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } catch (error) { - return reply.json({ - message: `error`, - error, - }); - } - } - default: { - return reply.empty(404); - } - } - }); - - httpServer.on("clientError", (err, socket) => { - socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); - }); - - httpServer.on("listening", () => { - logger.log("internal server listening on port", { port: HTTP_SERVER_PORT + 100 }); - }); - - return httpServer; - } - - listen() { - this.#httpServer.listen(this.port, this.host); - this.#internalHttpServer.listen(this.port + 100, "127.0.0.1"); - } -} - -const coordinator = new TaskCoordinator(HTTP_SERVER_PORT); -coordinator.listen(); diff --git a/apps/coordinator/src/util.ts b/apps/coordinator/src/util.ts deleted file mode 100644 index 18464f230b..0000000000 --- a/apps/coordinator/src/util.ts +++ /dev/null @@ -1,31 +0,0 @@ -export const boolFromEnv = (env: string, defaultValue: boolean): boolean => { - const value = process.env[env]; - - if (!value) { - return defaultValue; - } - - return ["1", "true"].includes(value); -}; - -export const numFromEnv = (env: string, defaultValue: number): number => { - const value = process.env[env]; - - if (!value) { - return defaultValue; - } - - return parseInt(value, 10); -}; - -export function safeJsonParse(json?: string): unknown { - if (!json) { - return; - } - - try { - return JSON.parse(json); - } catch (e) { - return null; - } -} diff --git a/apps/coordinator/tsconfig.json b/apps/coordinator/tsconfig.json deleted file mode 100644 index e03fd02412..0000000000 --- a/apps/coordinator/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "esModuleInterop": true, - "resolveJsonModule": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true, - "paths": { - "@trigger.dev/core/v3": ["../../packages/core/src/v3"], - "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] - } - } -} diff --git a/apps/docker-provider/.env.example b/apps/docker-provider/.env.example deleted file mode 100644 index 75c54083d1..0000000000 --- a/apps/docker-provider/.env.example +++ /dev/null @@ -1,11 +0,0 @@ -HTTP_SERVER_PORT=8050 - -PLATFORM_WS_PORT=3030 -PLATFORM_SECRET=provider-secret -SECURE_CONNECTION=false - -OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel - -# Use this if you are on macOS -# COORDINATOR_HOST="host.docker.internal" -# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/docker-provider/.gitignore b/apps/docker-provider/.gitignore deleted file mode 100644 index 5c84119d63..0000000000 --- a/apps/docker-provider/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -dist/ -node_modules/ -.env \ No newline at end of file diff --git a/apps/docker-provider/Containerfile b/apps/docker-provider/Containerfile deleted file mode 100644 index bea730bda8..0000000000 --- a/apps/docker-provider/Containerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine - -WORKDIR /app - -FROM node-20-alpine AS pruner - -COPY --chown=node:node . . -RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker -RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - -FROM node-20-alpine AS base - -RUN apk add --no-cache dumb-init docker - -COPY --chown=node:node .gitignore .gitignore -COPY --from=pruner --chown=node:node /app/out/json/ . -COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml - -FROM base AS dev-deps -RUN corepack enable -ENV NODE_ENV development - -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile - -FROM base AS builder -RUN corepack enable - -COPY --from=pruner --chown=node:node /app/out/full/ . -COPY --from=dev-deps --chown=node:node /app/ . -COPY --chown=node:node turbo.json turbo.json - -RUN pnpm run -r --filter docker-provider build:bundle - -FROM base AS runner - -RUN corepack enable -ENV NODE_ENV production - -COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs - -EXPOSE 8000 - -USER node - -CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/docker-provider/README.md b/apps/docker-provider/README.md deleted file mode 100644 index 647db280a5..0000000000 --- a/apps/docker-provider/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Docker provider - -The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions. diff --git a/apps/docker-provider/package.json b/apps/docker-provider/package.json deleted file mode 100644 index f3e4015ef0..0000000000 --- a/apps/docker-provider/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "docker-provider", - "private": true, - "version": "0.0.1", - "description": "", - "main": "dist/index.cjs", - "scripts": { - "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", - "build:image": "docker build -f Containerfile . -t docker-provider", - "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", - "start": "tsx src/index.ts", - "typecheck": "tsc --noEmit" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@trigger.dev/core": "workspace:*", - "execa": "^8.0.1" - }, - "devDependencies": { - "dotenv": "^16.4.2", - "esbuild": "^0.19.11", - "tsx": "^4.7.0" - } -} \ No newline at end of file diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts deleted file mode 100644 index a0b0554fb2..0000000000 --- a/apps/docker-provider/src/index.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { $, type ExecaChildProcess, execa } from "execa"; -import { - ProviderShell, - TaskOperations, - TaskOperationsCreateOptions, - TaskOperationsIndexOptions, - TaskOperationsRestoreOptions, -} from "@trigger.dev/core/v3/apps"; -import { SimpleLogger } from "@trigger.dev/core/v3/apps"; -import { isExecaChildProcess } from "@trigger.dev/core/v3/apps"; -import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly"; -import { setTimeout } from "node:timers/promises"; -import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3"; - -const MACHINE_NAME = process.env.MACHINE_NAME || "local"; -const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020; -const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1"; -const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host"; - -const OTEL_EXPORTER_OTLP_ENDPOINT = - process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318"; - -const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes( - process.env.FORCE_CHECKPOINT_SIMULATION ?? "false" -); - -const logger = new SimpleLogger(`[${MACHINE_NAME}]`); - -type TaskOperationsInitReturn = { - canCheckpoint: boolean; - willSimulate: boolean; -}; - -class DockerTaskOperations implements TaskOperations { - #initialized = false; - #canCheckpoint = false; - - constructor(private opts = { forceSimulate: false }) {} - - async init(): Promise { - if (this.#initialized) { - return this.#getInitReturn(this.#canCheckpoint); - } - - logger.log("Initializing task operations"); - - const testCheckpoint = await testDockerCheckpoint(); - - if (testCheckpoint.ok) { - return this.#getInitReturn(true); - } - - logger.error(testCheckpoint.message, testCheckpoint.error); - return this.#getInitReturn(false); - } - - #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn { - this.#canCheckpoint = canCheckpoint; - - if (canCheckpoint) { - if (!this.#initialized) { - logger.log("Full checkpoint support!"); - } - } - - this.#initialized = true; - - const willSimulate = !canCheckpoint || this.opts.forceSimulate; - - if (willSimulate) { - logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", { - forceSimulate: this.opts.forceSimulate, - }); - } - - return { - canCheckpoint, - willSimulate, - }; - } - - async index(opts: TaskOperationsIndexOptions) { - await this.init(); - - const containerName = this.#getIndexContainerName(opts.shortCode); - - logger.log(`Indexing task ${opts.imageRef}`, { - host: COORDINATOR_HOST, - port: COORDINATOR_PORT, - }); - - logger.debug( - await execa("docker", [ - "run", - `--network=${DOCKER_NETWORK}`, - "--rm", - `--env=INDEX_TASKS=true`, - `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`, - `--env=TRIGGER_API_URL=${opts.apiUrl}`, - `--env=TRIGGER_ENV_ID=${opts.envId}`, - `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, - `--env=POD_NAME=${containerName}`, - `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, - `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--name=${containerName}`, - `${opts.imageRef}`, - ]) - ); - } - - async create(opts: TaskOperationsCreateOptions) { - await this.init(); - - const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber); - - const runArgs = [ - "run", - `--network=${DOCKER_NETWORK}`, - "--detach", - `--env=TRIGGER_ENV_ID=${opts.envId}`, - `--env=TRIGGER_RUN_ID=${opts.runId}`, - `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, - `--env=POD_NAME=${containerName}`, - `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, - `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, - `--name=${containerName}`, - ]; - - if (process.env.ENFORCE_MACHINE_PRESETS) { - runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`); - } - - if (opts.dequeuedAt) { - runArgs.push(`--env=TRIGGER_RUN_DEQUEUED_AT_MS=${opts.dequeuedAt}`); - } - - runArgs.push(`${opts.image}`); - - try { - logger.debug(await execa("docker", runArgs)); - } catch (error) { - if (!isExecaChildProcess(error)) { - throw error; - } - - logger.error("Create failed:", { - opts, - exitCode: error.exitCode, - escapedCommand: error.escapedCommand, - stdout: error.stdout, - stderr: error.stderr, - }); - } - } - - async restore(opts: TaskOperationsRestoreOptions) { - await this.init(); - - const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber); - - if (!this.#canCheckpoint || this.opts.forceSimulate) { - logger.log("Simulating restore"); - - const unpause = logger.debug(await $`docker unpause ${containerName}`); - - if (unpause.exitCode !== 0) { - throw new Error("docker unpause command failed"); - } - - await this.#sendPostStart(containerName); - return; - } - - const { exitCode } = logger.debug( - await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}` - ); - - if (exitCode !== 0) { - throw new Error("docker start command failed"); - } - - await this.#sendPostStart(containerName); - } - - async delete(opts: { runId: string }) { - await this.init(); - - const containerName = this.#getRunContainerName(opts.runId); - await this.#sendPreStop(containerName); - - logger.log("noop: delete"); - } - - async get(opts: { runId: string }) { - await this.init(); - - logger.log("noop: get"); - } - - #getIndexContainerName(suffix: string) { - return `task-index-${suffix}`; - } - - #getRunContainerName(suffix: string, attemptNumber?: number) { - return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`; - } - - async #sendPostStart(containerName: string): Promise { - try { - const port = await this.#getHttpServerPort(containerName); - logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore")); - } catch (error) { - logger.error("postStart error", { error }); - throw new Error("postStart command failed"); - } - } - - async #sendPreStop(containerName: string): Promise { - try { - const port = await this.#getHttpServerPort(containerName); - logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate")); - } catch (error) { - logger.error("preStop error", { error }); - throw new Error("preStop command failed"); - } - } - - async #getHttpServerPort(containerName: string): Promise { - // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes - // FIXME: Skip this in prod - const logs = logger.debug(await $`docker logs ${containerName}`); - const matches = logs.stdout.match(/http server listening on port (?[0-9]+)/); - - const port = Number(matches?.groups?.port); - - if (!port) { - throw new Error("failed to extract port from logs"); - } - - return port; - } - - async #runLifecycleCommand( - containerName: string, - port: number, - type: THookType, - cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses, - retryCount = 0 - ): Promise { - try { - return await execa("docker", [ - "exec", - containerName, - "busybox", - "wget", - "-q", - "-O-", - `127.0.0.1:${port}/${type}?cause=${cause}`, - ]); - } catch (error: any) { - if (type === "postStart" && retryCount < 6) { - logger.debug(`retriable ${type} error`, { retryCount, message: error?.message }); - await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50)); - - return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1); - } - - logger.error(`final ${type} error`, { message: error?.message }); - throw new Error(`${type} command failed after ${retryCount - 1} retries`); - } - } -} - -const provider = new ProviderShell({ - tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }), - type: "docker", -}); - -provider.listen(); - -function exponentialBackoff( - retryCount: number, - exponential: number, - minDelay: number, - maxDelay: number, - jitter: number -): number { - // Calculate the delay using the exponential backoff formula - const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay); - - // Calculate the jitter - const jitterValue = Math.random() * jitter; - - // Return the calculated delay with jitter - return delay + jitterValue; -} diff --git a/apps/docker-provider/tsconfig.json b/apps/docker-provider/tsconfig.json deleted file mode 100644 index f87adfc2d7..0000000000 --- a/apps/docker-provider/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "strict": true, - "skipLibCheck": true, - "paths": { - "@trigger.dev/core/v3": ["../../packages/core/src/v3"], - "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] - } - } -} diff --git a/apps/kubernetes-provider/.env.example b/apps/kubernetes-provider/.env.example deleted file mode 100644 index f21ee29bac..0000000000 --- a/apps/kubernetes-provider/.env.example +++ /dev/null @@ -1,9 +0,0 @@ -HTTP_SERVER_PORT=8060 - -PLATFORM_WS_PORT=3030 -PLATFORM_SECRET=provider-secret -SECURE_CONNECTION=false - -# Use this if you are on macOS -# COORDINATOR_HOST="host.docker.internal" -# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/kubernetes-provider/.gitignore b/apps/kubernetes-provider/.gitignore deleted file mode 100644 index 5c84119d63..0000000000 --- a/apps/kubernetes-provider/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -dist/ -node_modules/ -.env \ No newline at end of file diff --git a/apps/kubernetes-provider/Containerfile b/apps/kubernetes-provider/Containerfile deleted file mode 100644 index fb96304c26..0000000000 --- a/apps/kubernetes-provider/Containerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine - -WORKDIR /app - -FROM node-20-alpine AS pruner - -COPY --chown=node:node . . -RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker -RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - -FROM node-20-alpine AS base - -RUN apk add --no-cache dumb-init - -COPY --chown=node:node .gitignore .gitignore -COPY --from=pruner --chown=node:node /app/out/json/ . -COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml - -FROM base AS dev-deps -RUN corepack enable -ENV NODE_ENV development - -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile - -FROM base AS builder -RUN corepack enable - -COPY --from=pruner --chown=node:node /app/out/full/ . -COPY --from=dev-deps --chown=node:node /app/ . -COPY --chown=node:node turbo.json turbo.json - -RUN pnpm run -r --filter kubernetes-provider build:bundle - -FROM base AS runner - -RUN corepack enable -ENV NODE_ENV production - -COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs - -EXPOSE 8000 - -USER node - -CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/kubernetes-provider/README.md b/apps/kubernetes-provider/README.md deleted file mode 100644 index 829c8f2154..0000000000 --- a/apps/kubernetes-provider/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Kubernetes provider - -The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions. diff --git a/apps/kubernetes-provider/package.json b/apps/kubernetes-provider/package.json deleted file mode 100644 index 6cb26e2c70..0000000000 --- a/apps/kubernetes-provider/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "kubernetes-provider", - "private": true, - "version": "0.0.1", - "description": "", - "main": "dist/index.cjs", - "scripts": { - "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", - "build:image": "docker build -f Containerfile . -t kubernetes-provider", - "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", - "start": "tsx src/index.ts", - "typecheck": "tsc --noEmit" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@kubernetes/client-node": "^0.20.0", - "@trigger.dev/core": "workspace:*", - "p-queue": "^8.0.1" - }, - "devDependencies": { - "dotenv": "^16.4.2", - "esbuild": "^0.19.11", - "tsx": "^4.7.0" - } -} \ No newline at end of file diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts deleted file mode 100644 index 23a6ad56ce..0000000000 --- a/apps/kubernetes-provider/src/index.ts +++ /dev/null @@ -1,783 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { - EnvironmentType, - MachinePreset, - PostStartCauses, - PreStopCauses, -} from "@trigger.dev/core/v3"; -import { - ProviderShell, - SimpleLogger, - TaskOperations, - TaskOperationsCreateOptions, - TaskOperationsIndexOptions, - TaskOperationsPrePullDeploymentOptions, - TaskOperationsRestoreOptions, -} from "@trigger.dev/core/v3/apps"; -import { PodCleaner } from "./podCleaner"; -import { TaskMonitor } from "./taskMonitor"; -import { UptimeHeartbeat } from "./uptimeHeartbeat"; -import { assertExhaustive } from "@trigger.dev/core"; -import { CustomLabelHelper } from "./labelHelper"; - -const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; -const NODE_NAME = process.env.NODE_NAME || "local"; -const OTEL_EXPORTER_OTLP_ENDPOINT = - process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"; -const COORDINATOR_HOST = process.env.COORDINATOR_HOST ?? undefined; -const COORDINATOR_PORT = process.env.COORDINATOR_PORT ?? undefined; -const KUBERNETES_NAMESPACE = process.env.KUBERNETES_NAMESPACE ?? "default"; - -const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300"); - -const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL; -const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60"); -const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25"); -const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10"); -const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10"); - -const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi"; -const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi"; - -// Image config -const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true"; -const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS; -const PAUSE_IMAGE = process.env.PAUSE_IMAGE || "registry.k8s.io/pause:3.9"; -const BUSYBOX_IMAGE = process.env.BUSYBOX_IMAGE || "registry.digitalocean.com/trigger/busybox"; -const DEPLOYMENT_IMAGE_PREFIX = process.env.DEPLOYMENT_IMAGE_PREFIX; -const RESTORE_IMAGE_PREFIX = process.env.RESTORE_IMAGE_PREFIX; -const UTILITY_IMAGE_PREFIX = process.env.UTILITY_IMAGE_PREFIX; - -const logger = new SimpleLogger(`[${NODE_NAME}]`); -logger.log(`running in ${RUNTIME_ENV} mode`); - -type Namespace = { - metadata: { - name: string; - }; -}; - -type ResourceQuantities = { - [K in "cpu" | "memory" | "ephemeral-storage"]?: string; -}; - -class KubernetesTaskOperations implements TaskOperations { - #namespace: Namespace = { - metadata: { - name: "default", - }, - }; - - #k8sApi: { - core: k8s.CoreV1Api; - batch: k8s.BatchV1Api; - apps: k8s.AppsV1Api; - }; - - #labelHelper = new CustomLabelHelper(); - - constructor(opts: { namespace?: string } = {}) { - if (opts.namespace) { - this.#namespace.metadata.name = opts.namespace; - } - - this.#k8sApi = this.#createK8sApi(); - } - - async init() { - // noop - } - - async index(opts: TaskOperationsIndexOptions) { - await this.#createJob( - { - metadata: { - name: this.#getIndexContainerName(opts.shortCode), - namespace: this.#namespace.metadata.name, - }, - spec: { - completions: 1, - backoffLimit: 0, - ttlSecondsAfterFinished: 300, - template: { - metadata: { - labels: { - ...this.#getSharedLabels(opts), - app: "task-index", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "index", - deployment: opts.deploymentId, - }, - }, - spec: { - ...this.#defaultPodSpec, - containers: [ - { - name: this.#getIndexContainerName(opts.shortCode), - image: getImageRef("deployment", opts.imageRef), - ports: [ - { - containerPort: 8000, - }, - ], - resources: { - limits: { - cpu: "1", - memory: "2G", - "ephemeral-storage": "2Gi", - }, - }, - lifecycle: { - preStop: { - exec: { - command: this.#getLifecycleCommand("preStop", "terminate"), - }, - }, - }, - env: [ - ...this.#getSharedEnv(opts.envId), - { - name: "INDEX_TASKS", - value: "true", - }, - { - name: "TRIGGER_SECRET_KEY", - value: opts.apiKey, - }, - { - name: "TRIGGER_API_URL", - value: opts.apiUrl, - }, - ], - }, - ], - }, - }, - }, - }, - this.#namespace - ); - } - - async create(opts: TaskOperationsCreateOptions) { - const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber); - - await this.#createPod( - { - metadata: { - name: containerName, - namespace: this.#namespace.metadata.name, - labels: { - ...this.#labelHelper.getAdditionalLabels("create"), - ...this.#getSharedLabels(opts), - app: "task-run", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "create", - run: opts.runId, - }, - }, - spec: { - ...this.#defaultPodSpec, - terminationGracePeriodSeconds: 60 * 60, - containers: [ - { - name: containerName, - image: getImageRef("deployment", opts.image), - ports: [ - { - containerPort: 8000, - }, - ], - resources: this.#getResourcesForMachine(opts.machine), - lifecycle: { - preStop: { - exec: { - command: this.#getLifecycleCommand("preStop", "terminate"), - }, - }, - }, - env: [ - ...this.#getSharedEnv(opts.envId), - { - name: "TRIGGER_RUN_ID", - value: opts.runId, - }, - ...(opts.dequeuedAt - ? [{ name: "TRIGGER_RUN_DEQUEUED_AT_MS", value: String(opts.dequeuedAt) }] - : []), - ], - volumeMounts: [ - { - name: "taskinfo", - mountPath: "/etc/taskinfo", - }, - ], - }, - ], - volumes: [ - { - name: "taskinfo", - emptyDir: {}, - }, - ], - }, - }, - this.#namespace - ); - } - - async restore(opts: TaskOperationsRestoreOptions) { - await this.#createPod( - { - metadata: { - name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`, - namespace: this.#namespace.metadata.name, - labels: { - ...this.#labelHelper.getAdditionalLabels("restore"), - ...this.#getSharedLabels(opts), - app: "task-run", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "restore", - run: opts.runId, - checkpoint: opts.checkpointId, - }, - }, - spec: { - ...this.#defaultPodSpec, - initContainers: [ - { - name: "pull-base-image", - image: getImageRef("deployment", opts.imageRef), - command: ["sleep", "0"], - }, - { - name: "populate-taskinfo", - image: getImageRef("utility", BUSYBOX_IMAGE), - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-c"], - args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"], - env: this.#coordinatorEnvVars, - volumeMounts: [ - { - name: "taskinfo", - mountPath: "/etc/taskinfo", - }, - ], - }, - ], - containers: [ - { - name: this.#getRunContainerName(opts.runId), - image: getImageRef("restore", opts.checkpointRef), - ports: [ - { - containerPort: 8000, - }, - ], - resources: this.#getResourcesForMachine(opts.machine), - lifecycle: { - postStart: { - exec: { - command: this.#getLifecycleCommand("postStart", "restore"), - }, - }, - preStop: { - exec: { - command: this.#getLifecycleCommand("preStop", "terminate"), - }, - }, - }, - volumeMounts: [ - { - name: "taskinfo", - mountPath: "/etc/taskinfo", - }, - ], - }, - ], - volumes: [ - { - name: "taskinfo", - emptyDir: {}, - }, - ], - }, - }, - this.#namespace - ); - } - - async delete(opts: { runId: string }) { - await this.#deletePod({ - runId: opts.runId, - namespace: this.#namespace, - }); - } - - async get(opts: { runId: string }) { - await this.#getPod(opts.runId, this.#namespace); - } - - async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) { - if (PRE_PULL_DISABLED) { - logger.debug("Pre-pull is disabled, skipping.", { opts }); - return; - } - - const metaName = this.#getPrePullContainerName(opts.shortCode); - - const metaLabels = { - ...this.#getSharedLabels(opts), - app: "task-prepull", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "prepull", - deployment: opts.deploymentId, - name: metaName, - } satisfies k8s.V1ObjectMeta["labels"]; - - await this.#createDaemonSet( - { - metadata: { - name: metaName, - namespace: this.#namespace.metadata.name, - labels: metaLabels, - }, - spec: { - selector: { - matchLabels: { - name: metaName, - }, - }, - template: { - metadata: { - labels: metaLabels, - }, - spec: { - ...this.#defaultPodSpec, - restartPolicy: "Always", - affinity: { - nodeAffinity: { - requiredDuringSchedulingIgnoredDuringExecution: { - nodeSelectorTerms: [ - { - matchExpressions: [ - { - key: "trigger.dev/pre-pull-disabled", - operator: "DoesNotExist", - }, - ], - }, - ], - }, - }, - }, - initContainers: [ - { - name: "prepull", - image: getImageRef("deployment", opts.imageRef), - command: ["/usr/bin/true"], - resources: { - limits: { - cpu: "0.25", - memory: "100Mi", - "ephemeral-storage": "1Gi", - }, - }, - }, - ], - containers: [ - { - name: "pause", - image: getImageRef("utility", PAUSE_IMAGE), - resources: { - limits: { - cpu: "1m", - memory: "12Mi", - }, - }, - }, - ], - }, - }, - }, - }, - this.#namespace - ); - } - - #envTypeToLabelValue(type: EnvironmentType) { - switch (type) { - case "PRODUCTION": - return "prod"; - case "STAGING": - return "stg"; - case "DEVELOPMENT": - return "dev"; - case "PREVIEW": - return "preview"; - } - } - - get #defaultPodSpec(): Omit { - const pullSecrets = ["registry-trigger", "registry-trigger-failover"]; - - if (ADDITIONAL_PULL_SECRETS) { - pullSecrets.push(...ADDITIONAL_PULL_SECRETS.split(",")); - } - - const imagePullSecrets = pullSecrets.map( - (name) => ({ name }) satisfies k8s.V1LocalObjectReference - ); - - return { - restartPolicy: "Never", - automountServiceAccountToken: false, - imagePullSecrets, - nodeSelector: { - nodetype: "worker", - }, - }; - } - - get #defaultResourceRequests(): ResourceQuantities { - return { - "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST, - }; - } - - get #defaultResourceLimits(): ResourceQuantities { - return { - "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT, - }; - } - - get #coordinatorHostEnvVar(): k8s.V1EnvVar { - return COORDINATOR_HOST - ? { - name: "COORDINATOR_HOST", - value: COORDINATOR_HOST, - } - : { - name: "COORDINATOR_HOST", - valueFrom: { - fieldRef: { - fieldPath: "status.hostIP", - }, - }, - }; - } - - get #coordinatorPortEnvVar(): k8s.V1EnvVar | undefined { - if (COORDINATOR_PORT) { - return { - name: "COORDINATOR_PORT", - value: COORDINATOR_PORT, - }; - } - } - - get #coordinatorEnvVars(): k8s.V1EnvVar[] { - const envVars = [this.#coordinatorHostEnvVar]; - - if (this.#coordinatorPortEnvVar) { - envVars.push(this.#coordinatorPortEnvVar); - } - - return envVars; - } - - #getSharedEnv(envId: string): k8s.V1EnvVar[] { - return [ - { - name: "TRIGGER_ENV_ID", - value: envId, - }, - { - name: "DEBUG", - value: process.env.DEBUG ? "1" : "0", - }, - { - name: "HTTP_SERVER_PORT", - value: "8000", - }, - { - name: "OTEL_EXPORTER_OTLP_ENDPOINT", - value: OTEL_EXPORTER_OTLP_ENDPOINT, - }, - { - name: "POD_NAME", - valueFrom: { - fieldRef: { - fieldPath: "metadata.name", - }, - }, - }, - { - name: "MACHINE_NAME", - valueFrom: { - fieldRef: { - fieldPath: "spec.nodeName", - }, - }, - }, - { - name: "TRIGGER_POD_SCHEDULED_AT_MS", - value: Date.now().toString(), - }, - ...this.#coordinatorEnvVars, - ]; - } - - #getSharedLabels( - opts: - | TaskOperationsIndexOptions - | TaskOperationsCreateOptions - | TaskOperationsRestoreOptions - | TaskOperationsPrePullDeploymentOptions - ): Record { - return { - env: opts.envId, - envtype: this.#envTypeToLabelValue(opts.envType), - org: opts.orgId, - project: opts.projectId, - }; - } - - #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities { - return { - cpu: `${preset.cpu * 0.75}`, - memory: `${preset.memory}G`, - }; - } - - #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities { - return { - cpu: `${preset.cpu}`, - memory: `${preset.memory}G`, - }; - } - - #getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements { - return { - requests: { - ...this.#defaultResourceRequests, - ...this.#getResourceRequestsForMachine(preset), - }, - limits: { - ...this.#defaultResourceLimits, - ...this.#getResourceLimitsForMachine(preset), - }, - }; - } - - #getLifecycleCommand( - type: THookType, - cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses - ) { - const retries = 5; - - // This will retry sending the lifecycle hook up to `retries` times - // The sleep is required as this may start running before the HTTP server is up - const exec = [ - "/bin/sh", - "-c", - `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`, - ]; - - logger.debug("getLifecycleCommand()", { exec }); - - return exec; - } - - #getIndexContainerName(suffix: string) { - return `task-index-${suffix}`; - } - - #getRunContainerName(suffix: string, attemptNumber?: number) { - return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`; - } - - #getPrePullContainerName(suffix: string) { - return `task-prepull-${suffix}`; - } - - #createK8sApi() { - const kubeConfig = new k8s.KubeConfig(); - - if (RUNTIME_ENV === "local") { - kubeConfig.loadFromDefault(); - } else if (RUNTIME_ENV === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - batch: kubeConfig.makeApiClient(k8s.BatchV1Api), - apps: kubeConfig.makeApiClient(k8s.AppsV1Api), - }; - } - - async #createPod(pod: k8s.V1Pod, namespace: Namespace) { - try { - const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #deletePod(opts: { runId: string; namespace: Namespace }) { - try { - const res = await this.#k8sApi.core.deleteNamespacedPod( - opts.runId, - opts.namespace.metadata.name - ); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #getPod(runId: string, namespace: Namespace) { - try { - const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name); - logger.debug(res.body); - return res.body; - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #createJob(job: k8s.V1Job, namespace: Namespace) { - try { - const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) { - try { - const res = await this.#k8sApi.apps.createNamespacedDaemonSet( - namespace.metadata.name, - daemonSet - ); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - #throwUnlessRecord(candidate: unknown): asserts candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - throw candidate; - } - } - - #handleK8sError(err: unknown) { - this.#throwUnlessRecord(err); - - if ("body" in err && err.body) { - logger.error(err.body); - this.#throwUnlessRecord(err.body); - - if (typeof err.body.message === "string") { - throw new Error(err.body?.message); - } else { - throw err.body; - } - } else { - logger.error(err); - throw err; - } - } -} - -type ImageType = "deployment" | "restore" | "utility"; - -function getImagePrefix(type: ImageType) { - switch (type) { - case "deployment": - return DEPLOYMENT_IMAGE_PREFIX; - case "restore": - return RESTORE_IMAGE_PREFIX; - case "utility": - return UTILITY_IMAGE_PREFIX; - default: - assertExhaustive(type); - } -} - -function getImageRef(type: ImageType, ref: string) { - const prefix = getImagePrefix(type); - return prefix ? `${prefix}/${ref}` : ref; -} - -const provider = new ProviderShell({ - tasks: new KubernetesTaskOperations({ - namespace: KUBERNETES_NAMESPACE, - }), - type: "kubernetes", -}); - -provider.listen(); - -const taskMonitor = new TaskMonitor({ - runtimeEnv: RUNTIME_ENV, - namespace: KUBERNETES_NAMESPACE, - onIndexFailure: async (deploymentId, details) => { - logger.log("Indexing failed", { deploymentId, details }); - - try { - provider.platformSocket.send("INDEXING_FAILED", { - deploymentId, - error: { - name: `Crashed with exit code ${details.exitCode}`, - message: details.reason, - stack: details.logs, - }, - overrideCompletion: details.overrideCompletion, - }); - } catch (error) { - logger.error(error); - } - }, - onRunFailure: async (runId, details) => { - logger.log("Run failed:", { runId, details }); - - try { - provider.platformSocket.send("WORKER_CRASHED", { runId, ...details }); - } catch (error) { - logger.error(error); - } - }, -}); - -taskMonitor.start(); - -const podCleaner = new PodCleaner({ - runtimeEnv: RUNTIME_ENV, - namespace: KUBERNETES_NAMESPACE, - intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS, -}); - -podCleaner.start(); - -if (UPTIME_HEARTBEAT_URL) { - const uptimeHeartbeat = new UptimeHeartbeat({ - runtimeEnv: RUNTIME_ENV, - namespace: KUBERNETES_NAMESPACE, - intervalInSeconds: UPTIME_INTERVAL_SECONDS, - pingUrl: UPTIME_HEARTBEAT_URL, - maxPendingRuns: UPTIME_MAX_PENDING_RUNS, - maxPendingIndeces: UPTIME_MAX_PENDING_INDECES, - maxPendingErrors: UPTIME_MAX_PENDING_ERRORS, - }); - - uptimeHeartbeat.start(); -} else { - logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable."); -} diff --git a/apps/kubernetes-provider/src/labelHelper.ts b/apps/kubernetes-provider/src/labelHelper.ts deleted file mode 100644 index 98cd3d68be..0000000000 --- a/apps/kubernetes-provider/src/labelHelper.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { assertExhaustive } from "@trigger.dev/core"; - -const CREATE_LABEL_ENV_VAR_PREFIX = "DEPLOYMENT_LABEL_"; -const RESTORE_LABEL_ENV_VAR_PREFIX = "RESTORE_LABEL_"; -const LABEL_SAMPLE_RATE_POSTFIX = "_SAMPLE_RATE"; - -type OperationType = "create" | "restore"; - -type CustomLabel = { - key: string; - value: string; - sampleRate: number; -}; - -export class CustomLabelHelper { - // Labels and sample rates are defined in environment variables so only need to be computed once - private createLabels?: CustomLabel[]; - private restoreLabels?: CustomLabel[]; - - private getLabelPrefix(type: OperationType) { - const prefix = type === "create" ? CREATE_LABEL_ENV_VAR_PREFIX : RESTORE_LABEL_ENV_VAR_PREFIX; - return prefix.toLowerCase(); - } - - private getLabelSampleRatePostfix() { - return LABEL_SAMPLE_RATE_POSTFIX.toLowerCase(); - } - - // Can only range from 0 to 1 - private fractionFromPercent(percent: number) { - return Math.min(1, Math.max(0, percent / 100)); - } - - private isLabelSampleRateEnvVar(key: string) { - return key.toLowerCase().endsWith(this.getLabelSampleRatePostfix()); - } - - private isLabelEnvVar(type: OperationType, key: string) { - const prefix = this.getLabelPrefix(type); - return key.toLowerCase().startsWith(prefix) && !this.isLabelSampleRateEnvVar(key); - } - - private getSampleRateEnvVarKey(type: OperationType, envKey: string) { - return `${envKey.toLowerCase()}${this.getLabelSampleRatePostfix()}`; - } - - private getLabelNameFromEnvVarKey(type: OperationType, key: string) { - return key - .slice(this.getLabelPrefix(type).length) - .toLowerCase() - .replace(/___/g, ".") - .replace(/__/g, "/") - .replace(/_/g, "-"); - } - - private getCaseInsensitiveEnvValue(key: string) { - for (const [envKey, value] of Object.entries(process.env)) { - if (envKey.toLowerCase() === key.toLowerCase()) { - return value; - } - } - } - - /** Returns the sample rate for a given label as fraction of 100 */ - private getSampleRateFromEnvVarKey(type: OperationType, envKey: string) { - // Apply default: always sample - const DEFAULT_SAMPLE_RATE_PERCENT = 100; - const defaultSampleRateFraction = this.fractionFromPercent(DEFAULT_SAMPLE_RATE_PERCENT); - - const value = this.getCaseInsensitiveEnvValue(this.getSampleRateEnvVarKey(type, envKey)); - - if (!value) { - return defaultSampleRateFraction; - } - - const sampleRatePercent = parseFloat(value || String(DEFAULT_SAMPLE_RATE_PERCENT)); - - if (isNaN(sampleRatePercent)) { - return defaultSampleRateFraction; - } - - const fractionalSampleRate = this.fractionFromPercent(sampleRatePercent); - - return fractionalSampleRate; - } - - private getCustomLabels(type: OperationType): CustomLabel[] { - switch (type) { - case "create": - if (this.createLabels) { - return this.createLabels; - } - break; - case "restore": - if (this.restoreLabels) { - return this.restoreLabels; - } - break; - default: - assertExhaustive(type); - } - - const customLabels: CustomLabel[] = []; - - for (const [envKey, value] of Object.entries(process.env)) { - const key = envKey.toLowerCase(); - - // Only process env vars that start with the expected prefix - if (!this.isLabelEnvVar(type, key)) { - continue; - } - - // Skip sample rates - deal with them separately - if (this.isLabelSampleRateEnvVar(key)) { - continue; - } - - const labelName = this.getLabelNameFromEnvVarKey(type, key); - const sampleRate = this.getSampleRateFromEnvVarKey(type, key); - - const label = { - key: labelName, - value: value || "", - sampleRate, - } satisfies CustomLabel; - - customLabels.push(label); - } - - return customLabels; - } - - getAdditionalLabels(type: OperationType): Record { - const labels = this.getCustomLabels(type); - - const additionalLabels: Record = {}; - - for (const { key, value, sampleRate } of labels) { - // Always apply label if sample rate is 1 - if (sampleRate === 1) { - additionalLabels[key] = value; - continue; - } - - if (Math.random() <= sampleRate) { - additionalLabels[key] = value; - continue; - } - } - - return additionalLabels; - } -} diff --git a/apps/kubernetes-provider/src/podCleaner.ts b/apps/kubernetes-provider/src/podCleaner.ts deleted file mode 100644 index 2cd23d1b6c..0000000000 --- a/apps/kubernetes-provider/src/podCleaner.ts +++ /dev/null @@ -1,323 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { SimpleLogger } from "@trigger.dev/core/v3/apps"; - -type PodCleanerOptions = { - runtimeEnv: "local" | "kubernetes"; - namespace?: string; - intervalInSeconds?: number; -}; - -export class PodCleaner { - private enabled = false; - private namespace = "default"; - private intervalInSeconds = 300; - - private logger = new SimpleLogger("[PodCleaner]"); - private k8sClient: { - core: k8s.CoreV1Api; - apps: k8s.AppsV1Api; - kubeConfig: k8s.KubeConfig; - }; - - constructor(private opts: PodCleanerOptions) { - if (opts.namespace) { - this.namespace = opts.namespace; - } - - if (opts.intervalInSeconds) { - this.intervalInSeconds = opts.intervalInSeconds; - } - - this.k8sClient = this.#createK8sClient(); - } - - #createK8sClient() { - const kubeConfig = new k8s.KubeConfig(); - - if (this.opts.runtimeEnv === "local") { - kubeConfig.loadFromDefault(); - } else if (this.opts.runtimeEnv === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - apps: kubeConfig.makeApiClient(k8s.AppsV1Api), - kubeConfig: kubeConfig, - }; - } - - #isRecord(candidate: unknown): candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - return false; - } else { - return true; - } - } - - #logK8sError(err: unknown, debugOnly = false) { - if (debugOnly) { - this.logger.debug("K8s API Error", err); - } else { - this.logger.error("K8s API Error", err); - } - } - - #handleK8sError(err: unknown) { - if (!this.#isRecord(err) || !this.#isRecord(err.body)) { - this.#logK8sError(err); - return; - } - - this.#logK8sError(err, true); - - if (typeof err.body.message === "string") { - this.#logK8sError({ message: err.body.message }); - return; - } - - this.#logK8sError({ body: err.body }); - } - - async #deletePods(opts: { - namespace: string; - dryRun?: boolean; - fieldSelector?: string; - labelSelector?: string; - }) { - return await this.k8sClient.core - .deleteCollectionNamespacedPod( - opts.namespace, - undefined, // pretty - undefined, // continue - opts.dryRun ? "All" : undefined, - opts.fieldSelector, - undefined, // gracePeriodSeconds - opts.labelSelector - ) - .catch(this.#handleK8sError.bind(this)); - } - - async #deleteDaemonSets(opts: { - namespace: string; - dryRun?: boolean; - fieldSelector?: string; - labelSelector?: string; - }) { - return await this.k8sClient.apps - .deleteCollectionNamespacedDaemonSet( - opts.namespace, - undefined, // pretty - undefined, // continue - opts.dryRun ? "All" : undefined, - opts.fieldSelector, - undefined, // gracePeriodSeconds - opts.labelSelector - ) - .catch(this.#handleK8sError.bind(this)); - } - - async #deleteCompletedRuns() { - this.logger.log("Deleting completed runs"); - - const start = Date.now(); - - const result = await this.#deletePods({ - namespace: this.namespace, - fieldSelector: "status.phase=Succeeded", - labelSelector: "app=task-run", - }); - - const elapsedMs = Date.now() - start; - - if (!result) { - this.logger.log("Deleting completed runs: No delete result", { elapsedMs }); - return; - } - - const total = (result.response as any)?.body?.items?.length ?? 0; - - this.logger.log("Deleting completed runs: Done", { total, elapsedMs }); - } - - async #deleteFailedRuns() { - this.logger.log("Deleting failed runs"); - - const start = Date.now(); - - const result = await this.#deletePods({ - namespace: this.namespace, - fieldSelector: "status.phase=Failed", - labelSelector: "app=task-run", - }); - - const elapsedMs = Date.now() - start; - - if (!result) { - this.logger.log("Deleting failed runs: No delete result", { elapsedMs }); - return; - } - - const total = (result.response as any)?.body?.items?.length ?? 0; - - this.logger.log("Deleting failed runs: Done", { total, elapsedMs }); - } - - async #deleteUnrecoverableRuns() { - await this.#deletePods({ - namespace: this.namespace, - fieldSelector: "status.phase=?", - labelSelector: "app=task-run", - }); - } - - async #deleteCompletedPrePulls() { - this.logger.log("Deleting completed pre-pulls"); - - const start = Date.now(); - - const result = await this.#deleteDaemonSets({ - namespace: this.namespace, - labelSelector: "app=task-prepull", - }); - - const elapsedMs = Date.now() - start; - - if (!result) { - this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs }); - return; - } - - const total = (result.response as any)?.body?.items?.length ?? 0; - - this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs }); - } - - async start() { - this.enabled = true; - this.logger.log("Starting"); - - const completedInterval = setInterval(async () => { - if (!this.enabled) { - clearInterval(completedInterval); - return; - } - - try { - await this.#deleteCompletedRuns(); - } catch (error) { - this.logger.error("Error deleting completed runs", error); - } - }, this.intervalInSeconds * 1000); - - const failedInterval = setInterval( - async () => { - if (!this.enabled) { - clearInterval(failedInterval); - return; - } - - try { - await this.#deleteFailedRuns(); - } catch (error) { - this.logger.error("Error deleting completed runs", error); - } - }, - // Use a longer interval for failed runs. This is only a backup in case the task monitor fails. - 2 * this.intervalInSeconds * 1000 - ); - - const completedPrePullInterval = setInterval( - async () => { - if (!this.enabled) { - clearInterval(completedPrePullInterval); - return; - } - - try { - await this.#deleteCompletedPrePulls(); - } catch (error) { - this.logger.error("Error deleting completed pre-pulls", error); - } - }, - 2 * this.intervalInSeconds * 1000 - ); - - // this.#launchTests(); - } - - async stop() { - if (!this.enabled) { - return; - } - - this.enabled = false; - this.logger.log("Shutting down.."); - } - - async #launchTests() { - const createPod = async ( - container: k8s.V1Container, - name: string, - labels?: Record - ) => { - this.logger.log("Creating pod:", name); - - const pod = { - metadata: { - name, - labels, - }, - spec: { - restartPolicy: "Never", - automountServiceAccountToken: false, - terminationGracePeriodSeconds: 1, - containers: [container], - }, - } satisfies k8s.V1Pod; - - await this.k8sClient.core - .createNamespacedPod(this.namespace, pod) - .catch(this.#handleK8sError.bind(this)); - }; - - const createIdlePod = async (name: string, labels?: Record) => { - const container = { - name, - image: "docker.io/library/busybox", - command: ["sh"], - args: ["-c", "sleep infinity"], - } satisfies k8s.V1Container; - - await createPod(container, name, labels); - }; - - const createCompletedPod = async (name: string, labels?: Record) => { - const container = { - name, - image: "docker.io/library/busybox", - command: ["sh"], - args: ["-c", "true"], - } satisfies k8s.V1Container; - - await createPod(container, name, labels); - }; - - const createFailedPod = async (name: string, labels?: Record) => { - const container = { - name, - image: "docker.io/library/busybox", - command: ["sh"], - args: ["-c", "false"], - } satisfies k8s.V1Container; - - await createPod(container, name, labels); - }; - - await createIdlePod("test-idle-1", { app: "task-run" }); - await createFailedPod("test-failed-1", { app: "task-run" }); - await createCompletedPod("test-completed-1", { app: "task-run" }); - } -} diff --git a/apps/kubernetes-provider/src/taskMonitor.ts b/apps/kubernetes-provider/src/taskMonitor.ts deleted file mode 100644 index aadcef18d8..0000000000 --- a/apps/kubernetes-provider/src/taskMonitor.ts +++ /dev/null @@ -1,459 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { SimpleLogger } from "@trigger.dev/core/v3/apps"; -import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core/v3/apps"; -import { setTimeout } from "timers/promises"; -import PQueue from "p-queue"; -import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3"; - -type FailureDetails = Prettify<{ - exitCode: number; - reason: string; - logs: string; - overrideCompletion: boolean; - errorCode: TaskRunInternalError["code"]; -}>; - -type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise; - -type RunFailureHandler = (runId: string, details: FailureDetails) => Promise; - -type TaskMonitorOptions = { - runtimeEnv: "local" | "kubernetes"; - onIndexFailure?: IndexFailureHandler; - onRunFailure?: RunFailureHandler; - namespace?: string; -}; - -export class TaskMonitor { - #enabled = false; - - #logger = new SimpleLogger("[TaskMonitor]"); - #taskInformer: ReturnType>; - #processedPods = new Map(); - #queue = new PQueue({ concurrency: 10 }); - - #k8sClient: { - core: k8s.CoreV1Api; - kubeConfig: k8s.KubeConfig; - }; - - private namespace = "default"; - private fieldSelector = "status.phase=Failed"; - private labelSelector = "app in (task-index, task-run)"; - - constructor(private opts: TaskMonitorOptions) { - if (opts.namespace) { - this.namespace = opts.namespace; - } - - this.#k8sClient = this.#createK8sClient(); - - this.#taskInformer = this.#createTaskInformer(); - this.#taskInformer.on("connect", this.#onInformerConnected.bind(this)); - this.#taskInformer.on("error", this.#onInformerError.bind(this)); - this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this)); - } - - #createTaskInformer() { - const listTasks = () => - this.#k8sClient.core.listNamespacedPod( - this.namespace, - undefined, - undefined, - undefined, - this.fieldSelector, - this.labelSelector - ); - - // Uses watch with local caching - // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes - const informer = k8s.makeInformer( - this.#k8sClient.kubeConfig, - `/api/v1/namespaces/${this.namespace}/pods`, - listTasks, - this.labelSelector, - this.fieldSelector - ); - - return informer; - } - - async #onInformerConnected() { - this.#logger.log("Connected"); - } - - async #onInformerError(error: any) { - this.#logger.error("Error:", error); - - // Automatic reconnect - await setTimeout(2_000); - this.#taskInformer.start(); - } - - #enqueueOnPodUpdated(pod: k8s.V1Pod) { - this.#queue.add(async () => { - try { - // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it - await this.#onPodUpdated(pod); - } catch (error) { - this.#logger.error("Caught onPodUpdated() error:", error); - } - }); - } - - async #onPodUpdated(pod: k8s.V1Pod) { - this.#logger.debug(`Updated: ${pod.metadata?.name}`); - this.#logger.debug("Updated", JSON.stringify(pod, null, 2)); - - // We only care about failures - if (pod.status?.phase !== "Failed") { - return; - } - - const podName = pod.metadata?.name; - - if (!podName) { - this.#logger.error("Pod is nameless", { pod }); - return; - } - - const containerStatus = pod.status.containerStatuses?.[0]; - - if (!containerStatus?.state) { - this.#logger.error("Pod failed, but container status doesn't have state", { - status: pod.status, - }); - return; - } - - if (this.#processedPods.has(podName)) { - this.#logger.debug("Pod update already processed", { - podName, - timestamp: this.#processedPods.get(podName), - }); - return; - } - - this.#processedPods.set(podName, Date.now()); - - const podStatus = this.#getPodStatusSummary(pod.status); - const containerState = this.#getContainerStateSummary(containerStatus.state); - const exitCode = containerState.exitCode ?? -1; - - if (exitCode === EXIT_CODE_ALREADY_HANDLED) { - this.#logger.debug("Ignoring pod failure, already handled by worker", { - podName, - }); - return; - } - - const rawLogs = await this.#getLogTail(podName); - - this.#logger.log(`${podName} failed with:`, { - podStatus, - containerState, - rawLogs, - }); - - const rawReason = podStatus.reason ?? containerState.reason ?? ""; - const message = podStatus.message ?? containerState.message ?? ""; - - let reason = rawReason || "Unknown error"; - let logs = rawLogs || ""; - - /** This will only override existing task errors. It will not crash the run. */ - let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO; - - let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR; - - switch (rawReason) { - case "Error": - reason = "Unknown error."; - errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR; - break; - case "Evicted": - if (message.startsWith("Pod ephemeral local storage usage")) { - reason = "Storage limit exceeded."; - errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED; - } else if (message) { - reason = `Evicted: ${message}`; - errorCode = TaskRunErrorCodes.POD_EVICTED; - } else { - reason = "Evicted for unknown reason."; - errorCode = TaskRunErrorCodes.POD_EVICTED; - } - - if (logs.startsWith("failed to try resolving symlinks")) { - logs = ""; - } - break; - case "OOMKilled": - reason = - "[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak."; - errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED; - break; - default: - break; - } - - const failureInfo = { - exitCode, - reason, - logs, - overrideCompletion: onlyOverrideExistingError, - errorCode, - } satisfies FailureDetails; - - const app = pod.metadata?.labels?.app; - - switch (app) { - case "task-index": - const deploymentId = pod.metadata?.labels?.deployment; - - if (!deploymentId) { - this.#logger.error("Index is missing ID", { pod }); - return; - } - - if (this.opts.onIndexFailure) { - await this.opts.onIndexFailure(deploymentId, failureInfo); - } - break; - case "task-run": - const runId = pod.metadata?.labels?.run; - - if (!runId) { - this.#logger.error("Run is missing ID", { pod }); - return; - } - - if (this.opts.onRunFailure) { - await this.opts.onRunFailure(runId, failureInfo); - } - break; - default: - this.#logger.error("Pod has invalid app label", { pod }); - return; - } - - await this.#deletePod(podName); - } - - async #getLogTail(podName: string) { - try { - const logs = await this.#k8sClient.core.readNamespacedPodLog( - podName, - this.namespace, - undefined, - undefined, - undefined, - 1024, // limitBytes - undefined, - undefined, - undefined, - 20 // tailLines - ); - - const responseBody = logs.body ?? ""; - - if (responseBody.startsWith("unable to retrieve container logs")) { - return ""; - } - - // Type is wrong, body may be undefined - return responseBody; - } catch (error) { - this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown"); - return ""; - } - } - - #getPodStatusSummary(status: k8s.V1PodStatus) { - return { - reason: status.reason, - message: status.message, - }; - } - - #getContainerStateSummary(state: k8s.V1ContainerState) { - return { - reason: state.terminated?.reason, - exitCode: state.terminated?.exitCode, - message: state.terminated?.message, - }; - } - - #createK8sClient() { - const kubeConfig = new k8s.KubeConfig(); - - if (this.opts.runtimeEnv === "local") { - kubeConfig.loadFromDefault(); - } else if (this.opts.runtimeEnv === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - kubeConfig: kubeConfig, - }; - } - - #isRecord(candidate: unknown): candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - return false; - } else { - return true; - } - } - - #logK8sError(err: unknown, debugOnly = false) { - if (debugOnly) { - this.#logger.debug("K8s API Error", err); - } else { - this.#logger.error("K8s API Error", err); - } - } - - #handleK8sError(err: unknown) { - if (!this.#isRecord(err) || !this.#isRecord(err.body)) { - this.#logK8sError(err); - return; - } - - this.#logK8sError(err, true); - - if (typeof err.body.message === "string") { - this.#logK8sError({ message: err.body.message }); - return; - } - - this.#logK8sError({ body: err.body }); - } - - #printStats(includeMoreDetails = false) { - this.#logger.log("Stats:", { - cacheSize: this.#taskInformer.list().length, - totalProcessed: this.#processedPods.size, - ...(includeMoreDetails && { - processedPods: this.#processedPods, - }), - }); - } - - async #deletePod(name: string) { - this.#logger.debug("Deleting pod:", name); - - await this.#k8sClient.core - .deleteNamespacedPod(name, this.namespace) - .catch(this.#handleK8sError.bind(this)); - } - - async start() { - this.#enabled = true; - - const interval = setInterval(() => { - if (!this.#enabled) { - clearInterval(interval); - return; - } - - this.#printStats(); - }, 300_000); - - await this.#taskInformer.start(); - - // this.#launchTests(); - } - - async stop() { - if (!this.#enabled) { - return; - } - - this.#enabled = false; - this.#logger.log("Shutting down.."); - - await this.#taskInformer.stop(); - - this.#printStats(true); - } - - async #launchTests() { - const createPod = async ( - container: k8s.V1Container, - name: string, - labels?: Record - ) => { - this.#logger.log("Creating pod:", name); - - const pod = { - metadata: { - name, - labels, - }, - spec: { - restartPolicy: "Never", - automountServiceAccountToken: false, - terminationGracePeriodSeconds: 1, - containers: [container], - }, - } satisfies k8s.V1Pod; - - await this.#k8sClient.core - .createNamespacedPod(this.namespace, pod) - .catch(this.#handleK8sError.bind(this)); - }; - - const createOomPod = async (name: string, labels?: Record) => { - const container = { - name, - image: "polinux/stress", - resources: { - limits: { - memory: "100Mi", - }, - }, - command: ["stress"], - args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"], - } satisfies k8s.V1Container; - - await createPod(container, name, labels); - }; - - const createNonZeroExitPod = async (name: string, labels?: Record) => { - const container = { - name, - image: "docker.io/library/busybox", - command: ["sh"], - args: ["-c", "exit 1"], - } satisfies k8s.V1Container; - - await createPod(container, name, labels); - }; - - const createOoDiskPod = async (name: string, labels?: Record) => { - const container = { - name, - image: "docker.io/library/busybox", - command: ["sh"], - args: [ - "-c", - "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity", - ], - resources: { - limits: { - "ephemeral-storage": "500Mi", - }, - }, - } satisfies k8s.V1Container; - - await createPod(container, name, labels); - }; - - await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" }); - await createOomPod("oom-task", { app: "task-index", deployment: "456" }); - await createOoDiskPod("ood-task", { app: "task-run", run: "abc" }); - } -} diff --git a/apps/kubernetes-provider/src/uptimeHeartbeat.ts b/apps/kubernetes-provider/src/uptimeHeartbeat.ts deleted file mode 100644 index 9ff63032f0..0000000000 --- a/apps/kubernetes-provider/src/uptimeHeartbeat.ts +++ /dev/null @@ -1,272 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { SimpleLogger } from "@trigger.dev/core/v3/apps"; - -type UptimeHeartbeatOptions = { - runtimeEnv: "local" | "kubernetes"; - pingUrl: string; - namespace?: string; - intervalInSeconds?: number; - maxPendingRuns?: number; - maxPendingIndeces?: number; - maxPendingErrors?: number; - leadingEdge?: boolean; -}; - -export class UptimeHeartbeat { - private enabled = false; - private namespace: string; - - private intervalInSeconds: number; - private maxPendingRuns: number; - private maxPendingIndeces: number; - private maxPendingErrors: number; - - private leadingEdge = true; - - private logger = new SimpleLogger("[UptimeHeartbeat]"); - private k8sClient: { - core: k8s.CoreV1Api; - kubeConfig: k8s.KubeConfig; - }; - - constructor(private opts: UptimeHeartbeatOptions) { - this.namespace = opts.namespace ?? "default"; - - this.intervalInSeconds = opts.intervalInSeconds ?? 60; - this.maxPendingRuns = opts.maxPendingRuns ?? 25; - this.maxPendingIndeces = opts.maxPendingIndeces ?? 10; - this.maxPendingErrors = opts.maxPendingErrors ?? 10; - - this.k8sClient = this.#createK8sClient(); - } - - #createK8sClient() { - const kubeConfig = new k8s.KubeConfig(); - - if (this.opts.runtimeEnv === "local") { - kubeConfig.loadFromDefault(); - } else if (this.opts.runtimeEnv === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - kubeConfig: kubeConfig, - }; - } - - #isRecord(candidate: unknown): candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - return false; - } else { - return true; - } - } - - #logK8sError(err: unknown, debugOnly = false) { - if (debugOnly) { - this.logger.debug("K8s API Error", err); - } else { - this.logger.error("K8s API Error", err); - } - } - - #handleK8sError(err: unknown) { - if (!this.#isRecord(err) || !this.#isRecord(err.body)) { - this.#logK8sError(err); - return; - } - - this.#logK8sError(err, true); - - if (typeof err.body.message === "string") { - this.#logK8sError({ message: err.body.message }); - return; - } - - this.#logK8sError({ body: err.body }); - } - - async #getPods(opts: { - namespace: string; - fieldSelector?: string; - labelSelector?: string; - }): Promise | undefined> { - const listReturn = await this.k8sClient.core - .listNamespacedPod( - opts.namespace, - undefined, // pretty - undefined, // allowWatchBookmarks - undefined, // _continue - opts.fieldSelector, - opts.labelSelector, - this.maxPendingRuns * 2, // limit - undefined, // resourceVersion - undefined, // resourceVersionMatch - undefined, // sendInitialEvents - this.intervalInSeconds, // timeoutSeconds, - undefined // watch - ) - .catch(this.#handleK8sError.bind(this)); - - return listReturn?.body.items; - } - - async #getPendingIndeces(): Promise | undefined> { - return await this.#getPods({ - namespace: this.namespace, - fieldSelector: "status.phase=Pending", - labelSelector: "app=task-index", - }); - } - - async #getPendingTasks(): Promise | undefined> { - return await this.#getPods({ - namespace: this.namespace, - fieldSelector: "status.phase=Pending", - labelSelector: "app=task-run", - }); - } - - #countPods(pods: Array): number { - return pods.length; - } - - #filterPendingPods( - pods: Array, - waitingReason: "CreateContainerError" | "RunContainerError" - ): Array { - return pods.filter((pod) => { - const containerStatus = pod.status?.containerStatuses?.[0]; - return containerStatus?.state?.waiting?.reason === waitingReason; - }); - } - - async #sendPing() { - this.logger.log("Sending ping"); - - const start = Date.now(); - const controller = new AbortController(); - - const timeoutMs = (this.intervalInSeconds * 1000) / 2; - - const fetchTimeout = setTimeout(() => { - controller.abort(); - }, timeoutMs); - - try { - const response = await fetch(this.opts.pingUrl, { - signal: controller.signal, - }); - - if (!response.ok) { - this.logger.error("Failed to send ping, response not OK", { - status: response.status, - }); - return; - } - - const elapsedMs = Date.now() - start; - this.logger.log("Ping sent", { elapsedMs }); - } catch (error) { - if (error instanceof DOMException && error.name === "AbortError") { - this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs }); - return; - } - - this.logger.error("Failed to send ping", error); - } finally { - clearTimeout(fetchTimeout); - } - } - - async #heartbeat() { - this.logger.log("Performing heartbeat"); - - const start = Date.now(); - - const pendingTasks = await this.#getPendingTasks(); - - if (!pendingTasks) { - this.logger.error("Failed to get pending tasks"); - return; - } - - const totalPendingTasks = this.#countPods(pendingTasks); - - const pendingIndeces = await this.#getPendingIndeces(); - - if (!pendingIndeces) { - this.logger.error("Failed to get pending indeces"); - return; - } - - const totalPendingIndeces = this.#countPods(pendingIndeces); - - const elapsedMs = Date.now() - start; - - this.logger.log("Finished heartbeat checks", { elapsedMs }); - - if (totalPendingTasks > this.maxPendingRuns) { - this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks }); - return; - } - - if (totalPendingIndeces > this.maxPendingIndeces) { - this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces }); - return; - } - - const totalCreateContainerErrors = this.#countPods( - this.#filterPendingPods(pendingTasks, "CreateContainerError") - ); - const totalRunContainerErrors = this.#countPods( - this.#filterPendingPods(pendingTasks, "RunContainerError") - ); - - if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) { - this.logger.log("Too many pending tasks with errors, skipping heartbeat", { - totalRunContainerErrors, - totalCreateContainerErrors, - }); - return; - } - - await this.#sendPing(); - - this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs }); - } - - async start() { - this.enabled = true; - this.logger.log("Starting"); - - if (this.leadingEdge) { - await this.#heartbeat(); - } - - const heartbeat = setInterval(async () => { - if (!this.enabled) { - clearInterval(heartbeat); - return; - } - - try { - await this.#heartbeat(); - } catch (error) { - this.logger.error("Error while heartbeating", error); - } - }, this.intervalInSeconds * 1000); - } - - async stop() { - if (!this.enabled) { - return; - } - - this.enabled = false; - this.logger.log("Shutting down.."); - } -} diff --git a/apps/kubernetes-provider/tsconfig.json b/apps/kubernetes-provider/tsconfig.json deleted file mode 100644 index 6ec7865b64..0000000000 --- a/apps/kubernetes-provider/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "strict": true, - "skipLibCheck": true, - "paths": { - "@trigger.dev/core": ["../../packages/core/src"], - "@trigger.dev/core/*": ["../../packages/core/src/*"], - "@trigger.dev/core/v3": ["../../packages/core/src/v3"], - "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] - } - } -} diff --git a/apps/supervisor/.env.example b/apps/supervisor/.env.example index 5cb86d5a33..f71cb88f3e 100644 --- a/apps/supervisor/.env.example +++ b/apps/supervisor/.env.example @@ -1,8 +1,8 @@ # This needs to match the token of the worker group you want to connect to TRIGGER_WORKER_TOKEN= -# This needs to match the MANAGED_WORKER_SECRET env var on the webapp -MANAGED_WORKER_SECRET=managed-secret +# Must match the webapp's MANAGED_WORKER_SECRET. Generate with: openssl rand -hex 16 +MANAGED_WORKER_SECRET= # Point this at the webapp in prod TRIGGER_API_URL=http://localhost:3030 @@ -14,4 +14,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:3030/otel # Optional settings DEBUG=1 -TRIGGER_DEQUEUE_INTERVAL_MS=1000 \ No newline at end of file +TRIGGER_DEQUEUE_INTERVAL_MS=1000 + +# Dev-only: stream this process's logs over a local telnet/TCP socket (nc localhost 6769). Uncomment to enable. +# SUPERVISOR_TELNET_LOGS_PORT=6769 diff --git a/apps/supervisor/.nvmrc b/apps/supervisor/.nvmrc index dc0bb0f439..5bcf9c6e6a 100644 --- a/apps/supervisor/.nvmrc +++ b/apps/supervisor/.nvmrc @@ -1 +1 @@ -v22.12.0 +v24.18.0 diff --git a/apps/supervisor/CLAUDE.md b/apps/supervisor/CLAUDE.md new file mode 100644 index 0000000000..9cc7c82fd9 --- /dev/null +++ b/apps/supervisor/CLAUDE.md @@ -0,0 +1,20 @@ +# Supervisor + +Node.js app that manages task execution containers. Receives work from the platform, starts Docker/Kubernetes containers, monitors execution, and reports results. + +## Key Directories + +- `src/services/` - Core service logic +- `src/workloadManager/` - Container orchestration abstraction (Docker or Kubernetes) +- `src/workloadServer/` - HTTP server for workload communication (heartbeats, snapshots) +- `src/clients/` - Platform communication (webapp) +- `src/env.ts` - Environment configuration + +## Architecture + +- **WorkloadManager**: Abstracts Docker vs Kubernetes execution +- **SupervisorSession**: Manages the dequeue loop with EWMA-based dynamic scaling +- **ResourceMonitor**: Tracks CPU/memory during execution +- **PodCleaner/FailedPodHandler**: Kubernetes-specific cleanup + +Communicates with the platform via Socket.io and HTTP. Receives task assignments through the dequeue protocol from the webapp. diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index d5bb5862e9..edc6ba2ee0 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -1,13 +1,13 @@ -FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine +FROM node:24.18.0-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS node-24-alpine WORKDIR /app -FROM node-22-alpine AS pruner +FROM node-24-alpine AS pruner COPY --chown=node:node . . -RUN npx -q turbo@2.5.4 prune --scope=supervisor --docker +RUN npx -q turbo@2.10.0 prune --scope=supervisor --docker -FROM node-22-alpine AS base +FROM node-24-alpine AS base RUN apk add --no-cache dumb-init @@ -16,7 +16,7 @@ COPY --from=pruner --chown=node:node /app/out/json/ . COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml -RUN corepack enable && corepack prepare pnpm@10.23.0 --activate +RUN corepack enable && corepack prepare pnpm@10.33.2 --activate FROM base AS deps-fetcher RUN apk add --no-cache python3-dev py3-setuptools make g++ gcc linux-headers @@ -34,6 +34,7 @@ COPY --from=dev-deps --chown=node:node /app/ . COPY --chown=node:node turbo.json turbo.json COPY --chown=node:node .configs/tsconfig.base.json .configs/tsconfig.base.json COPY --chown=node:node scripts/updateVersion.ts scripts/updateVersion.ts +COPY --chown=node:node scripts/retry-prisma-generate.mjs scripts/retry-prisma-generate.mjs RUN pnpm run generate && \ pnpm run --filter supervisor... build&& \ diff --git a/apps/supervisor/package.json b/apps/supervisor/package.json index e9609bf154..6f70b603b4 100644 --- a/apps/supervisor/package.json +++ b/apps/supervisor/package.json @@ -14,15 +14,19 @@ }, "dependencies": { "@aws-sdk/client-ecr": "^3.839.0", + "@internal/compute": "workspace:*", "@kubernetes/client-node": "^1.0.0", "@trigger.dev/core": "workspace:*", "dockerode": "^4.0.6", + "ioredis": "~5.6.0", + "p-limit": "^6.2.0", "prom-client": "^15.1.0", - "socket.io": "4.7.4", + "socket.io": "4.8.3", "std-env": "^3.8.0", "zod": "3.25.76" }, "devDependencies": { + "@internal/testcontainers": "workspace:*", "@types/dockerode": "^3.3.33" } } diff --git a/apps/supervisor/src/backpressure/backpressureMetrics.ts b/apps/supervisor/src/backpressure/backpressureMetrics.ts new file mode 100644 index 0000000000..9b622357a3 --- /dev/null +++ b/apps/supervisor/src/backpressure/backpressureMetrics.ts @@ -0,0 +1,42 @@ +import { Counter, Gauge, type Registry } from "prom-client"; + +/** Prometheus metrics for dequeue backpressure. */ +export class BackpressureMetrics { + /** 1 while backpressure is engaged (computed signal, set even in dry-run). */ + readonly engaged: Gauge; + /** 1 when running in dry-run (gates inert). */ + readonly dryRun: Gauge; + /** Dequeue attempts the gate skipped - or would have, in dry-run (labelled). */ + readonly skipsTotal: Counter; + /** Verdict source reads that failed (threw). */ + readonly readFailuresTotal: Counter; + + constructor(opts: { register: Registry; prefix?: string }) { + const prefix = opts.prefix ?? "supervisor_backpressure"; + + this.engaged = new Gauge({ + name: `${prefix}_engaged`, + help: "1 while dequeue backpressure is engaged (computed signal, regardless of dry-run)", + registers: [opts.register], + }); + + this.dryRun = new Gauge({ + name: `${prefix}_dry_run`, + help: "1 when dequeue backpressure is in dry-run mode (gates inert)", + registers: [opts.register], + }); + + this.skipsTotal = new Counter({ + name: `${prefix}_skipped_dequeues_total`, + help: "Dequeue attempts skipped by backpressure (or would be, in dry-run)", + labelNames: ["dry_run"], + registers: [opts.register], + }); + + this.readFailuresTotal = new Counter({ + name: `${prefix}_read_failures_total`, + help: "Verdict source reads that threw", + registers: [opts.register], + }); + } +} diff --git a/apps/supervisor/src/backpressure/backpressureMonitor.test.ts b/apps/supervisor/src/backpressure/backpressureMonitor.test.ts new file mode 100644 index 0000000000..e6c00394bb --- /dev/null +++ b/apps/supervisor/src/backpressure/backpressureMonitor.test.ts @@ -0,0 +1,408 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { Registry } from "prom-client"; +import { BackpressureMonitor, type BackpressureSignalSource } from "./backpressureMonitor.js"; +import { BackpressureMetrics } from "./backpressureMetrics.js"; + +function countingSource(verdict: { engaged: boolean } | null): { + source: BackpressureSignalSource; + reads: () => number; +} { + let reads = 0; + return { + source: { + read: async () => { + reads++; + return verdict; + }, + }, + reads: () => reads, + }; +} + +describe("BackpressureMonitor", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("when disabled, never skips dequeue and never reads the signal source", () => { + // Even though the source would report "engaged", a disabled monitor must be + // a complete no-op: this is the backwards-compatibility guarantee. + const { source, reads } = countingSource({ engaged: true }); + const monitor = new BackpressureMonitor({ enabled: false, source }); + + monitor.start(); + + expect(monitor.shouldSkipDequeue()).toBe(false); + expect(reads()).toBe(0); + + monitor.stop(); + }); + + it("when enabled and the source reports engaged, skips dequeue after a refresh", async () => { + const { source } = countingSource({ engaged: true }); + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); // flush the initial async read + + expect(monitor.shouldSkipDequeue()).toBe(true); + + monitor.stop(); + }); + + it("when enabled and the source reports clear, does not skip dequeue", async () => { + const { source } = countingSource({ engaged: false }); + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(monitor.shouldSkipDequeue()).toBe(false); + + monitor.stop(); + }); + + it("fails open (stops skipping) when the source throws", async () => { + let call = 0; + const source: BackpressureSignalSource = { + read: async () => { + call++; + if (call === 1) { + return { engaged: true }; + } + throw new Error("signal source unreachable"); + }, + }; + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); // engaged from the first read + + await vi.advanceTimersByTimeAsync(1000); // next refresh throws + expect(monitor.shouldSkipDequeue()).toBe(false); // fail-open: a dead source must not pin the brake + + monitor.stop(); + }); + + it("holds an engaged verdict while reads fail, then releases past the max age", async () => { + let call = 0; + const source: BackpressureSignalSource = { + read: async () => { + call++; + if (call === 1) { + return { engaged: true, ts: Date.now() }; + } + throw new Error("signal source unreachable"); + }, + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); + + await vi.advanceTimersByTimeAsync(5000); + expect(monitor.shouldSkipDequeue()).toBe(true); // read failing, verdict held + + await vi.advanceTimersByTimeAsync(11_000); + expect(monitor.shouldSkipDequeue()).toBe(false); // past max age, released + + monitor.stop(); + }); + + it("releases immediately on an explicit null even when a grace window is configured", async () => { + let engaged: boolean | null = true; + const source: BackpressureSignalSource = { + read: async () => (engaged === null ? null : { engaged, ts: Date.now() }), + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); + + engaged = null; + await vi.advanceTimersByTimeAsync(1000); + expect(monitor.shouldSkipDequeue()).toBe(false); // null is an answer, not a failure + + monitor.stop(); + }); + + it("fails open when the source reports unknown (null)", async () => { + const { source } = countingSource(null); + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(monitor.shouldSkipDequeue()).toBe(false); + + monitor.stop(); + }); + + it("fails open when the cached verdict goes stale (older than max age)", async () => { + // Source stops updating (e.g. hangs) after the first read; the verdict ages out. + const source: BackpressureSignalSource = { + read: async () => ({ engaged: true, ts: Date.now() }), + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1_000_000, // effectively only the initial read fires + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); + + await vi.advanceTimersByTimeAsync(15_001); // verdict now older than max age + expect(monitor.shouldSkipDequeue()).toBe(false); + + monitor.stop(); + }); + + it("does not read the source on the hot path (reads are driven by the refresh tick)", async () => { + const { source, reads } = countingSource({ engaged: true }); + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(reads()).toBe(1); // just the initial refresh + + for (let i = 0; i < 1000; i++) { + monitor.shouldSkipDequeue(); + } + + expect(reads()).toBe(1); // hot-path calls performed zero I/O + + monitor.stop(); + }); + + it("does not start an overlapping refresh while one is in flight", async () => { + let reads = 0; + const source: BackpressureSignalSource = { + // Never resolves - simulates a hung read. + read: () => { + reads++; + return new Promise<{ engaged: boolean } | null>(() => {}); + }, + }; + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(3000); // several intervals while the first read hangs + + expect(reads).toBe(1); // in-flight guard prevents stacking + + monitor.stop(); + }); + + it("stops refreshing after stop()", async () => { + const { source, reads } = countingSource({ engaged: true }); + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + const readsAtStop = reads(); + + monitor.stop(); + await vi.advanceTimersByTimeAsync(5000); + + expect(reads()).toBe(readsAtStop); + }); + + it("isEngaged reflects the hard engaged state (the signal for freezing scale-up)", async () => { + const { source } = countingSource({ engaged: true }); + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(monitor.isEngaged()).toBe(true); + + monitor.stop(); + }); + + it("isEngaged is false when clear and when stale", async () => { + const source: BackpressureSignalSource = { + read: async () => ({ engaged: true, ts: Date.now() }), + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1_000_000, + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.isEngaged()).toBe(true); + + await vi.advanceTimersByTimeAsync(15_001); // stale โ†’ fail-open + expect(monitor.isEngaged()).toBe(false); + + monitor.stop(); + }); + + it("ramps the dequeue gate after release instead of resuming instantly", async () => { + let engaged = true; + let rnd = 0.5; + const source: BackpressureSignalSource = { read: async () => ({ engaged }) }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + rampMs: 10_000, + random: () => rnd, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); // hard engaged + + // Release: the next refresh observes the clear verdict and starts the ramp. + engaged = false; + await vi.advanceTimersByTimeAsync(1000); + expect(monitor.isEngaged()).toBe(false); + + // Just after release (progress ~0): skip probability ~1, so skip regardless. + rnd = 0.99; + expect(monitor.shouldSkipDequeue()).toBe(true); + + // Halfway through the ramp (progress 0.5): skip probability 0.5. + await vi.advanceTimersByTimeAsync(5000); + rnd = 0.4; + expect(monitor.shouldSkipDequeue()).toBe(true); // 0.4 < 0.5 โ†’ skip + rnd = 0.6; + expect(monitor.shouldSkipDequeue()).toBe(false); // 0.6 โ‰ฅ 0.5 โ†’ allow + + // Past the ramp window: never skip. + await vi.advanceTimersByTimeAsync(5000); + rnd = 0.0; + expect(monitor.shouldSkipDequeue()).toBe(false); + + monitor.stop(); + }); + + it("fails open on an engaged verdict with no timestamp when staleness is enforced", async () => { + // A verdict claiming engaged but carrying no ts can't be checked for freshness; + // when maxVerdictAgeMs is set we must not trust it (else a dead producer could + // pin the brake forever). + const { source } = countingSource({ engaged: true }); // no ts + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(monitor.computeEngaged()).toBe(false); + expect(monitor.shouldSkipDequeue()).toBe(false); + + monitor.stop(); + }); + + it("in dry-run, the gates are inert but computeEngaged still reflects the real signal", async () => { + const { source } = countingSource({ engaged: true }); + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + dryRun: true, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(monitor.computeEngaged()).toBe(true); // real signal, for observability/metrics + expect(monitor.isEngaged()).toBe(false); // inert: no scale-up freeze + expect(monitor.shouldSkipDequeue()).toBe(false); // inert: no dequeue skip + + monitor.stop(); + }); + + it("logs on verdict transitions", async () => { + let engaged = true; + const source: BackpressureSignalSource = { read: async () => ({ engaged }) }; + const logs: Array<{ message: string; meta?: Record }> = []; + const logger = { + info: (message: string, meta?: Record) => logs.push({ message, meta }), + error: (message: string, meta?: Record) => logs.push({ message, meta }), + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + logger, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(logs.some((l) => l.meta?.engaged === true)).toBe(true); + + engaged = false; + await vi.advanceTimersByTimeAsync(1000); + expect(logs.some((l) => l.meta?.engaged === false)).toBe(true); + + monitor.stop(); + }); + + it("records prometheus metrics", async () => { + const { source } = countingSource({ engaged: true }); + const register = new Registry(); + const metrics = new BackpressureMetrics({ register }); + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + metrics, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(await register.metrics()).toContain("supervisor_backpressure_engaged 1"); + + monitor.shouldSkipDequeue(); + expect(await register.metrics()).toMatch( + /supervisor_backpressure_skipped_dequeues_total\{dry_run="false"\} [1-9]/ + ); + + monitor.stop(); + }); + + it("resumes instantly when no ramp is configured", async () => { + let engaged = true; + const source: BackpressureSignalSource = { read: async () => ({ engaged }) }; + const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); + + engaged = false; + await vi.advanceTimersByTimeAsync(1000); + expect(monitor.shouldSkipDequeue()).toBe(false); // no ramp โ†’ instant resume + + monitor.stop(); + }); +}); diff --git a/apps/supervisor/src/backpressure/backpressureMonitor.ts b/apps/supervisor/src/backpressure/backpressureMonitor.ts new file mode 100644 index 0000000000..aa16fdeaa6 --- /dev/null +++ b/apps/supervisor/src/backpressure/backpressureMonitor.ts @@ -0,0 +1,202 @@ +import type { BackpressureMetrics } from "./backpressureMetrics.js"; + +export interface BackpressureLogger { + info(message: string, meta?: Record): void; + error(message: string, meta?: Record): void; +} + +export type BackpressureVerdict = { + engaged: boolean; + /** Epoch ms the verdict was produced. Used for consumer-side staleness fail-open. */ + ts?: number; +}; + +/** + * Source of the current backpressure verdict. `read()` returns `null` when the source + * answered but there is no verdict - the monitor treats that as "not engaged" + * (fail-open). A thrown error is different: the read itself failed, so the monitor + * keeps the previous verdict until it ages past `maxVerdictAgeMs`. + */ +export interface BackpressureSignalSource { + read(): Promise; +} + +export type BackpressureMonitorOptions = { + enabled: boolean; + source: BackpressureSignalSource; + refreshIntervalMs?: number; + /** + * If set, an engaged verdict older than this is released (fail-open), bounding how + * long a dead source can hold the brake. Reads that fail keep the last verdict, so + * this doubles as the grace window for riding out a transient source outage. + */ + maxVerdictAgeMs?: number; + /** + * If set, after backpressure releases the dequeue gate stays partially engaged + * for this long, skipping a linearly-decaying fraction of attempts so the + * aggregate dequeue rate ramps from ~0 to full instead of snapping to full and + * re-flooding a freshly-recovered cluster. 0/unset = instant resume. + */ + rampMs?: number; + /** Injectable RNG for the resume ramp; defaults to Math.random. */ + random?: () => number; + /** + * When true, the gates are inert (never skip dequeues, never freeze scale-up). + * computeEngaged() still reflects the real signal so it can be observed. + */ + dryRun?: boolean; + logger?: BackpressureLogger; + metrics?: BackpressureMetrics; +}; + +const DEFAULT_REFRESH_INTERVAL_MS = 1000; + +export class BackpressureMonitor { + private verdict: BackpressureVerdict | null = null; + private timer?: ReturnType; + private refreshInFlight = false; + private wasEngaged = false; + private releasedAt?: number; + private readFailing = false; + + constructor(private readonly opts: BackpressureMonitorOptions) { + this.opts.metrics?.dryRun.set(this.opts.dryRun ? 1 : 0); + } + + start(): void { + if (!this.opts.enabled) { + return; + } + + void this.refreshTick(); + this.timer = setInterval( + () => void this.refreshTick(), + this.opts.refreshIntervalMs ?? DEFAULT_REFRESH_INTERVAL_MS + ); + } + + /** Skip a tick if the previous refresh is still in flight, so slow/hung reads can't stack. */ + private async refreshTick(): Promise { + if (this.refreshInFlight) { + return; + } + this.refreshInFlight = true; + try { + await this.refresh(); + } finally { + this.refreshInFlight = false; + } + } + + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = undefined; + } + } + + /** + * Raw hard backpressure state: true while the (fresh) verdict says engaged, + * ignoring dry-run. Used for observability/metrics so the real signal is + * visible even when the gates are inert. + */ + computeEngaged(): boolean { + const verdict = this.verdict; + if (verdict?.engaged !== true) { + return false; + } + + // When staleness enforcement is on, an engaged verdict must carry a fresh + // timestamp. A missing or stale ts can't be trusted (a dead producer could + // otherwise pin the brake forever), so fail open. + const maxAge = this.opts.maxVerdictAgeMs; + if (maxAge !== undefined) { + if (verdict.ts === undefined || Date.now() - verdict.ts > maxAge) { + return false; + } + } + + return true; + } + + /** + * Effective hard state: the signal for freezing consumer-pool scale-up. Inert + * (false) in dry-run. Hot-path read, no I/O. + */ + isEngaged(): boolean { + return this.opts.dryRun ? false : this.computeEngaged(); + } + + /** Hot-path read: synchronous, never performs I/O. Inert (false) in dry-run. */ + shouldSkipDequeue(): boolean { + const wouldSkip = this.computeShouldSkip(); + if (wouldSkip) { + this.opts.metrics?.skipsTotal.inc({ dry_run: this.opts.dryRun ? "true" : "false" }); + } + return this.opts.dryRun ? false : wouldSkip; + } + + private computeShouldSkip(): boolean { + if (this.computeEngaged()) { + return true; + } + + // Post-release ramp: skip a linearly-decaying fraction of attempts so the + // aggregate dequeue rate climbs back to full over rampMs rather than snapping. + const rampMs = this.opts.rampMs; + if (rampMs && this.releasedAt !== undefined) { + const elapsed = Date.now() - this.releasedAt; + if (elapsed < rampMs) { + const skipProbability = 1 - elapsed / rampMs; + return (this.opts.random ?? Math.random)() < skipProbability; + } + } + + return false; + } + + private async refresh(): Promise { + let next: BackpressureVerdict | null = null; + let readError: unknown; + try { + next = await this.opts.source.read(); + } catch (error) { + readError = error; + } + + if (readError === undefined) { + this.verdict = next; // an explicit null means "no pressure", so honour it + this.readFailing = false; + } else { + const held = this.opts.maxVerdictAgeMs !== undefined; + if (!held) { + this.verdict = null; // unbounded hold could pin the brake forever + } + this.opts.metrics?.readFailuresTotal.inc(); + if (!this.readFailing) { + this.readFailing = true; // log once per outage, not once per tick + this.opts.logger?.error("backpressure read failed", { + reason: String(readError), + heldPreviousVerdict: held, + engaged: this.computeEngaged(), + }); + } + } + + // Track the engagedโ†’released transition to anchor the resume ramp. Use the + // staleness-aware state so a stale verdict doesn't pin wasEngaged / the gauge. + const nowEngaged = this.computeEngaged(); + this.opts.metrics?.engaged.set(nowEngaged ? 1 : 0); + + if (nowEngaged !== this.wasEngaged) { + this.opts.logger?.info("backpressure verdict changed", { + engaged: nowEngaged, + dryRun: !!this.opts.dryRun, + }); + } + if (this.wasEngaged && !nowEngaged) { + this.releasedAt = Date.now(); + } + this.wasEngaged = nowEngaged; + } +} diff --git a/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts new file mode 100644 index 0000000000..12cd92e2d8 --- /dev/null +++ b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts @@ -0,0 +1,92 @@ +import { describe, it, expect } from "vitest"; +import { K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js"; +import { podCountFromList, withTimeout } from "../clients/kubernetes.js"; + +describe("podCountFromList", () => { + it("returns items.length when the list is not truncated", () => { + expect(podCountFromList({ items: [{}], metadata: {} })).toBe(1); + }); + + it("returns zero for an empty namespace", () => { + expect(podCountFromList({ items: [], metadata: {} })).toBe(0); + }); + + it("adds remainingItemCount when the list is truncated", () => { + const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: 24492 } }; + expect(podCountFromList(list)).toBe(24493); + }); + + it("throws when truncated but remainingItemCount is absent", () => { + const list = { items: [{}], metadata: { _continue: "tok" } }; + expect(() => podCountFromList(list)).toThrow(/remainingItemCount/); + }); + + it("throws when truncated but remainingItemCount is negative", () => { + const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: -1 } }; + expect(() => podCountFromList(list)).toThrow(/remainingItemCount/); + }); +}); + +describe("withTimeout", () => { + it("rejects once the deadline passes", async () => { + await expect(withTimeout(new Promise(() => {}), 10, "pod count list")).rejects.toThrow( + /timed out/ + ); + }); + + it("passes a value through when it settles first", async () => { + await expect(withTimeout(Promise.resolve(7), 1000, "pod count list")).resolves.toBe(7); + }); +}); + +describe("K8sPodCountSignalSource", () => { + it("engages at the engage threshold and reports the count", async () => { + const counts: number[] = []; + const source = new K8sPodCountSignalSource({ + fetchPodCount: async () => 10000, + engageThreshold: 10000, + releaseThreshold: 5000, + reportPodCount: (c) => counts.push(c), + }); + const verdict = await source.read(); + expect(verdict.engaged).toBe(true); + expect(typeof verdict.ts).toBe("number"); + expect(counts).toEqual([10000]); + }); + + it("does not engage below the engage threshold", async () => { + const source = new K8sPodCountSignalSource({ + fetchPodCount: async () => 9999, + engageThreshold: 10000, + releaseThreshold: 5000, + }); + expect((await source.read()).engaged).toBe(false); + }); + + it("stays engaged in the hysteresis band, releases only below release threshold", async () => { + let count = 10000; + const source = new K8sPodCountSignalSource({ + fetchPodCount: async () => count, + engageThreshold: 10000, + releaseThreshold: 5000, + }); + expect((await source.read()).engaged).toBe(true); // engage + count = 7000; + expect((await source.read()).engaged).toBe(true); // band -> still engaged + count = 4999; + expect((await source.read()).engaged).toBe(false); // below release -> off + count = 7000; + expect((await source.read()).engaged).toBe(false); // band again -> stays off + }); + + it("propagates fetch failures (monitor fails open on throw)", async () => { + const source = new K8sPodCountSignalSource({ + fetchPodCount: async () => { + throw new Error("connection refused"); + }, + engageThreshold: 10000, + releaseThreshold: 5000, + }); + await expect(source.read()).rejects.toThrow("connection refused"); + }); +}); diff --git a/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts new file mode 100644 index 0000000000..949ebbfa02 --- /dev/null +++ b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts @@ -0,0 +1,30 @@ +import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js"; + +export type K8sPodCountSignalSourceOptions = { + fetchPodCount: () => Promise; + engageThreshold: number; + releaseThreshold: number; + reportPodCount?: (count: number) => void; +}; + +// Engage/release with hysteresis so a count hovering near the line doesn't flap. +export class K8sPodCountSignalSource implements BackpressureSignalSource { + private engaged = false; + + constructor(private readonly opts: K8sPodCountSignalSourceOptions) {} + + async read(): Promise { + const count = await this.opts.fetchPodCount(); + this.opts.reportPodCount?.(count); + + if (this.engaged) { + if (count < this.opts.releaseThreshold) { + this.engaged = false; + } + } else if (count >= this.opts.engageThreshold) { + this.engaged = true; + } + + return { engaged: this.engaged, ts: Date.now() }; + } +} diff --git a/apps/supervisor/src/backpressure/redisBackpressureSignalSource.test.ts b/apps/supervisor/src/backpressure/redisBackpressureSignalSource.test.ts new file mode 100644 index 0000000000..dc8040a4c5 --- /dev/null +++ b/apps/supervisor/src/backpressure/redisBackpressureSignalSource.test.ts @@ -0,0 +1,65 @@ +import { redisTest } from "@internal/testcontainers"; +import { Redis } from "ioredis"; +import { describe, expect } from "vitest"; +import { RedisBackpressureSignalSource } from "./redisBackpressureSignalSource.js"; + +const KEY = "backpressure:test"; + +describe("RedisBackpressureSignalSource", () => { + redisTest("returns null when the key is absent", async ({ redisOptions }) => { + const redis = new Redis(redisOptions); + try { + const source = new RedisBackpressureSignalSource(redis, KEY); + expect(await source.read()).toBeNull(); + } finally { + await redis.quit(); + } + }); + + redisTest("parses a valid engaged verdict", async ({ redisOptions }) => { + const redis = new Redis(redisOptions); + try { + await redis.set(KEY, JSON.stringify({ engaged: true, ts: 1_700_000_000_000 })); + const source = new RedisBackpressureSignalSource(redis, KEY); + expect(await source.read()).toEqual({ engaged: true, ts: 1_700_000_000_000 }); + } finally { + await redis.quit(); + } + }); + + redisTest("parses a clear verdict", async ({ redisOptions }) => { + const redis = new Redis(redisOptions); + try { + await redis.set(KEY, JSON.stringify({ engaged: false })); + const source = new RedisBackpressureSignalSource(redis, KEY); + expect(await source.read()).toEqual({ engaged: false }); + } finally { + await redis.quit(); + } + }); + + redisTest("returns null for malformed JSON (fail-open)", async ({ redisOptions }) => { + const redis = new Redis(redisOptions); + try { + await redis.set(KEY, "not json {"); + const source = new RedisBackpressureSignalSource(redis, KEY); + expect(await source.read()).toBeNull(); + } finally { + await redis.quit(); + } + }); + + redisTest( + "returns null for valid JSON of the wrong shape (fail-open)", + async ({ redisOptions }) => { + const redis = new Redis(redisOptions); + try { + await redis.set(KEY, JSON.stringify({ foo: "bar" })); + const source = new RedisBackpressureSignalSource(redis, KEY); + expect(await source.read()).toBeNull(); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/apps/supervisor/src/backpressure/redisBackpressureSignalSource.ts b/apps/supervisor/src/backpressure/redisBackpressureSignalSource.ts new file mode 100644 index 0000000000..4f8a54c624 --- /dev/null +++ b/apps/supervisor/src/backpressure/redisBackpressureSignalSource.ts @@ -0,0 +1,35 @@ +import type { Redis } from "ioredis"; +import { z } from "zod"; +import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js"; + +const VerdictSchema = z.object({ + engaged: z.boolean(), + ts: z.number().optional(), +}); + +/** Reads the backpressure verdict from a Redis key written by the cluster-side aggregator. */ +export class RedisBackpressureSignalSource implements BackpressureSignalSource { + constructor( + private readonly redis: Redis, + private readonly key: string + ) {} + + async read(): Promise { + const raw = await this.redis.get(this.key); + if (raw === null) { + return null; + } + + // A malformed or wrong-shaped value is treated as unknown (null) so the + // monitor fails open rather than acting on garbage. + let json: unknown; + try { + json = JSON.parse(raw); + } catch { + return null; + } + + const parsed = VerdictSchema.safeParse(json); + return parsed.success ? parsed.data : null; + } +} diff --git a/apps/supervisor/src/clients/kubernetes.ts b/apps/supervisor/src/clients/kubernetes.ts index f66e57e435..1e511a68e6 100644 --- a/apps/supervisor/src/clients/kubernetes.ts +++ b/apps/supervisor/src/clients/kubernetes.ts @@ -1,7 +1,5 @@ import * as k8s from "@kubernetes/client-node"; -import { Informer } from "@kubernetes/client-node"; -import { ListPromise } from "@kubernetes/client-node"; -import { KubernetesObject } from "@kubernetes/client-node"; +import type { Informer, KubernetesObject, ListPromise } from "@kubernetes/client-node"; import { assertExhaustive } from "@trigger.dev/core/utils"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; @@ -53,3 +51,92 @@ function getKubeConfig() { } export { k8s }; + +/** + * createPodCountFetcher sizes a namespace's pod collection with a single `limit=1` + * list: one pod transferred, no informer, no watch cache. + * + * This is an ESTIMATE, not an exact count. Kubernetes documents `remainingItemCount` + * as intended for estimating collection size and reserves the right not to set it or + * make it exact. Counting exactly would mean paginating the whole collection, which is + * what this deliberately avoids. Treat the value as a tight estimate from a quorum read + * at request time, and set thresholds with that in mind. + * + * Two request-shape constraints, both load-bearing. A label or field selector makes + * the apiserver omit `remainingItemCount` entirely, and setting `resourceVersion` + * serves a cached count instead of a quorum read - so neither is passed. + */ +export function createPodCountFetcher( + api: K8sApi, + namespace: string, + timeoutMs: number +): () => Promise { + const serverTimeoutSeconds = Math.max(1, Math.floor(timeoutMs / 1000)); + let pending: Promise | undefined; + + return async () => { + if (pending) { + throw new Error("pod count list still in flight from a previous tick"); + } + + const request = api.core.listNamespacedPod({ + namespace, + limit: 1, + timeoutSeconds: serverTimeoutSeconds, + }); + + pending = request + .catch(() => {}) + .finally(() => { + pending = undefined; + }); + + return podCountFromList(await withTimeout(request, timeoutMs, "pod count list")); + }; +} + +/** + * podCountFromList turns a `limit=1` pod list into a population estimate. + * + * `remainingItemCount` is only set when the list is truncated, so `_continue` is the + * truncation signal: absent means the returned page is the whole collection and its + * length is exact. When truncated the total leans on `remainingItemCount`, which is + * documented as an estimate - so the result is an estimate too. Truncated without a + * usable count is unknowable, so it throws rather than returning a low number the + * caller would act on. + */ +export function podCountFromList(list: { + items: unknown[]; + metadata?: { _continue?: string; remainingItemCount?: number }; +}): number { + if (!list.metadata?._continue) { + return list.items.length; + } + + const remaining = list.metadata.remainingItemCount; + if (typeof remaining !== "number" || !Number.isFinite(remaining) || remaining < 0) { + throw new Error("pod list truncated but remainingItemCount absent or invalid"); + } + + return list.items.length + remaining; +} + +/** + * withTimeout rejects if `promise` outlives `timeoutMs`, so a hung request cannot + * freeze the caller. It cannot cancel: the k8s client threads no AbortSignal through to + * fetch, so an abandoned request keeps running. Callers must therefore also bound the + * request server-side (`timeoutSeconds`) and refuse to start a second one while the + * first is pending, or a blackholed connection accumulates one socket per attempt. + */ +export function withTimeout(promise: Promise, timeoutMs: number, what: string): Promise { + let timer: NodeJS.Timeout; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${what} timed out after ${timeoutMs}ms`)), + timeoutMs + ); + timer.unref(); + }); + + return Promise.race([promise, deadline]).finally(() => clearTimeout(timer)); +} diff --git a/apps/supervisor/src/env.test.ts b/apps/supervisor/src/env.test.ts new file mode 100644 index 0000000000..b02d117513 --- /dev/null +++ b/apps/supervisor/src/env.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect, vi } from "vitest"; + +// Mock std-env before importing env.ts so the module-level `Env.parse(stdEnv)` +// doesn't fail in a test environment that lacks required vars. +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + }, +})); + +const { Env } = await import("./env.js"); + +// Minimal env that satisfies all required fields; everything else has defaults. +const base = { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", +}; + +describe("Env superRefine - backpressure source awareness", () => { + it("pod-count source can be enabled without a Redis host", () => { + expect(() => + Env.parse({ + ...base, + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true", + }) + ).not.toThrow(); + }); + + it("redis source requires a Redis host", () => { + expect(() => + Env.parse({ + ...base, + TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED: "true", + }) + ).toThrow(); + }); + + it("both sources can be enabled together (with a Redis host)", () => { + expect(() => + Env.parse({ + ...base, + TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED: "true", + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST: "localhost", + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true", + }) + ).not.toThrow(); + }); + + it("rejects pod-count release >= engage when the source is enabled", () => { + expect(() => + Env.parse({ + ...base, + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true", + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE: "100", + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE: "100", + }) + ).toThrow(); + }); +}); diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 1605a21637..6830d5b864 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -1,119 +1,355 @@ import { randomUUID } from "crypto"; import { env as stdEnv } from "std-env"; import { z } from "zod"; -import { AdditionalEnvVars, BoolEnv } from "./envUtil.js"; - -const Env = z.object({ - // This will come from `spec.nodeName` in k8s - TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()), - TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30), - - // Required settings - TRIGGER_API_URL: z.string().url(), - TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file - MANAGED_WORKER_SECRET: z.string(), - OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners - - // Workload API settings (coordinator mode) - the workload API is what the run controller connects to - TRIGGER_WORKLOAD_API_ENABLED: BoolEnv.default(true), - TRIGGER_WORKLOAD_API_PROTOCOL: z - .string() - .transform((s) => z.enum(["http", "https"]).parse(s.toLowerCase())) - .default("http"), - TRIGGER_WORKLOAD_API_DOMAIN: z.string().optional(), // If unset, will use orchestrator-specific default - TRIGGER_WORKLOAD_API_HOST_INTERNAL: z.string().default("0.0.0.0"), - TRIGGER_WORKLOAD_API_PORT_INTERNAL: z.coerce.number().default(8020), // This is the port the workload API listens on - TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020), // This is the exposed port passed to the run controller - - // Runner settings - RUNNER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().optional(), - RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(), - RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv) - RUNNER_PRETTY_LOGS: BoolEnv.default(false), - - // Dequeue settings (provider mode) - TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true), - TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250), - TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000), - TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(1), - TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT: z.coerce.number().int().default(1), - TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(10), - TRIGGER_DEQUEUE_SCALING_STRATEGY: z.enum(["none", "smooth", "aggressive"]).default("none"), - TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS: z.coerce.number().int().default(5000), // 5 seconds - TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS: z.coerce.number().int().default(30000), // 30 seconds - TRIGGER_DEQUEUE_SCALING_TARGET_RATIO: z.coerce.number().default(1.0), // Target ratio of queue items to consumers (1.0 = 1 item per consumer) - TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA: z.coerce.number().min(0).max(1).default(0.3), // Smooths queue length measurements (0=historical, 1=current) - TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS: z.coerce.number().int().positive().default(1000), // Batch window for metrics processing (ms) - TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR: z.coerce.number().min(0).max(1).default(0.7), // Smooths consumer count changes after EWMA (0=no scaling, 1=immediate) - - // Optional services - TRIGGER_WARM_START_URL: z.string().optional(), - TRIGGER_CHECKPOINT_URL: z.string().optional(), - TRIGGER_METADATA_URL: z.string().optional(), - - // Used by the resource monitor - RESOURCE_MONITOR_ENABLED: BoolEnv.default(false), - RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL: z.coerce.number().optional(), - RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(), - - // Docker settings - DOCKER_API_VERSION: z.string().optional(), - DOCKER_PLATFORM: z.string().optional(), // e.g. linux/amd64, linux/arm64 - DOCKER_STRIP_IMAGE_DIGEST: BoolEnv.default(true), - DOCKER_REGISTRY_USERNAME: z.string().optional(), - DOCKER_REGISTRY_PASSWORD: z.string().optional(), - DOCKER_REGISTRY_URL: z.string().optional(), // e.g. https://index.docker.io/v1 - DOCKER_ENFORCE_MACHINE_PRESETS: BoolEnv.default(true), - DOCKER_AUTOREMOVE_EXITED_CONTAINERS: BoolEnv.default(true), - /** - * Network mode to use for all runners. Supported standard values are: `bridge`, `host`, `none`, and `container:`. - * Any other value is taken as a custom network's name to which all runners should connect to. - * - * Accepts a list of comma-separated values to attach to multiple networks. Additional networks are interpreted as network names and will be attached after container creation. - * - * **WARNING**: Specifying multiple networks will slightly increase startup times. - * - * @default "host" - */ - DOCKER_RUNNER_NETWORKS: z.string().default("host"), - - // Kubernetes settings - KUBERNETES_FORCE_ENABLED: BoolEnv.default(false), - KUBERNETES_NAMESPACE: z.string().default("default"), - KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"), - KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv - KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), - KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), - KUBERNETES_STRIP_IMAGE_DIGEST: BoolEnv.default(false), - KUBERNETES_CPU_REQUEST_MIN_CORES: z.coerce.number().min(0).default(0), - KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit - KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0), - KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit - KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB - KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods - - // Placement tags settings - PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), - PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"), - - // Metrics - METRICS_ENABLED: BoolEnv.default(true), - METRICS_COLLECT_DEFAULTS: BoolEnv.default(true), - METRICS_HOST: z.string().default("127.0.0.1"), - METRICS_PORT: z.coerce.number().int().default(9090), - - // Pod cleaner - POD_CLEANER_ENABLED: BoolEnv.default(true), - POD_CLEANER_INTERVAL_MS: z.coerce.number().int().default(10000), - POD_CLEANER_BATCH_SIZE: z.coerce.number().int().default(500), - - // Failed pod handler - FAILED_POD_HANDLER_ENABLED: BoolEnv.default(true), - FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS: z.coerce.number().int().default(1000), - - // Debug - DEBUG: BoolEnv.default(false), - SEND_RUN_DEBUG_LOGS: BoolEnv.default(false), -}); +import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js"; + +export const Env = z + .object({ + // This will come from `spec.nodeName` in k8s + TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()), + TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30), + + // Opt-in, dev-only: stream this process's logs over a local telnet/TCP socket on this port. + SUPERVISOR_TELNET_LOGS_PORT: z.coerce.number().optional(), + + // Required settings + TRIGGER_API_URL: z.string().url(), + TRIGGER_WORKER_TOKEN: z.string().min(1), // accepts file:// path to read from a file + MANAGED_WORKER_SECRET: z.string(), + + // Deployment token: sign a token into TRIGGER_DEPLOYMENT_ID at pod creation and verify it on + // inbound workload calls. "disabled" = off; "log" = mint + verify + metrics only; "enforce" = + // also reject invalid tokens. + WORKLOAD_TOKEN_SECRET: z.string().optional(), + WORKLOAD_TOKEN_ENFORCEMENT: z.enum(["disabled", "log", "enforce"]).default("disabled"), + DELETE_CHECKPOINTS_ON_COMPLETION: BoolEnv.default(false), // irreversible; enable per cluster + // Absolute expiry for minted deployment tokens. Deterministic (no wall-clock issued-at) so every + // pod of a deployment carries an identical token; bump before this date. Must outlive any run. + WORKLOAD_TOKEN_EXP: z.string().datetime().default("2032-01-01T00:00:00.000Z"), + OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners + + // Workload API settings (coordinator mode) - the workload API is what the run controller connects to + TRIGGER_WORKLOAD_API_ENABLED: BoolEnv.default(true), + TRIGGER_WORKLOAD_API_PROTOCOL: z + .string() + .transform((s) => z.enum(["http", "https"]).parse(s.toLowerCase())) + .default("http"), + TRIGGER_WORKLOAD_API_DOMAIN: z.string().optional(), // If unset, will use orchestrator-specific default + TRIGGER_WORKLOAD_API_HOST_INTERNAL: z.string().default("0.0.0.0"), + TRIGGER_WORKLOAD_API_PORT_INTERNAL: z.coerce.number().default(8020), // This is the port the workload API listens on + TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020), // This is the exposed port passed to the run controller + + // Runner settings + RUNNER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().optional(), + RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(), + RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv) + RUNNER_PRETTY_LOGS: BoolEnv.default(false), + + // Dequeue settings (provider mode) + TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true), + // Which worker-queue class this supervisor fleet serves. "default" pulls the + // region queue (standard/agent runs); "scheduled" pulls the dedicated + // scheduled-lineage queue. Run a separate fleet per class for isolation. + TRIGGER_WORKER_QUEUE_CLASS: z.enum(["default", "scheduled"]).default("default"), + TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250), + TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000), + TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(1), + TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT: z.coerce.number().int().default(1), + TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(10), + TRIGGER_DEQUEUE_SCALING_STRATEGY: z.enum(["none", "smooth", "aggressive"]).default("none"), + TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS: z.coerce.number().int().default(5000), // 5 seconds + TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS: z.coerce.number().int().default(30000), // 30 seconds + TRIGGER_DEQUEUE_SCALING_TARGET_RATIO: z.coerce.number().default(1.0), // Target ratio of queue items to consumers (1.0 = 1 item per consumer) + TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA: z.coerce.number().min(0).max(1).default(0.3), // Smooths queue length measurements (0=historical, 1=current) + TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS: z.coerce.number().int().positive().default(1000), // Batch window for metrics processing (ms) + TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR: z.coerce.number().min(0).max(1).default(0.7), // Smooths consumer count changes after EWMA (0=no scaling, 1=immediate) + + // Dequeue backpressure - off by default. When enabled, the supervisor reads a + // verdict from Redis (written by the cluster-side aggregator) and pauses dequeues + // while the worker cluster can't schedule pods. Disabled = total no-op: no Redis + // client is created, no reads happen, and the dequeue loop is unaffected. + TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED: BoolEnv.default(false), + // Safety default: even when enabled, backpressure only logs what it would do. + // Set to false to actually skip dequeues / freeze scale-up. + TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN: BoolEnv.default(true), + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY: z.string().default("engine:dequeue:backpressure"), + TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS: z.coerce.number().int().positive().default(1000), + TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS: z.coerce.number().int().min(0).default(30_000), // Resume ramp window after release; 0 = instant resume + + TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS: z.coerce + .number() + .int() + .positive() + .default(120_000), // Grace window: held verdict older than this โ†’ fail-open + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST: z.string().optional(), + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PORT: z.coerce.number().int().optional(), + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_USERNAME: z.string().optional(), + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD: z.string().optional(), + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_TLS_DISABLED: BoolEnv.default(false), + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: BoolEnv.default(false), + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN: BoolEnv.default(true), + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE: z.coerce + .number() + .int() + .positive() + .default(10_000), + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE: z.coerce + .number() + .int() + .positive() + .default(5_000), + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS: z.coerce + .number() + .int() + .positive() + .default(5_000), + // Hard timeout on the apiserver /metrics scrape. A hung request would otherwise + // never settle and freeze the monitor's refresh loop (fail-open silently). + TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS: z.coerce + .number() + .int() + .positive() + .default(10_000), + + // Optional services + TRIGGER_WARM_START_URL: z.string().optional(), + TRIGGER_WARM_START_DISPATCH_URL: z.string().optional(), + TRIGGER_CHECKPOINT_URL: z.string().optional(), + TRIGGER_METADATA_URL: z.string().optional(), + + // Warm-start delivery verification: after a warm-start hit, probe the + // platform and cold-start the run if no runner acted on the dispatch + TRIGGER_WARM_START_VERIFY_ENABLED: BoolEnv.default(false), + TRIGGER_WARM_START_VERIFY_DELAY_MS: z.coerce + .number() + .int() + .min(1_000) + .max(60_000) + .default(10_000), + + // Used by the resource monitor + RESOURCE_MONITOR_ENABLED: BoolEnv.default(false), + RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL: z.coerce.number().optional(), + RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(), + + // Docker settings + DOCKER_API_VERSION: z.string().optional(), + DOCKER_PLATFORM: z.string().optional(), // e.g. linux/amd64, linux/arm64 + DOCKER_STRIP_IMAGE_DIGEST: BoolEnv.default(true), + DOCKER_REGISTRY_USERNAME: z.string().optional(), + DOCKER_REGISTRY_PASSWORD: z.string().optional(), + DOCKER_REGISTRY_URL: z.string().optional(), // e.g. https://index.docker.io/v1 + DOCKER_ENFORCE_MACHINE_PRESETS: BoolEnv.default(true), + DOCKER_AUTOREMOVE_EXITED_CONTAINERS: BoolEnv.default(true), + /** + * Network mode to use for all runners. Supported standard values are: `bridge`, `host`, `none`, and `container:`. + * Any other value is taken as a custom network's name to which all runners should connect to. + * + * Accepts a list of comma-separated values to attach to multiple networks. Additional networks are interpreted as network names and will be attached after container creation. + * + * **WARNING**: Specifying multiple networks will slightly increase startup times. + * + * @default "host" + */ + DOCKER_RUNNER_NETWORKS: z.string().default("host"), + + // Compute settings + COMPUTE_GATEWAY_URL: z.string().url().optional(), + COMPUTE_GATEWAY_AUTH_TOKEN: z.string().optional(), + COMPUTE_GATEWAY_TIMEOUT_MS: z.coerce.number().int().default(30_000), + COMPUTE_SNAPSHOTS_ENABLED: BoolEnv.default(false), + COMPUTE_TRACE_SPANS_ENABLED: BoolEnv.default(true), + COMPUTE_TRACE_OTLP_ENDPOINT: z.string().url().optional(), // Override for span export (derived from TRIGGER_API_URL if unset) + COMPUTE_SNAPSHOT_DELAY_MS: z.coerce.number().int().min(0).max(60_000).default(5_000), + COMPUTE_SNAPSHOT_DISPATCH_LIMIT: z.coerce.number().int().min(1).max(100).default(10), + // Instance create retries for transient placement failures (1 = no retries) + COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS: z.coerce.number().int().min(1).max(10).default(3), + COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS: z.coerce + .number() + .int() + .min(0) + .max(10_000) + .default(250), + + // Kubernetes settings + KUBERNETES_FORCE_ENABLED: BoolEnv.default(false), + KUBERNETES_NAMESPACE: z.string().default("default"), + KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"), + KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv + KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), + KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), + KUBERNETES_STRIP_IMAGE_DIGEST: BoolEnv.default(false), + KUBERNETES_CPU_REQUEST_MIN_CORES: z.coerce.number().min(0).default(0), + KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit + KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0), + KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit + + // Per-preset overrides of the global KUBERNETES_CPU_REQUEST_RATIO + KUBERNETES_CPU_REQUEST_RATIO_MICRO: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_CPU_REQUEST_RATIO_SMALL_1X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_CPU_REQUEST_RATIO_SMALL_2X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_1X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_2X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_CPU_REQUEST_RATIO_LARGE_1X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_CPU_REQUEST_RATIO_LARGE_2X: z.coerce.number().min(0).max(1).optional(), + + // Per-preset overrides of the global KUBERNETES_MEMORY_REQUEST_RATIO + KUBERNETES_MEMORY_REQUEST_RATIO_MICRO: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_1X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_2X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_1X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_2X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_1X: z.coerce.number().min(0).max(1).optional(), + KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_2X: z.coerce.number().min(0).max(1).optional(), + + KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB + KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods + + // Pod DNS config โ€” override the cluster default ndots to `KUBERNETES_POD_DNS_NDOTS`. + // Default k8s ndots is 5: any name with fewer than 5 dots (e.g. `api.example.com`, 2 dots) is first walked + // through every entry in the cluster search list (`.svc.cluster.local`, `svc.cluster.local`, `cluster.local`) + // before being tried as-is, turning one resolution into 4+ CoreDNS queries (ร—2 with A+AAAA). + // Overriding the default can be useful to cut CoreDNS query amplification for external domains. + // Note: before enabling, make sure no code path relies on search-list expansion for names with dots โ‰ฅ the value + // set here โ€” those names will now hit their as-is form first and could resolve externally before falling back. + KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED: BoolEnv.default(false), + KUBERNETES_POD_DNS_NDOTS: z.coerce.number().int().min(1).max(15).default(2), + // Large machine affinity settings - large-* presets prefer a dedicated pool + KUBERNETES_LARGE_MACHINE_AFFINITY_ENABLED: BoolEnv.default(false), + KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_KEY: z + .string() + .trim() + .min(1) + .default("node.cluster.x-k8s.io/machinepool"), + KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_VALUE: z + .string() + .trim() + .min(1) + .default("large-machines"), + KUBERNETES_LARGE_MACHINE_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(100), + + // Project affinity settings - pods from the same project prefer the same node + KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false), + KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50), + KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z + .string() + .trim() + .min(1) + .default("kubernetes.io/hostname"), + + // Schedule affinity settings - runs from schedule trees prefer a dedicated pool + KUBERNETES_SCHEDULED_RUN_AFFINITY_ENABLED: BoolEnv.default(false), + KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_KEY: z + .string() + .trim() + .min(1) + .default("node.cluster.x-k8s.io/machinepool"), + KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_VALUE: z + .string() + .trim() + .min(1) + .default("scheduled-runs"), + KUBERNETES_SCHEDULED_RUN_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(80), + KUBERNETES_SCHEDULED_RUN_ANTI_AFFINITY_WEIGHT: z.coerce + .number() + .int() + .min(1) + .max(100) + .default(20), + + KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod + KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only + + // Placement tags settings + PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), + PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"), + + // Metrics + METRICS_ENABLED: BoolEnv.default(true), + METRICS_COLLECT_DEFAULTS: BoolEnv.default(true), + METRICS_HOST: z.string().default("127.0.0.1"), + METRICS_PORT: z.coerce.number().int().default(9090), + + // Pod cleaner + POD_CLEANER_ENABLED: BoolEnv.default(true), + POD_CLEANER_INTERVAL_MS: z.coerce.number().int().default(10000), + POD_CLEANER_BATCH_SIZE: z.coerce.number().int().default(500), + + // Failed pod handler + FAILED_POD_HANDLER_ENABLED: BoolEnv.default(true), + FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS: z.coerce.number().int().default(1000), + + // Debug + DEBUG: BoolEnv.default(false), + SEND_RUN_DEBUG_LOGS: BoolEnv.default(false), + + // Wide-event observability - off by default. Emits one flat-keyed JSON + // line per natural unit of work (dequeue iteration, HTTP request, socket + // lifecycle). High-QPS hotpath, so the kill switch must be honoured. + TRIGGER_WIDE_EVENTS_ENABLED: BoolEnv.default(false), + // When true, also emit wide events for high-frequency HTTP routes + // (heartbeat, snapshots-since, logs/debug). Off in prod to keep event + // volume manageable; on in test environments for full-fidelity debugging. + TRIGGER_WIDE_EVENTS_NOISY_ROUTES: BoolEnv.default(false), + }) + .superRefine((data, ctx) => { + if ( + data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED && + data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE >= + data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE must be less than TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE", + path: ["TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE"], + }); + } + if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_METADATA_URL) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "TRIGGER_METADATA_URL is required when COMPUTE_SNAPSHOTS_ENABLED is true", + path: ["TRIGGER_METADATA_URL"], + }); + } + if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_WORKLOAD_API_DOMAIN) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "TRIGGER_WORKLOAD_API_DOMAIN is required when COMPUTE_SNAPSHOTS_ENABLED is true", + path: ["TRIGGER_WORKLOAD_API_DOMAIN"], + }); + } + if (data.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled" && !data.WORKLOAD_TOKEN_SECRET) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "WORKLOAD_TOKEN_SECRET is required when WORKLOAD_TOKEN_ENFORCEMENT is not disabled", + path: ["WORKLOAD_TOKEN_SECRET"], + }); + } + if (data.DELETE_CHECKPOINTS_ON_COMPLETION && data.WORKLOAD_TOKEN_ENFORCEMENT === "disabled") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "DELETE_CHECKPOINTS_ON_COMPLETION needs WORKLOAD_TOKEN_ENFORCEMENT set to log or enforce: the tenancy it deletes by comes from the deployment token, so with tokens disabled it would silently reclaim nothing", + path: ["DELETE_CHECKPOINTS_ON_COMPLETION"], + }); + } + if ( + data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED && + !data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST is required when TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED is true", + path: ["TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST"], + }); + } + }) + .transform((data) => ({ + ...data, + COMPUTE_TRACE_OTLP_ENDPOINT: data.COMPUTE_TRACE_OTLP_ENDPOINT ?? `${data.TRIGGER_API_URL}/otel`, + })); export const env = Env.parse(stdEnv); diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts index c3d35758f1..378830f8ab 100644 --- a/apps/supervisor/src/envUtil.test.ts +++ b/apps/supervisor/src/envUtil.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { BoolEnv, AdditionalEnvVars } from "./envUtil.js"; +import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js"; describe("BoolEnv", () => { it("should parse string 'true' as true", () => { @@ -78,3 +78,128 @@ describe("AdditionalEnvVars", () => { }); }); }); + +describe("NodeLabelValue", () => { + it("should keep a clean value untouched", () => { + expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker"); + }); + + it("should trim surrounding whitespace, which Kubernetes would reject", () => { + expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker"); + expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker"); + }); + + it("should treat a whitespace-only value as the empty off-switch", () => { + expect(NodeLabelValue.parse("")).toBe(""); + expect(NodeLabelValue.parse(" ")).toBe(""); + }); + + it("should still apply a default only when unset", () => { + const withDefault = NodeLabelValue.default("v4-worker"); + expect(withDefault.parse(undefined)).toBe("v4-worker"); + expect(withDefault.parse("")).toBe(""); + }); + + it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => { + for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) { + expect(NodeLabelValue.safeParse(invalid).success).toBe(false); + } + }); +}); + +describe("Tolerations", () => { + it("should parse key=value entries as Equal", () => { + expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }, + ]); + }); + + it("should parse entries without a value as Exists", () => { + expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([ + { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" }, + ]); + }); + + it("should keep an empty value as an exact match for a valueless taint", () => { + expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" }, + ]); + + expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Exists", effect: "NoSchedule" }, + ]); + }); + + it("should parse an empty string as no tolerations", () => { + expect(Tolerations.parse("")).toEqual([]); + expect(Tolerations.parse(" ")).toEqual([]); + }); + + it("should skip blank entries and trim whitespace", () => { + expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([ + { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" }, + ]); + }); + + it("should reject a missing effect, an unknown effect, and an empty key", () => { + for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) { + expect(Tolerations.safeParse(invalid).success).toBe(false); + } + }); + + it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => { + expect( + Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule") + ).toEqual([ + { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" }, + { key: "spot", operator: "Exists", effect: "NoExecute" }, + { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" }, + ]); + }); + + it("should accept a DNS-subdomain prefixed key", () => { + expect( + Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule") + ).toEqual([ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "Equal", + value: "scheduled-runs", + effect: "NoSchedule", + }, + ]); + }); + + it("should reject a key or value that Kubernetes would reject at pod create", () => { + for (const invalid of [ + "dedicated=prod runs:NoSchedule", + "ded icated=runs:NoSchedule", + "dedicated=-runs:NoSchedule", + `dedicated=${"r".repeat(64)}:NoSchedule`, + `${"a".repeat(64)}=runs:NoSchedule`, + `example.com/${"a".repeat(64)}=runs:NoSchedule`, + "a/b/c=runs:NoSchedule", + "Example.com/pool=runs:NoSchedule", + ]) { + expect(Tolerations.safeParse(invalid).success).toBe(false); + } + }); + + it("should bound the prefix and the name separately, as Kubernetes does", () => { + const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`; + expect(longestPrefix.length).toBe(253); + + expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1); + expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false); + }); + + it("should tolerate whitespace around the separators", () => { + expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }, + ]); + }); + + it("should reject a stray extra effect instead of folding it into the value", () => { + expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false); + }); +}); diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts index 917f984cc3..67811f76fc 100644 --- a/apps/supervisor/src/envUtil.ts +++ b/apps/supervisor/src/envUtil.ts @@ -16,6 +16,136 @@ export const BoolEnv = baseBoolEnv as Omit & { default: (value: boolean) => z.ZodDefault; }; +const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/; +const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/; +const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/; +const QUALIFIED_NAME_MAX = 63; +const DNS_SUBDOMAIN_MAX = 253; +const LABEL_VALUE_MAX = 63; + +/** + * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream. + */ +function isLabelValue(value: string): boolean { + return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value); +} + +/** + * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and + * label keys: an optional DNS subdomain prefix before the slash, then the name. + * The two halves have different length limits and different case rules, so a + * single pattern with one overall bound gets both ends wrong. + */ +function isQualifiedName(key: string): boolean { + const slashIdx = key.indexOf("/"); + + if (slashIdx === -1) { + return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key); + } + + const prefix = key.slice(0, slashIdx); + const name = key.slice(slashIdx + 1); + + return ( + prefix.length <= DNS_SUBDOMAIN_MAX && + DNS_SUBDOMAIN.test(prefix) && + name.length <= QUALIFIED_NAME_MAX && + QUALIFIED_NAME.test(name) + ); +} + +/** + * A node label value. Trimmed because Kubernetes rejects surrounding whitespace + * outright, so a padded value fails every pod create. Deliberately no `min(1)`: + * empty is the off-switch, and the Helm chart ships empty by default. + */ +export const NodeLabelValue = z.string().trim().refine(isLabelValue, { + message: + "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters", +}); + +/** + * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect` + * for the Exists operator. Keys and values are checked against the Kubernetes + * naming rules here so a typo fails at startup, rather than 422ing every single + * pod create with the cause buried in an API server message. + */ +export const Tolerations = z.string().transform((val, ctx) => { + return val + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => { + const colonIdx = entry.lastIndexOf(":"); + if (colonIdx === -1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration format (missing effect): "${entry}"`, + }); + return z.NEVER; + } + + const effect = entry.slice(colonIdx + 1).trim(); + const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"]; + if (!validEffects.includes(effect)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join( + ", " + )}`, + }); + return z.NEVER; + } + + const keyValue = entry.slice(0, colonIdx); + const eqIdx = keyValue.indexOf("="); + const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim(); + + if (!key) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration format (empty key): "${entry}"`, + }); + return z.NEVER; + } + + if (!isQualifiedName(key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`, + }); + return z.NEVER; + } + + if (eqIdx === -1) { + return { key, operator: "Exists" as const, effect }; + } + + const value = keyValue.slice(eqIdx + 1).trim(); + if (!value) { + logger.warn( + 'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.', + { entry, key } + ); + } + + if (!isLabelValue(value)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`, + }); + return z.NEVER; + } + + return { + key, + operator: "Equal" as const, + value, + effect, + }; + }); +}); + export const AdditionalEnvVars = z.preprocess((val) => { if (typeof val !== "string") { return val; diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 0e274b3039..542841bd6e 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -1,5 +1,6 @@ import { SupervisorSession } from "@trigger.dev/core/v3/workers"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; +import { formatLogLine, startTelnetLogServer } from "@trigger.dev/core/v3/telnetLogServer"; import { env } from "./env.js"; import { WorkloadServer } from "./workloadServer/index.js"; import type { WorkloadManagerOptions, WorkloadManager } from "./workloadManager/types.js"; @@ -14,39 +15,108 @@ import { } from "./resourceMonitor.js"; import { KubernetesWorkloadManager } from "./workloadManager/kubernetes.js"; import { DockerWorkloadManager } from "./workloadManager/docker.js"; +import { ComputeWorkloadManager } from "./workloadManager/compute.js"; import { HttpServer, CheckpointClient, isKubernetesEnvironment, } from "@trigger.dev/core/v3/serverOnly"; -import { createK8sApi } from "./clients/kubernetes.js"; -import { collectDefaultMetrics } from "prom-client"; +import { createK8sApi, createPodCountFetcher } from "./clients/kubernetes.js"; +import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client"; import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; import { FailedPodHandler } from "./services/failedPodHandler.js"; import { getWorkerToken } from "./workerToken.js"; +import { mintDeploymentToken } from "./workloadToken.js"; +import { OtlpTraceService } from "./services/otlpTraceService.js"; +import { + WarmStartVerificationService, + type WarmStartTimings, +} from "./services/warmStartVerificationService.js"; +import { extractTraceparent, getRestoreRunnerId } from "./util.js"; +import { Redis } from "ioredis"; +import { BackpressureMonitor } from "./backpressure/backpressureMonitor.js"; +import { RedisBackpressureSignalSource } from "./backpressure/redisBackpressureSignalSource.js"; +import { BackpressureMetrics } from "./backpressure/backpressureMetrics.js"; +import { K8sPodCountSignalSource } from "./backpressure/k8sPodCountSignalSource.js"; +import { + fromContext, + recordPhaseSince, + runWideEvent, + setExtra, + setMeta, + type WideEventOptions, +} from "./wideEvents/index.js"; if (env.METRICS_COLLECT_DEFAULTS) { collectDefaultMetrics({ register }); } +const workloadCreateDuration = new Histogram({ + name: "workload_create_duration_seconds", + help: "Duration of workload manager create calls. A create may include backend-internal retries, so one observation can span multiple attempts.", + labelNames: ["backend", "outcome"], + buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], + registers: [register], +}); + +const outboundRequestsTotal = new Counter({ + name: "supervisor_outbound_request_total", + help: "Count of outbound HTTP requests from the supervisor, by target name, method, response status, and outcome (ok, http_error, invalid_response, network_error).", + labelNames: ["name", "method", "status", "outcome"], + registers: [register], +}); + +const outboundRequestDuration = new Histogram({ + name: "supervisor_outbound_request_duration_seconds", + help: "Duration of outbound HTTP requests from the supervisor, by target name and outcome. Includes the HTTP client's internal retries and backoff.", + labelNames: ["name", "outcome"], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 11, 12.5, 15, 20, 30, 60], + registers: [register], +}); + class ManagedSupervisor { private readonly workerSession: SupervisorSession; private readonly metricsServer?: HttpServer; private readonly workloadServer: WorkloadServer; private readonly workloadManager: WorkloadManager; + private readonly workloadManagerBackend: "compute" | "kubernetes" | "docker"; + private readonly computeManager?: ComputeWorkloadManager; private readonly logger = new SimpleStructuredLogger("managed-supervisor"); private readonly resourceMonitor: ResourceMonitor; private readonly checkpointClient?: CheckpointClient; + private readonly warmStartVerifier?: WarmStartVerificationService; private readonly podCleaner?: PodCleaner; private readonly failedPodHandler?: FailedPodHandler; + private readonly tracing?: OtlpTraceService; + private readonly backpressureMonitors: BackpressureMonitor[] = []; + private readonly backpressureRedis?: Redis; private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED); private readonly warmStartUrl = env.TRIGGER_WARM_START_URL; + private readonly warmStartDispatchUrl = + env.TRIGGER_WARM_START_DISPATCH_URL ?? env.TRIGGER_WARM_START_URL; + + private readonly wideEventOpts: WideEventOptions = { + service: "supervisor", + env: { nodeId: env.TRIGGER_WORKER_INSTANCE_NAME }, + enabled: env.TRIGGER_WIDE_EVENTS_ENABLED, + }; + private readonly wideEventsNoisyRoutes = env.TRIGGER_WIDE_EVENTS_NOISY_ROUTES; constructor() { - const { TRIGGER_WORKER_TOKEN, MANAGED_WORKER_SECRET, ...envWithoutSecrets } = env; + // Strip secret-like env vars before debug-logging the rest. Add any new + // secret env var here so it never lands in the DEBUG "Starting up" log. + const { + TRIGGER_WORKER_TOKEN, + MANAGED_WORKER_SECRET, + COMPUTE_GATEWAY_AUTH_TOKEN, + DOCKER_REGISTRY_PASSWORD, + TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD, + WORKLOAD_TOKEN_SECRET, + ...envWithoutSecrets + } = env; if (env.DEBUG) { this.logger.debug("Starting up", { envWithoutSecrets }); @@ -69,6 +139,7 @@ class ManagedSupervisor { snapshotPollIntervalSeconds: env.RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS, additionalEnvVars: env.RUNNER_ADDITIONAL_ENV_VARS, dockerAutoremove: env.DOCKER_AUTOREMOVE_EXITED_CONTAINERS, + checkpointsEnabled: !!env.TRIGGER_CHECKPOINT_URL, } satisfies WorkloadManagerOptions; this.resourceMonitor = env.RESOURCE_MONITOR_ENABLED @@ -77,9 +148,54 @@ class ManagedSupervisor { : new DockerResourceMonitor(new Docker()) : new NoopResourceMonitor(); - this.workloadManager = this.isKubernetes - ? new KubernetesWorkloadManager(workloadManagerOptions) - : new DockerWorkloadManager(workloadManagerOptions); + if (env.COMPUTE_GATEWAY_URL) { + if (!env.TRIGGER_WORKLOAD_API_DOMAIN) { + throw new Error("TRIGGER_WORKLOAD_API_DOMAIN is not set, cannot create compute manager"); + } + + const callbackUrl = `${env.TRIGGER_WORKLOAD_API_PROTOCOL}://${env.TRIGGER_WORKLOAD_API_DOMAIN}:${env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL}/api/v1/compute/snapshot-complete`; + + if (env.COMPUTE_TRACE_SPANS_ENABLED) { + this.tracing = new OtlpTraceService({ + endpointUrl: env.COMPUTE_TRACE_OTLP_ENDPOINT, + }); + } + + const computeManager = new ComputeWorkloadManager({ + ...workloadManagerOptions, + gateway: { + url: env.COMPUTE_GATEWAY_URL, + authToken: env.COMPUTE_GATEWAY_AUTH_TOKEN, + timeoutMs: env.COMPUTE_GATEWAY_TIMEOUT_MS, + }, + snapshots: { + enabled: env.COMPUTE_SNAPSHOTS_ENABLED, + delayMs: env.COMPUTE_SNAPSHOT_DELAY_MS, + dispatchLimit: env.COMPUTE_SNAPSHOT_DISPATCH_LIMIT, + callbackUrl, + }, + tracing: this.tracing, + runner: { + instanceName: env.TRIGGER_WORKER_INSTANCE_NAME, + otelEndpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT, + prettyLogs: env.RUNNER_PRETTY_LOGS, + sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, + }, + createRetry: { + maxAttempts: env.COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS, + baseDelayMs: env.COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS, + }, + }); + this.computeManager = computeManager; + this.workloadManager = computeManager; + this.workloadManagerBackend = "compute"; + } else if (this.isKubernetes) { + this.workloadManager = new KubernetesWorkloadManager(workloadManagerOptions); + this.workloadManagerBackend = "kubernetes"; + } else { + this.workloadManager = new DockerWorkloadManager(workloadManagerOptions); + this.workloadManagerBackend = "docker"; + } if (this.isKubernetes) { if (env.POD_CLEANER_ENABLED) { @@ -119,8 +235,86 @@ class ManagedSupervisor { ); } + // Redis-verdict source (external aggregator). Keeps existing metric names. + if (env.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED) { + this.backpressureRedis = new Redis({ + host: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST, + port: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PORT, + username: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_USERNAME, + password: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD, + ...(env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_TLS_DISABLED ? {} : { tls: {} }), + maxRetriesPerRequest: null, + }); + this.backpressureRedis.on("error", (error) => + this.logger.error("Backpressure redis error", { error: error.message }) + ); + this.backpressureMonitors.push( + new BackpressureMonitor({ + enabled: true, + source: new RedisBackpressureSignalSource( + this.backpressureRedis, + env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY + ), + refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS, + maxVerdictAgeMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS, + rampMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS, + dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN, + logger: this.logger, + metrics: new BackpressureMetrics({ register }), + }) + ); + this.logger.log("๐Ÿ›‘ Dequeue backpressure enabled (redis source)", { + key: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY, + refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS, + dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN, + }); + } + + // Pod-count source (in-process apiserver scrape). Namespaced metrics so the + // redis source's metric names are preserved. + if (env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED) { + // RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here. + const podCountGauge = new Gauge({ + name: "supervisor_cluster_pod_count", + help: "Pod objects in the workload namespace, counted for backpressure", + registers: [register], + }); + this.backpressureMonitors.push( + new BackpressureMonitor({ + enabled: true, + source: new K8sPodCountSignalSource({ + fetchPodCount: createPodCountFetcher( + createK8sApi(), + env.KUBERNETES_NAMESPACE, + env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS + ), + engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE, + releaseThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE, + reportPodCount: (count) => podCountGauge.set(count), + }), + refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS, + maxVerdictAgeMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS, + rampMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS, + dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN, + logger: this.logger, + metrics: new BackpressureMetrics({ + register, + prefix: "supervisor_backpressure_pod_count", + }), + }) + ); + this.logger.log("๐Ÿ›‘ Dequeue backpressure enabled (pod-count source)", { + engage: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE, + release: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE, + refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS, + dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN, + }); + } + + const workerToken = getWorkerToken(); + this.workerSession = new SupervisorSession({ - workerToken: getWorkerToken(), + workerToken, apiUrl: env.TRIGGER_API_URL, instanceName: env.TRIGGER_WORKER_INSTANCE_NAME, managedWorkerSecret: env.MANAGED_WORKER_SECRET, @@ -128,6 +322,7 @@ class ManagedSupervisor { dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS, queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED, maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT, + queueClass: env.TRIGGER_WORKER_QUEUE_CLASS, metricsRegistry: register, scaling: { strategy: env.TRIGGER_DEQUEUE_SCALING_STRATEGY, @@ -139,18 +334,24 @@ class ManagedSupervisor { ewmaAlpha: env.TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA, batchWindowMs: env.TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS, dampingFactor: env.TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR, + // Freeze scale-up while backpressure is hard-engaged (not during the resume + // ramp). Undefined when backpressure is disabled โ†’ no effect on scaling. + shouldPauseScaling: () => this.backpressureMonitors.some((m) => m.isEngaged()), }, runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED, heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, + onHttpRequestComplete: ({ name, method, status, outcome, durationMs }) => { + outboundRequestsTotal.inc({ name, method, status, outcome }); + outboundRequestDuration.observe({ name, outcome }, durationMs / 1000); + }, preDequeue: async () => { - if (!env.RESOURCE_MONITOR_ENABLED) { - return {}; - } + // Synchronous, hot-path-safe cached read; false when no monitors are active. + const skipForBackpressure = this.backpressureMonitors.some((m) => m.shouldSkipDequeue()); - if (this.isKubernetes) { - // Not used in k8s for now - return {}; + if (!env.RESOURCE_MONITOR_ENABLED || this.isKubernetes) { + // Resource monitor is not used in k8s; backpressure is the only gate there. + return { skipDequeue: skipForBackpressure }; } const resources = await this.resourceMonitor.getNodeResources(); @@ -160,7 +361,10 @@ class ManagedSupervisor { cpu: resources.cpuAvailable, memory: resources.memoryAvailable, }, - skipDequeue: resources.cpuAvailable < 0.25 || resources.memoryAvailable < 0.25, + skipDequeue: + skipForBackpressure || + resources.cpuAvailable < 0.25 || + resources.memoryAvailable < 0.25, }; }, preSkip: async () => { @@ -181,103 +385,197 @@ class ManagedSupervisor { }); } + if (env.TRIGGER_WARM_START_VERIFY_ENABLED && this.warmStartUrl) { + this.logger.log("Warm-start delivery verification enabled", { + delayMs: env.TRIGGER_WARM_START_VERIFY_DELAY_MS, + }); + + this.warmStartVerifier = new WarmStartVerificationService({ + workerClient: this.workerSession.httpClient, + delayMs: env.TRIGGER_WARM_START_VERIFY_DELAY_MS, + createWorkload: (message, timings) => this.createWorkload(message, timings), + wideEventOpts: this.wideEventOpts, + }); + } + this.workerSession.on("runNotification", async ({ time, run }) => { - this.logger.log("runNotification", { time, run }); + this.logger.verbose("runNotification", { time, run }); this.workloadServer.notifyRun({ run }); }); - this.workerSession.on("runQueueMessage", async ({ time, message }) => { - this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message); - - if (message.completedWaitpoints.length > 0) { - this.logger.debug("Run has completed waitpoints", { - runId: message.run.id, - completedWaitpoints: message.completedWaitpoints.length, - }); - } - - if (!message.image) { - this.logger.error("Run has no image", { runId: message.run.id }); - return; - } - - const { checkpoint, ...rest } = message; - - if (checkpoint) { - this.logger.log("Restoring run", { runId: message.run.id }); - - if (!this.checkpointClient) { - this.logger.error("No checkpoint client", { runId: message.run.id }); - return; - } - - try { - const didRestore = await this.checkpointClient.restoreRun({ - runFriendlyId: message.run.friendlyId, - snapshotFriendlyId: message.snapshot.friendlyId, - body: { - ...rest, - checkpoint, + this.workerSession.on( + "runQueueMessage", + async ({ time, message, dequeueResponseMs, pollingIntervalMs }) => { + this.logger.verbose(`Received message with timestamp ${time.toLocaleString()}`, message); + + const traceparent = extractTraceparent(message.run.traceContext); + + await runWideEvent( + { + ...this.wideEventOpts, + op: "dequeue", + kind: "inbound", + traceparent, + setup: (state) => { + setMeta(state, "run_id", message.run.friendlyId); + setMeta(state, "env_id", message.environment.id); + setMeta(state, "org_id", message.organization.id); + setMeta(state, "project_id", message.project.id); + if (message.deployment.friendlyId) { + setMeta(state, "deployment_id", message.deployment.friendlyId); + } + setMeta(state, "machine_preset", message.run.machine.name); + state.extras.iteration = "dequeue"; + state.extras.dequeue_response_ms = dequeueResponseMs; + state.extras.polling_interval_ms = pollingIntervalMs; + state.extras.completed_waitpoints = message.completedWaitpoints.length; }, - }); - - if (didRestore) { - this.logger.log("Restore successful", { runId: message.run.id }); - } else { - this.logger.error("Restore failed", { runId: message.run.id }); + }, + async () => { + if (message.completedWaitpoints.length > 0) { + this.logger.debug("Run has completed waitpoints", { + runId: message.run.id, + completedWaitpoints: message.completedWaitpoints.length, + }); + } + + if (!message.image) { + setExtra(fromContext(), "path_taken", "skipped_no_image"); + this.logger.error("Run has no image", { runId: message.run.id }); + return; + } + + const { checkpoint, ...rest } = message; + + // Register trace context early so snapshot spans work for all paths + // (cold create, restore, warm start). Re-registration on restore is safe + // since dequeue always provides fresh context. + if (this.computeManager?.traceSpansEnabled && traceparent) { + this.workloadServer.registerRunTraceContext(message.run.friendlyId, { + traceparent, + envId: message.environment.id, + orgId: message.organization.id, + projectId: message.project.id, + }); + } + + if (checkpoint) { + setExtra(fromContext(), "path_taken", "restore"); + this.logger.debug("Restoring run", { runId: message.run.id }); + + if (this.computeManager) { + const restoreStart = performance.now(); + try { + const runnerId = getRestoreRunnerId(message.run.friendlyId, checkpoint.id); + + const didRestore = await this.computeManager.restore({ + snapshotId: checkpoint.location, + runnerId, + runFriendlyId: message.run.friendlyId, + snapshotFriendlyId: message.snapshot.friendlyId, + machine: message.run.machine, + traceContext: message.run.traceContext, + envId: message.environment.id, + orgId: message.organization.id, + projectId: message.project.id, + hasPrivateLink: message.organization.hasPrivateLink, + dequeuedAt: message.dequeuedAt, + }); + recordPhaseSince("restore", restoreStart, undefined); + setExtra(fromContext(), "did_restore", didRestore); + + if (didRestore) { + this.logger.debug("Compute restore successful", { + runId: message.run.id, + runnerId, + }); + } else { + this.logger.error("Compute restore failed", { + runId: message.run.id, + runnerId, + }); + } + } catch (error) { + recordPhaseSince( + "restore", + restoreStart, + error instanceof Error ? error : new Error(String(error)) + ); + this.logger.error("Failed to restore run (compute)", { error }); + } + + return; + } + + if (!this.checkpointClient) { + this.logger.error("No checkpoint client", { runId: message.run.id }); + return; + } + + const restoreStart = performance.now(); + try { + const didRestore = await this.checkpointClient.restoreRun({ + runFriendlyId: message.run.friendlyId, + snapshotFriendlyId: message.snapshot.friendlyId, + body: { + ...rest, + checkpoint, + }, + }); + recordPhaseSince("restore", restoreStart, undefined); + setExtra(fromContext(), "did_restore", didRestore); + + if (didRestore) { + this.logger.debug("Restore successful", { runId: message.run.id }); + } else { + this.logger.error("Restore failed", { runId: message.run.id }); + } + } catch (error) { + recordPhaseSince( + "restore", + restoreStart, + error instanceof Error ? error : new Error(String(error)) + ); + this.logger.error("Failed to restore run", { error }); + } + + return; + } + + this.logger.debug("Scheduling run", { runId: message.run.id }); + + const warmStartStart = performance.now(); + const didWarmStart = await this.tryWarmStart(message, traceparent); + const warmStartCheckMs = Math.round(performance.now() - warmStartStart); + recordPhaseSince("warm_start", warmStartStart, undefined); + setExtra(fromContext(), "did_warm_start", didWarmStart); + + if (didWarmStart) { + setExtra(fromContext(), "path_taken", "warm_start"); + this.logger.debug("Warm start successful", { runId: message.run.id }); + // A hit only means the response was written to the long-poll + // socket, not that the runner received it. Schedule a delivery + // verification that cold-starts the run if nobody acts on it. + this.warmStartVerifier?.schedule(message, { + dequeueResponseMs, + pollingIntervalMs, + warmStartCheckMs, + }); + return; + } + + setExtra(fromContext(), "path_taken", "cold_create"); + + await this.createWorkload(message, { + dequeueResponseMs, + pollingIntervalMs, + warmStartCheckMs, + }); } - } catch (error) { - this.logger.error("Failed to restore run", { error }); - } - - return; - } - - this.logger.log("Scheduling run", { runId: message.run.id }); - - const didWarmStart = await this.tryWarmStart(message); - - if (didWarmStart) { - this.logger.log("Warm start successful", { runId: message.run.id }); - return; + ); } - - try { - if (!message.deployment.friendlyId) { - // mostly a type guard, deployments always exists for deployed environments - // a proper fix would be to use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments. - throw new Error("Deployment is missing"); - } - - await this.workloadManager.create({ - dequeuedAt: message.dequeuedAt, - envId: message.environment.id, - envType: message.environment.type, - image: message.image, - machine: message.run.machine, - orgId: message.organization.id, - projectId: message.project.id, - deploymentFriendlyId: message.deployment.friendlyId, - deploymentVersion: message.backgroundWorker.version, - runId: message.run.id, - runFriendlyId: message.run.friendlyId, - version: message.version, - nextAttemptNumber: message.run.attemptNumber, - snapshotId: message.snapshot.id, - snapshotFriendlyId: message.snapshot.friendlyId, - placementTags: message.placementTags, - }); - - // Disabled for now - // this.resourceMonitor.blockResources({ - // cpu: message.run.machine.cpu, - // memory: message.run.machine.memory, - // }); - } catch (error) { - this.logger.error("Failed to create workload", { error }); - } - }); + ); if (env.METRICS_ENABLED) { this.metricsServer = new HttpServer({ @@ -296,6 +594,11 @@ class ManagedSupervisor { host: env.TRIGGER_WORKLOAD_API_HOST_INTERNAL, workerClient: this.workerSession.httpClient, checkpointClient: this.checkpointClient, + computeManager: this.computeManager, + tracing: this.tracing, + snapshotCallbackSecret: workerToken, + wideEventOpts: this.wideEventOpts, + wideEventsNoisyRoutes: this.wideEventsNoisyRoutes, }); this.workloadServer.on("runConnected", this.onRunConnected.bind(this)); @@ -304,6 +607,8 @@ class ManagedSupervisor { async onRunConnected({ run }: { run: { friendlyId: string } }) { this.logger.debug("Run connected", { run }); + // The dispatched run reached a runner on this node - no fallback needed. + this.warmStartVerifier?.cancel(run.friendlyId); this.workerSession.subscribeToRunNotifications([run.friendlyId]); } @@ -312,25 +617,128 @@ class ManagedSupervisor { this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]); } - private async tryWarmStart(dequeuedMessage: DequeuedMessage): Promise { + private async createWorkload(message: DequeuedMessage, timings: WarmStartTimings) { + const createStart = performance.now(); + try { + if (!message.deployment.friendlyId) { + // mostly a type guard, deployments always exists for deployed environments + // a proper fix would be to use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments. + throw new Error("Deployment is missing"); + } + + if (!message.image) { + // same type-guard situation as deployment above + throw new Error("Image is missing"); + } + + const deploymentToken = await mintDeploymentToken({ + deployment: message.deployment.friendlyId, + deployment_version: message.backgroundWorker.version, + environment_id: message.environment.id, + environment_type: message.environment.type, + org_id: message.organization.id, + project_id: message.project.id, + }); + + await this.workloadManager.create({ + dequeuedAt: message.dequeuedAt, + dequeueResponseMs: timings.dequeueResponseMs, + pollingIntervalMs: timings.pollingIntervalMs, + warmStartCheckMs: timings.warmStartCheckMs, + envId: message.environment.id, + envType: message.environment.type, + image: message.image, + machine: message.run.machine, + orgId: message.organization.id, + projectId: message.project.id, + deploymentFriendlyId: message.deployment.friendlyId, + deploymentVersion: message.backgroundWorker.version, + runtime: message.backgroundWorker.runtime, + deploymentToken, + runId: message.run.id, + runFriendlyId: message.run.friendlyId, + version: message.version, + nextAttemptNumber: message.run.attemptNumber, + snapshotId: message.snapshot.id, + snapshotFriendlyId: message.snapshot.friendlyId, + placementTags: message.placementTags, + traceContext: message.run.traceContext, + annotations: message.run.annotations, + hasPrivateLink: message.organization.hasPrivateLink, + }); + recordPhaseSince("workload_create", createStart, undefined); + workloadCreateDuration.observe( + { backend: this.workloadManagerBackend, outcome: "success" }, + (performance.now() - createStart) / 1000 + ); + + // Disabled for now + // this.resourceMonitor.blockResources({ + // cpu: message.run.machine.cpu, + // memory: message.run.machine.memory, + // }); + } catch (error) { + recordPhaseSince( + "workload_create", + createStart, + error instanceof Error ? error : new Error(String(error)) + ); + workloadCreateDuration.observe( + { backend: this.workloadManagerBackend, outcome: "error" }, + (performance.now() - createStart) / 1000 + ); + this.logger.error("Failed to create workload", { + runId: message.run.friendlyId, + error, + }); + } + } + + private async tryWarmStart( + dequeuedMessage: DequeuedMessage, + traceparent: string | undefined + ): Promise { if (!this.warmStartUrl) { return false; } - const warmStartUrlWithPath = new URL("/warm-start", this.warmStartUrl); + const warmStartUrlWithPath = new URL("/warm-start", this.warmStartDispatchUrl); + + const headers: Record = { + "Content-Type": "application/json", + }; + // Propagate the inbound W3C traceparent so the upstream warm-start + // receiver continues the same trace instead of minting a new one. Gated + // by the same kill switch as the wide-event emission so the whole PR is + // a no-op on the wire when disabled. + if (this.wideEventOpts.enabled && traceparent) { + headers.traceparent = traceparent; + } + + const requestStart = performance.now(); + const record = ( + status: string, + outcome: "ok" | "http_error" | "invalid_response" | "network_error" + ) => { + outboundRequestsTotal.inc({ name: "warm_start", method: "POST", status, outcome }); + outboundRequestDuration.observe( + { name: "warm_start", outcome }, + (performance.now() - requestStart) / 1000 + ); + }; try { const res = await fetch(warmStartUrlWithPath.href, { method: "POST", - headers: { - "Content-Type": "application/json", - }, + headers, body: JSON.stringify({ dequeuedMessage }), }); if (!res.ok) { + record(String(res.status), "http_error"); this.logger.error("Warm start failed", { runId: dequeuedMessage.run.id, + statusCode: res.status, }); return false; } @@ -339,6 +747,7 @@ class ManagedSupervisor { const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data); if (!parsedData.success) { + record(String(res.status), "invalid_response"); this.logger.error("Warm start response invalid", { runId: dequeuedMessage.run.id, data, @@ -346,8 +755,11 @@ class ManagedSupervisor { return false; } + record(String(res.status), "ok"); + return parsedData.data.didWarmStart; } catch (error) { + record("none", "network_error"); this.logger.error("Warm start error", { runId: dequeuedMessage.run.id, error, @@ -360,6 +772,7 @@ class ManagedSupervisor { this.logger.log("Starting up"); // Optional services + this.backpressureMonitors.forEach((m) => m.start()); await this.podCleaner?.start(); await this.failedPodHandler?.start(); await this.metricsServer?.start(); @@ -380,14 +793,29 @@ class ManagedSupervisor { async stop() { this.logger.log("Shutting down"); + // Stop the verifier first: its timer can otherwise fire mid-shutdown and + // cold-create a workload on a node that is going down. + this.warmStartVerifier?.stop(); + await this.workloadServer.stop(); await this.workerSession.stop(); // Optional services + this.backpressureMonitors.forEach((m) => m.stop()); + await this.backpressureRedis?.quit(); await this.podCleaner?.stop(); await this.failedPodHandler?.stop(); await this.metricsServer?.stop(); } } +// Opt-in, dev-only: mirror this process's structured logs to a local telnet/TCP stream. +if (env.SUPERVISOR_TELNET_LOGS_PORT && env.SUPERVISOR_TELNET_LOGS_PORT > 0) { + const telnetLogServer = startTelnetLogServer({ + port: env.SUPERVISOR_TELNET_LOGS_PORT, + name: "supervisor", + }); + SimpleStructuredLogger.onLog = (log) => telnetLogServer.broadcast(formatLogLine(log)); +} + const worker = new ManagedSupervisor(); worker.start(); diff --git a/apps/supervisor/src/services/computeSnapshotService.test.ts b/apps/supervisor/src/services/computeSnapshotService.test.ts new file mode 100644 index 0000000000..0877f5e9d7 --- /dev/null +++ b/apps/supervisor/src/services/computeSnapshotService.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it, vi } from "vitest"; +import { setTimeout as sleep } from "node:timers/promises"; +import { ComputeSnapshotService } from "./computeSnapshotService.js"; +import type { ComputeWorkloadManager } from "../workloadManager/compute.js"; +import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers"; + +// The TimerWheel ticks every 100ms, so a 200ms delay dispatches within ~300ms. +const DELAY_MS = 200; +// Long enough that a pending snapshot would certainly have dispatched. +const SETTLE_MS = 600; + +function createService() { + const snapshot = vi.fn( + async (_opts: { runnerId: string; metadata: Record }) => true + ); + + const computeManager = { + snapshotDelayMs: DELAY_MS, + snapshotDispatchLimit: 1, + snapshot, + } as unknown as ComputeWorkloadManager; + + const submitSuspendCompletion = vi.fn(async () => ({ success: true })); + + const service = new ComputeSnapshotService({ + computeManager, + workerClient: { submitSuspendCompletion } as unknown as SupervisorHttpClient, + wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "test-secret", + }); + + return { service, snapshot, submitSuspendCompletion }; +} + +function dispatchedMetadata(snapshot: { + mock: { calls: Array }>> }; +}) { + const metadata = snapshot.mock.calls[0]?.[0]?.metadata; + if (!metadata) { + throw new Error("Snapshot was not dispatched"); + } + return metadata; +} + +function delayedSnapshot(runnerId = "runner-1") { + return { + runnerId, + runFriendlyId: "run_1", + snapshotFriendlyId: "snapshot_1", + }; +} + +describe("ComputeSnapshotService", () => { + it("refuses to construct with an empty callback secret", () => { + const computeManager = { + snapshotDelayMs: DELAY_MS, + snapshotDispatchLimit: 1, + snapshot: vi.fn(async () => true), + } as unknown as ComputeWorkloadManager; + + expect( + () => + new ComputeSnapshotService({ + computeManager, + workerClient: {} as SupervisorHttpClient, + wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "", + }) + ).toThrow(); + }); + + it("dispatches a scheduled snapshot after the delay", async () => { + const { service, snapshot } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + expect(snapshot).toHaveBeenCalledWith({ + runnerId: "runner-1", + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_1", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), + }); + } finally { + service.stop(); + } + }); + + it("cancel before the delay expires prevents the dispatch", async () => { + const { service, snapshot } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + expect(service.cancel("run_1")).toBe(true); + + await sleep(SETTLE_MS); + expect(snapshot).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("cancel returns false when nothing is pending", () => { + const { service } = createService(); + try { + expect(service.cancel("run_1")).toBe(false); + } finally { + service.stop(); + } + }); + + it("cancel with a matching runnerId cancels the pending snapshot", async () => { + const { service, snapshot } = createService(); + try { + service.schedule("run_1", delayedSnapshot("runner-a")); + + expect(service.cancel("run_1", "runner-a")).toBe(true); + + await sleep(SETTLE_MS); + expect(snapshot).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("cancel with a different runnerId leaves the pending snapshot alone", async () => { + const { service, snapshot } = createService(); + try { + service.schedule("run_1", delayedSnapshot("runner-a")); + + // A stale runner for a reassigned run must not cancel the new runner's snapshot. + expect(service.cancel("run_1", "runner-b")).toBe(false); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + expect(snapshot).toHaveBeenCalledWith(expect.objectContaining({ runnerId: "runner-a" })); + } finally { + service.stop(); + } + }); + + it("re-scheduling the same run replaces the pending snapshot", async () => { + const { service, snapshot } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + service.schedule("run_1", { + runnerId: "runner-1", + runFriendlyId: "run_1", + snapshotFriendlyId: "snapshot_2", + }); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + await sleep(SETTLE_MS); + + expect(snapshot).toHaveBeenCalledTimes(1); + expect(snapshot).toHaveBeenCalledWith({ + runnerId: "runner-1", + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_2", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), + }); + } finally { + service.stop(); + } + }); + + it("accepts a snapshot callback with the dispatched token", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata, + }); + + expect(result).toEqual({ ok: true, status: 200 }); + expect(submitSuspendCompletion).toHaveBeenCalledWith({ + runId: "run_1", + snapshotId: "snapshot_1", + body: { + success: true, + checkpoint: { + type: "COMPUTE", + location: "compute_snapshot_1", + }, + }, + }); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback without a valid token", async () => { + const { service, submitSuspendCompletion } = createService(); + try { + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback whose token is for a different snapshot", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { ...metadata, snapshotFriendlyId: "snapshot_2" }, + }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); +}); diff --git a/apps/supervisor/src/services/computeSnapshotService.ts b/apps/supervisor/src/services/computeSnapshotService.ts new file mode 100644 index 0000000000..6af307d0e3 --- /dev/null +++ b/apps/supervisor/src/services/computeSnapshotService.ts @@ -0,0 +1,411 @@ +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; +import pLimit from "p-limit"; +import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; +import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic"; +import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers"; +import { type SnapshotCallbackPayload } from "@internal/compute"; +import type { ComputeWorkloadManager } from "../workloadManager/compute.js"; +import { TimerWheel } from "./timerWheel.js"; +import type { OtlpTraceService } from "./otlpTraceService.js"; +import { + emitOneShot, + fromContext, + recordPhaseSince, + runWideEvent, + setExtra, + setMeta, + type WideEventOptions, +} from "../wideEvents/index.js"; + +const SNAPSHOT_CALLBACK_NONCE_METADATA_KEY = "snapshotCallbackNonce"; +const SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY = "snapshotCallbackToken"; + +// Domain-separation label so the callback-signing key is derived from, rather +// than equal to, the secret used for other protocols. Bump the suffix to rotate. +const SNAPSHOT_CALLBACK_KEY_INFO = "compute-snapshot-callback-v1"; + +type DelayedSnapshot = { + runnerId: string; + runFriendlyId: string; + snapshotFriendlyId: string; +}; + +export type RunTraceContext = { + traceparent: string; + envId: string; + orgId: string; + projectId: string; +}; + +export type ComputeSnapshotServiceOptions = { + computeManager: ComputeWorkloadManager; + workerClient: SupervisorHttpClient; + tracing?: OtlpTraceService; + wideEventOpts: WideEventOptions; + snapshotCallbackSecret: string; +}; + +export class ComputeSnapshotService { + private readonly logger = new SimpleStructuredLogger("compute-snapshot-service"); + + private static readonly MAX_TRACE_CONTEXTS = 10_000; + private readonly runTraceContexts = new Map(); + private readonly timerWheel: TimerWheel; + private readonly dispatchLimit: ReturnType; + + private readonly computeManager: ComputeWorkloadManager; + private readonly workerClient: SupervisorHttpClient; + private readonly tracing?: OtlpTraceService; + private readonly wideEventOpts: WideEventOptions; + private readonly snapshotCallbackKey: Buffer; + + constructor(opts: ComputeSnapshotServiceOptions) { + this.computeManager = opts.computeManager; + this.workerClient = opts.workerClient; + this.tracing = opts.tracing; + this.wideEventOpts = opts.wideEventOpts; + + // Reject an empty secret up front: an empty HMAC key would make callback + // tokens forgeable by anyone. Guarding here (rather than only at env parse) + // also covers the case where the secret is read from an empty file. + if (!opts.snapshotCallbackSecret) { + throw new Error("snapshotCallbackSecret must not be empty"); + } + // Derive a dedicated key by domain separation so the raw secret is never + // used directly as a MAC key for this protocol. + this.snapshotCallbackKey = createHmac("sha256", opts.snapshotCallbackSecret) + .update(SNAPSHOT_CALLBACK_KEY_INFO) + .digest(); + + this.dispatchLimit = pLimit(this.computeManager.snapshotDispatchLimit); + this.timerWheel = new TimerWheel({ + delayMs: this.computeManager.snapshotDelayMs, + onExpire: (item) => { + this.dispatchLimit(() => this.dispatch(item.data)).catch((error) => { + this.logger.error("Snapshot dispatch failed", { + runId: item.data.runFriendlyId, + runnerId: item.data.runnerId, + error, + }); + }); + }, + }); + this.timerWheel.start(); + } + + /** Schedule a delayed snapshot for a run. Replaces any pending snapshot for the same run. */ + schedule(runFriendlyId: string, data: DelayedSnapshot) { + this.timerWheel.submit(runFriendlyId, data); + emitOneShot({ + ...this.wideEventOpts, + op: "snapshot.schedule", + kind: "event", + populate: (state) => { + state.meta.run_id = runFriendlyId; + state.meta.snapshot_id = data.snapshotFriendlyId; + state.extras.runner_id = data.runnerId; + state.extras.delay_ms = this.computeManager.snapshotDelayMs; + }, + }); + this.logger.debug("Snapshot scheduled", { + runFriendlyId, + snapshotFriendlyId: data.snapshotFriendlyId, + delayMs: this.computeManager.snapshotDelayMs, + }); + } + + /** + * Cancel a pending delayed snapshot. Returns true if one was cancelled. + * When `runnerId` is given, only a snapshot scheduled for that same runner + * is cancelled - a stale runner for a run that has since been reassigned + * must not cancel the new runner's pending snapshot. + */ + cancel(runFriendlyId: string, runnerId?: string): boolean { + if (runnerId) { + const pending = this.timerWheel.peek(runFriendlyId); + if (pending && pending.data.runnerId !== runnerId) { + return false; + } + } + const cancelled = this.timerWheel.cancel(runFriendlyId); + if (cancelled) { + emitOneShot({ + ...this.wideEventOpts, + op: "snapshot.canceled", + kind: "event", + populate: (state) => { + state.meta.run_id = runFriendlyId; + }, + }); + this.logger.debug("Snapshot cancelled", { runFriendlyId }); + } + return cancelled; + } + + /** Handle the callback from the gateway after a snapshot completes or fails. */ + async handleCallback(body: SnapshotCallbackPayload) { + const snapshotId = body.status === "completed" ? body.snapshot_id : undefined; + const runId = body.metadata?.runId; + const snapshotFriendlyId = body.metadata?.snapshotFriendlyId; + + // Enrich the wrapping route's wide event with snapshot metadata. The + // `/api/v1/compute/snapshot-complete` route is registered with `wideRoute`, + // so `fromContext()` returns the State of that route and these calls + // become extras/meta on the same wide event - no nested emission. + const state = fromContext(); + if (state) { + state.extras["snapshot.status"] = body.status; + if (body.instance_id) state.extras["snapshot.instance_id"] = body.instance_id; + if (body.duration_ms !== undefined) state.extras["snapshot.duration_ms"] = body.duration_ms; + if (snapshotId) state.extras["snapshot.id"] = snapshotId; + if (body.status === "failed" && body.error) state.extras["snapshot.error"] = body.error; + } + if (runId) setMeta(state, "run_id", runId); + if (snapshotFriendlyId) setMeta(state, "snapshot_id", snapshotFriendlyId); + + this.logger.debug("Snapshot callback", { + snapshotId, + instanceId: body.instance_id, + status: body.status, + error: body.status === "failed" ? body.error : undefined, + runId, + snapshotFriendlyId, + durationMs: body.duration_ms, + }); + + if (!runId || !snapshotFriendlyId) { + this.logger.error("Snapshot callback missing metadata", { + status: body.status, + instanceId: body.instance_id, + metadataKeys: Object.keys(body.metadata ?? {}), + }); + return { ok: false as const, status: 400 }; + } + + if (!this.#verifyCallbackToken(body.metadata, runId, snapshotFriendlyId)) { + this.logger.error("Snapshot callback failed token verification", { + runId, + snapshotFriendlyId, + instanceId: body.instance_id, + }); + return { ok: false as const, status: 401 }; + } + + this.#emitSnapshotSpan(runId, body.duration_ms, snapshotId); + + if (body.status === "completed") { + const submitStart = performance.now(); + const result = await this.workerClient.submitSuspendCompletion({ + runId, + snapshotId: snapshotFriendlyId, + body: { + success: true, + checkpoint: { + type: "COMPUTE", + location: body.snapshot_id, + }, + }, + }); + recordPhaseSince( + "submit_completion", + submitStart, + result.success ? undefined : new Error(String(result.error)) + ); + + if (result.success) { + this.logger.debug("Suspend completion submitted", { + runId, + instanceId: body.instance_id, + snapshotId: body.snapshot_id, + }); + } else { + setExtra(state, "submit_completion.error", String(result.error)); + this.logger.error("Failed to submit suspend completion", { + runId, + snapshotFriendlyId, + error: result.error, + }); + } + } else { + const submitStart = performance.now(); + const result = await this.workerClient.submitSuspendCompletion({ + runId, + snapshotId: snapshotFriendlyId, + body: { + success: false, + error: body.error ?? "Snapshot failed", + }, + }); + recordPhaseSince( + "submit_completion", + submitStart, + result.success ? undefined : new Error(String(result.error)) + ); + + if (!result.success) { + setExtra(state, "submit_completion.error", String(result.error)); + this.logger.error("Failed to submit suspend failure", { + runId, + snapshotFriendlyId, + error: result.error, + }); + } + } + + return { ok: true as const, status: 200 }; + } + + registerTraceContext(runFriendlyId: string, ctx: RunTraceContext) { + // Evict oldest entries if we've hit the cap. This is best-effort: on a busy + // supervisor, entries for long-lived runs may be evicted before their snapshot + // callback arrives, causing those snapshot spans to be silently dropped. + // That's acceptable - trace spans are observability sugar, not correctness. + if (this.runTraceContexts.size >= ComputeSnapshotService.MAX_TRACE_CONTEXTS) { + const firstKey = this.runTraceContexts.keys().next().value; + if (firstKey) { + this.runTraceContexts.delete(firstKey); + } + } + + this.runTraceContexts.set(runFriendlyId, ctx); + } + + /** Stop the timer wheel, dropping pending snapshots. */ + stop(): string[] { + // Intentionally drop pending snapshots rather than dispatching them. The supervisor + // is shutting down, so our callback URL will be dead by the time the gateway responds. + // Runners detect the supervisor is gone and reconnect to a new instance, which + // re-triggers the snapshot workflow. Snapshots are an optimization, not a correctness + // requirement - runs continue fine without them. + const remaining = this.timerWheel.stop(); + const droppedRuns = remaining.map((item) => item.key); + + if (droppedRuns.length > 0) { + this.logger.info("Stopped, dropped pending snapshots", { count: droppedRuns.length }); + this.logger.debug("Dropped snapshot details", { runs: droppedRuns }); + } + + return droppedRuns; + } + + /** Dispatch a snapshot request to the gateway. */ + private async dispatch(snapshot: DelayedSnapshot): Promise { + await runWideEvent( + { + ...this.wideEventOpts, + op: "snapshot.dispatch", + kind: "scheduled", + setup: (state) => { + state.meta.run_id = snapshot.runFriendlyId; + state.meta.snapshot_id = snapshot.snapshotFriendlyId; + state.extras.runner_id = snapshot.runnerId; + }, + }, + async () => { + const callbackNonce = randomBytes(16).toString("hex"); + const result = await this.computeManager.snapshot({ + runnerId: snapshot.runnerId, + metadata: { + runId: snapshot.runFriendlyId, + snapshotFriendlyId: snapshot.snapshotFriendlyId, + [SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]: callbackNonce, + [SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]: this.#createCallbackToken( + callbackNonce, + snapshot.runFriendlyId, + snapshot.snapshotFriendlyId + ), + }, + }); + + if (!result) { + throw new Error("Snapshot dispatch returned no result"); + } + } + ); + } + + #createCallbackToken(nonce: string, runFriendlyId: string, snapshotFriendlyId: string): string { + return createHmac("sha256", this.snapshotCallbackKey) + .update(nonce) + .update("\0") + .update(runFriendlyId) + .update("\0") + .update(snapshotFriendlyId) + .digest("hex"); + } + + /** + * Verify that a callback carries a token this supervisor issued for the given + * run and snapshot. The token binds only the identifiers known at dispatch + * time (nonce, run, snapshot); it intentionally does not cover result fields + * such as the snapshot location or status/error, which are produced by the + * gateway after the snapshot and so cannot be signed in advance. Verification + * is also stateless, so a token is not single-use. + * + * This closes the primary risk (a caller that can merely reach the endpoint + * cannot mint a valid token, so cannot forge a result for an arbitrary run). + * It does not defend against an attacker who can observe a genuine callback + * and then replay it or alter its unsigned result fields - that relies on the + * gateway->supervisor callback channel being authenticated and encrypted. + */ + #verifyCallbackToken( + metadata: Record | undefined, + runFriendlyId: string, + snapshotFriendlyId: string + ): boolean { + const nonce = metadata?.[SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]; + const token = metadata?.[SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]; + + if (!nonce || !token) { + return false; + } + + const expected = this.#createCallbackToken(nonce, runFriendlyId, snapshotFriendlyId); + const expectedBuffer = Buffer.from(expected, "hex"); + const tokenBuffer = Buffer.from(token, "hex"); + + return ( + expectedBuffer.length === tokenBuffer.length && timingSafeEqual(expectedBuffer, tokenBuffer) + ); + } + + #emitSnapshotSpan(runFriendlyId: string, durationMs?: number, snapshotId?: string) { + if (!this.tracing) return; + + const ctx = this.runTraceContexts.get(runFriendlyId); + if (!ctx) return; + + const parsed = parseTraceparent(ctx.traceparent); + if (!parsed) return; + + const endEpochMs = Date.now(); + const startEpochMs = durationMs ? endEpochMs - durationMs : endEpochMs; + + const spanAttributes: Record = { + "compute.type": "snapshot", + }; + + if (durationMs !== undefined) { + spanAttributes["compute.total_ms"] = durationMs; + } + + if (snapshotId) { + spanAttributes["compute.snapshot_id"] = snapshotId; + } + + this.tracing.emit({ + traceId: parsed.traceId, + parentSpanId: parsed.spanId, + spanName: "compute.snapshot", + startTimeMs: startEpochMs, + endTimeMs: endEpochMs, + resourceAttributes: { + "ctx.environment.id": ctx.envId, + "ctx.organization.id": ctx.orgId, + "ctx.project.id": ctx.projectId, + "ctx.run.id": runFriendlyId, + }, + spanAttributes, + }); + } +} diff --git a/apps/supervisor/src/services/failedPodHandler.test.ts b/apps/supervisor/src/services/failedPodHandler.test.ts index d05783288e..110e580644 100644 --- a/apps/supervisor/src/services/failedPodHandler.test.ts +++ b/apps/supervisor/src/services/failedPodHandler.test.ts @@ -1,10 +1,11 @@ import { describe, it, expect, beforeAll, afterEach } from "vitest"; import { FailedPodHandler } from "./failedPodHandler.js"; -import { K8sApi, createK8sApi } from "../clients/kubernetes.js"; +import { type K8sApi, createK8sApi } from "../clients/kubernetes.js"; import { Registry } from "prom-client"; import { setTimeout } from "timers/promises"; -describe("FailedPodHandler Integration Tests", () => { +// These tests require live K8s cluster credentials - skip by default +describe.skipIf(!process.env.K8S_INTEGRATION_TESTS)("FailedPodHandler Integration Tests", () => { const k8s = createK8sApi(); const namespace = "integration-test"; const register = new Registry(); @@ -13,7 +14,7 @@ describe("FailedPodHandler Integration Tests", () => { // Create the test namespace if it doesn't exist try { await k8s.core.readNamespace({ name: namespace }); - } catch (error) { + } catch (_error) { await k8s.core.createNamespace({ body: { metadata: { diff --git a/apps/supervisor/src/services/failedPodHandler.ts b/apps/supervisor/src/services/failedPodHandler.ts index 0721724376..ca458e0ba4 100644 --- a/apps/supervisor/src/services/failedPodHandler.ts +++ b/apps/supervisor/src/services/failedPodHandler.ts @@ -1,10 +1,11 @@ +import type { Informer, V1Pod } from "@kubernetes/client-node"; import { LogLevel, SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import { K8sApi } from "../clients/kubernetes.js"; +import type { Registry } from "prom-client"; +import { Counter, Histogram } from "prom-client"; +import { setTimeout } from "timers/promises"; +import type { K8sApi } from "../clients/kubernetes.js"; import { createK8sApi } from "../clients/kubernetes.js"; -import { Informer, V1Pod } from "@kubernetes/client-node"; -import { Counter, Registry, Histogram } from "prom-client"; import { register } from "../metrics.js"; -import { setTimeout } from "timers/promises"; type PodStatus = "Pending" | "Running" | "Succeeded" | "Failed" | "Unknown" | "GracefulShutdown"; @@ -151,7 +152,7 @@ export class FailedPodHandler { } private async onPodCompleted(pod: V1Pod) { - this.logger.info("pod-completed", this.podSummary(pod)); + this.logger.debug("pod-completed", this.podSummary(pod)); this.informerEventsTotal.inc({ namespace: this.namespace, verb: "add" }); if (!pod.metadata?.name) { @@ -165,7 +166,7 @@ export class FailedPodHandler { } if (pod.metadata?.deletionTimestamp) { - this.logger.info("pod-completed: pod is being deleted", this.podSummary(pod)); + this.logger.verbose("pod-completed: pod is being deleted", this.podSummary(pod)); return; } @@ -188,7 +189,7 @@ export class FailedPodHandler { } private async onPodSucceeded(pod: V1Pod) { - this.logger.info("pod-succeeded", this.podSummary(pod)); + this.logger.debug("pod-succeeded", this.podSummary(pod)); this.processedPodsTotal.inc({ namespace: this.namespace, status: this.podStatus(pod), @@ -196,7 +197,7 @@ export class FailedPodHandler { } private async onPodFailed(pod: V1Pod) { - this.logger.info("pod-failed", this.podSummary(pod)); + this.logger.debug("pod-failed", this.podSummary(pod)); try { await this.processFailedPod(pod); @@ -208,7 +209,7 @@ export class FailedPodHandler { } private async processFailedPod(pod: V1Pod) { - this.logger.info("pod-failed: processing pod", this.podSummary(pod)); + this.logger.verbose("pod-failed: processing pod", this.podSummary(pod)); const mainContainer = pod.status?.containerStatuses?.find((c) => c.name === "run-controller"); @@ -231,7 +232,7 @@ export class FailedPodHandler { } private async deletePod(pod: V1Pod) { - this.logger.info("pod-failed: deleting pod", this.podSummary(pod)); + this.logger.verbose("pod-failed: deleting pod", this.podSummary(pod)); try { await this.k8s.core.deleteNamespacedPod({ name: pod.metadata!.name!, diff --git a/apps/supervisor/src/services/otlpTraceService.test.ts b/apps/supervisor/src/services/otlpTraceService.test.ts new file mode 100644 index 0000000000..95053021ca --- /dev/null +++ b/apps/supervisor/src/services/otlpTraceService.test.ts @@ -0,0 +1,177 @@ +import { describe, it, expect } from "vitest"; +import { buildPayload } from "./otlpTraceService.js"; + +describe("buildPayload", () => { + it("builds valid OTLP JSON with timing attributes", () => { + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + parentSpanId: "1234567890abcdef", + spanName: "compute.provision", + startTimeMs: 1000, + endTimeMs: 1250, + resourceAttributes: { + "ctx.environment.id": "env_123", + "ctx.organization.id": "org_456", + "ctx.project.id": "proj_789", + "ctx.run.id": "run_abc", + }, + spanAttributes: { + "compute.total_ms": 250, + "compute.gateway.schedule_ms": 1, + "compute.cache.image_cached": true, + }, + }); + + expect(payload.resourceSpans).toHaveLength(1); + + const resourceSpan = payload.resourceSpans[0]!; + + // $trigger=true so the webapp accepts it + const triggerAttr = resourceSpan.resource.attributes.find((a) => a.key === "$trigger"); + expect(triggerAttr).toEqual({ key: "$trigger", value: { boolValue: true } }); + + // Resource attributes + const envAttr = resourceSpan.resource.attributes.find((a) => a.key === "ctx.environment.id"); + expect(envAttr).toEqual({ + key: "ctx.environment.id", + value: { stringValue: "env_123" }, + }); + + // Span basics + const span = resourceSpan.scopeSpans[0]!.spans[0]!; + expect(span.name).toBe("compute.provision"); + expect(span.traceId).toBe("abcd1234abcd1234abcd1234abcd1234"); + expect(span.parentSpanId).toBe("1234567890abcdef"); + + // Integer attribute + const totalMs = span.attributes.find((a) => a.key === "compute.total_ms"); + expect(totalMs).toEqual({ key: "compute.total_ms", value: { intValue: 250 } }); + + // Boolean attribute + const cached = span.attributes.find((a) => a.key === "compute.cache.image_cached"); + expect(cached).toEqual({ key: "compute.cache.image_cached", value: { boolValue: true } }); + }); + + it("generates a valid 16-char hex span ID", () => { + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "test", + startTimeMs: 1000, + endTimeMs: 1001, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + expect(span.spanId).toMatch(/^[0-9a-f]{16}$/); + }); + + it("converts timestamps to nanoseconds", () => { + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "test", + startTimeMs: 1000, + endTimeMs: 1250, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + expect(span.startTimeUnixNano).toBe("1000000000"); + expect(span.endTimeUnixNano).toBe("1250000000"); + }); + + it("converts real epoch timestamps without precision loss", () => { + // Date.now() values exceed Number.MAX_SAFE_INTEGER when multiplied by 1e6 + const startMs = 1711929600000; // 2024-04-01T00:00:00Z + const endMs = 1711929600250; + + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "test", + startTimeMs: startMs, + endTimeMs: endMs, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + expect(span.startTimeUnixNano).toBe("1711929600000000000"); + expect(span.endTimeUnixNano).toBe("1711929600250000000"); + }); + + it("preserves sub-millisecond precision from performance.now() arithmetic", () => { + // provisionStartEpochMs = Date.now() - (performance.now() - startMs) produces fractional ms. + // Use small epoch + fraction to avoid IEEE 754 noise in the fractional part. + const startMs = 1000.322; + const endMs = 1045.789; + + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "test", + startTimeMs: startMs, + endTimeMs: endMs, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + expect(span.startTimeUnixNano).toBe("1000322000"); + expect(span.endTimeUnixNano).toBe("1045789000"); + }); + + it("sub-ms precision affects ordering for real epoch values", () => { + // Two spans within the same millisecond should have different nanosecond timestamps + const spanA = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "a", + startTimeMs: 1711929600000.3, + endTimeMs: 1711929600001, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const spanB = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "b", + startTimeMs: 1711929600000.7, + endTimeMs: 1711929600001, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const startA = BigInt(spanA.resourceSpans[0]!.scopeSpans[0]!.spans[0]!.startTimeUnixNano); + const startB = BigInt(spanB.resourceSpans[0]!.scopeSpans[0]!.spans[0]!.startTimeUnixNano); + // A should sort before B (both in the same ms but different sub-ms positions) + expect(startA).toBeLessThan(startB); + }); + + it("omits parentSpanId when not provided", () => { + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "test", + startTimeMs: 1000, + endTimeMs: 1001, + resourceAttributes: {}, + spanAttributes: {}, + }); + + const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + expect(span.parentSpanId).toBeUndefined(); + }); + + it("handles double values for non-integer numbers", () => { + const payload = buildPayload({ + traceId: "abcd1234abcd1234abcd1234abcd1234", + spanName: "test", + startTimeMs: 1000, + endTimeMs: 1001, + resourceAttributes: {}, + spanAttributes: { "compute.cpu": 0.25 }, + }); + + const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + const cpu = span.attributes.find((a) => a.key === "compute.cpu"); + expect(cpu).toEqual({ key: "compute.cpu", value: { doubleValue: 0.25 } }); + }); +}); diff --git a/apps/supervisor/src/services/otlpTraceService.ts b/apps/supervisor/src/services/otlpTraceService.ts new file mode 100644 index 0000000000..da3310711d --- /dev/null +++ b/apps/supervisor/src/services/otlpTraceService.ts @@ -0,0 +1,104 @@ +import { randomBytes } from "crypto"; +import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; + +export type OtlpTraceServiceOptions = { + endpointUrl: string; + timeoutMs?: number; +}; + +export type OtlpTraceSpan = { + traceId: string; + parentSpanId?: string; + spanName: string; + startTimeMs: number; + endTimeMs: number; + resourceAttributes: Record; + spanAttributes: Record; +}; + +export class OtlpTraceService { + private readonly logger = new SimpleStructuredLogger("otlp-trace"); + + constructor(private opts: OtlpTraceServiceOptions) {} + + /** Fire-and-forget: build payload and send to the configured OTLP endpoint */ + emit(span: OtlpTraceSpan): void { + const payload = buildPayload(span); + + fetch(`${this.opts.endpointUrl}/v1/traces`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(this.opts.timeoutMs ?? 5_000), + }).catch((err) => { + this.logger.warn("failed to send compute trace span", { + error: err instanceof Error ? err.message : String(err), + }); + }); + } +} + +// โ”€โ”€ Payload builder (internal) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** @internal Exported for tests only */ +export function buildPayload(span: OtlpTraceSpan) { + const spanId = randomBytes(8).toString("hex"); + + return { + resourceSpans: [ + { + resource: { + attributes: [ + { key: "$trigger", value: { boolValue: true } }, + ...toOtlpAttributes(span.resourceAttributes), + ], + }, + scopeSpans: [ + { + scope: { name: "supervisor.compute" }, + spans: [ + { + traceId: span.traceId, + spanId, + parentSpanId: span.parentSpanId, + name: span.spanName, + kind: 3, // SPAN_KIND_CLIENT + startTimeUnixNano: msToNano(span.startTimeMs), + endTimeUnixNano: msToNano(span.endTimeMs), + attributes: toOtlpAttributes(span.spanAttributes), + status: { code: 1 }, // STATUS_CODE_OK + }, + ], + }, + ], + }, + ], + }; +} + +function toOtlpAttributes( + attrs: Record +): Array<{ key: string; value: Record }> { + return Object.entries(attrs).map(([key, value]) => ({ + key, + value: toOtlpValue(value), + })); +} + +function toOtlpValue(value: string | number | boolean): Record { + if (typeof value === "string") return { stringValue: value }; + if (typeof value === "boolean") return { boolValue: value }; + if (Number.isInteger(value)) return { intValue: value }; + return { doubleValue: value }; +} + +/** + * Convert epoch milliseconds to nanosecond string, preserving sub-ms precision. + * Fractional ms from performance.now() arithmetic carry meaningful microsecond + * data that affects span sort ordering when events happen within the same ms. + */ +function msToNano(ms: number): string { + const wholeMs = Math.trunc(ms); + const fracNs = Math.round((ms - wholeMs) * 1_000_000); + return String(BigInt(wholeMs) * 1_000_000n + BigInt(fracNs)); +} diff --git a/apps/supervisor/src/services/podCleaner.test.ts b/apps/supervisor/src/services/podCleaner.test.ts index 36bb5de6d1..827f12600e 100644 --- a/apps/supervisor/src/services/podCleaner.test.ts +++ b/apps/supervisor/src/services/podCleaner.test.ts @@ -1,10 +1,11 @@ import { PodCleaner } from "./podCleaner.js"; -import { K8sApi, createK8sApi } from "../clients/kubernetes.js"; +import { type K8sApi, createK8sApi } from "../clients/kubernetes.js"; import { setTimeout } from "timers/promises"; import { describe, it, expect, beforeAll, afterEach } from "vitest"; import { Registry } from "prom-client"; -describe("PodCleaner Integration Tests", () => { +// These tests require live K8s cluster credentials - skip by default +describe.skipIf(!process.env.K8S_INTEGRATION_TESTS)("PodCleaner Integration Tests", () => { const k8s = createK8sApi(); const namespace = "integration-test"; const register = new Registry(); @@ -13,7 +14,7 @@ describe("PodCleaner Integration Tests", () => { // Create the test namespace, only if it doesn't exist try { await k8s.core.readNamespace({ name: namespace }); - } catch (error) { + } catch (_error) { await k8s.core.createNamespace({ body: { metadata: { @@ -324,7 +325,7 @@ async function waitForPodDeletion({ name: podName, }); await setTimeout(waitMs); - } catch (error) { + } catch (_error) { // Pod was deleted return; } diff --git a/apps/supervisor/src/services/podCleaner.ts b/apps/supervisor/src/services/podCleaner.ts index 56eaaeb88a..1ee1f240bd 100644 --- a/apps/supervisor/src/services/podCleaner.ts +++ b/apps/supervisor/src/services/podCleaner.ts @@ -1,8 +1,9 @@ +import { IntervalService } from "@trigger.dev/core/v3"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import { K8sApi } from "../clients/kubernetes.js"; +import type { Registry } from "prom-client"; +import { Counter, Gauge } from "prom-client"; +import type { K8sApi } from "../clients/kubernetes.js"; import { createK8sApi } from "../clients/kubernetes.js"; -import { IntervalService } from "@trigger.dev/core/v3"; -import { Counter, Gauge, Registry } from "prom-client"; import { register } from "../metrics.js"; export type PodCleanerOptions = { @@ -90,7 +91,7 @@ export class PodCleaner { status: "succeeded", }); - this.logger.info("Deleted batch of pods", { continuationToken }); + this.logger.debug("Deleted batch of pods", { continuationToken }); } catch (err) { this.logger.error("Failed to delete batch of pods", { err: err instanceof Error ? err.message : String(err), diff --git a/apps/supervisor/src/services/timerWheel.test.ts b/apps/supervisor/src/services/timerWheel.test.ts new file mode 100644 index 0000000000..e685a26b1b --- /dev/null +++ b/apps/supervisor/src/services/timerWheel.test.ts @@ -0,0 +1,271 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { TimerWheel } from "./timerWheel.js"; + +describe("TimerWheel", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("dispatches item after delay", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + wheel.submit("run-1", "snapshot-data"); + + // Not yet + vi.advanceTimersByTime(2900); + expect(dispatched).toEqual([]); + + // After delay + vi.advanceTimersByTime(200); + expect(dispatched).toEqual(["run-1"]); + + wheel.stop(); + }); + + it("cancels item before it fires", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + wheel.submit("run-1", "data"); + + vi.advanceTimersByTime(1000); + expect(wheel.cancel("run-1")).toBe(true); + + vi.advanceTimersByTime(5000); + expect(dispatched).toEqual([]); + expect(wheel.size).toBe(0); + + wheel.stop(); + }); + + it("peek returns the pending item without removing it", () => { + const wheel = new TimerWheel({ delayMs: 3000, onExpire: () => {} }); + + wheel.start(); + wheel.submit("run-1", "data"); + + expect(wheel.peek("run-1")).toEqual({ key: "run-1", data: "data" }); + expect(wheel.size).toBe(1); + expect(wheel.peek("run-2")).toBeUndefined(); + + // Dispatched items are no longer peekable + vi.advanceTimersByTime(3100); + expect(wheel.peek("run-1")).toBeUndefined(); + + wheel.stop(); + }); + + it("cancel returns false for unknown key", () => { + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: () => {}, + }); + expect(wheel.cancel("nonexistent")).toBe(false); + }); + + it("deduplicates: resubmitting same key replaces the entry", () => { + const dispatched: { key: string; data: string }[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push({ key: item.key, data: item.data }), + }); + + wheel.start(); + wheel.submit("run-1", "old-data"); + + vi.advanceTimersByTime(1000); + wheel.submit("run-1", "new-data"); + + // Original would have fired at t=3000, but was replaced + // New one fires at t=1000+3000=4000 + vi.advanceTimersByTime(2100); + expect(dispatched).toEqual([]); + + vi.advanceTimersByTime(1000); + expect(dispatched).toEqual([{ key: "run-1", data: "new-data" }]); + + wheel.stop(); + }); + + it("handles many concurrent items", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + + for (let i = 0; i < 1000; i++) { + wheel.submit(`run-${i}`, `data-${i}`); + } + expect(wheel.size).toBe(1000); + + vi.advanceTimersByTime(3100); + expect(dispatched.length).toBe(1000); + expect(wheel.size).toBe(0); + + wheel.stop(); + }); + + it("handles items submitted at different times", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + + wheel.submit("run-1", "data"); + vi.advanceTimersByTime(1000); + wheel.submit("run-2", "data"); + vi.advanceTimersByTime(1000); + wheel.submit("run-3", "data"); + + // t=2000: nothing yet + expect(dispatched).toEqual([]); + + // t=3100: run-1 fires + vi.advanceTimersByTime(1100); + expect(dispatched).toEqual(["run-1"]); + + // t=4100: run-2 fires + vi.advanceTimersByTime(1000); + expect(dispatched).toEqual(["run-1", "run-2"]); + + // t=5100: run-3 fires + vi.advanceTimersByTime(1000); + expect(dispatched).toEqual(["run-1", "run-2", "run-3"]); + + wheel.stop(); + }); + + it("setDelay changes delay for new items only", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + + wheel.submit("run-1", "data"); // 3s delay + + vi.advanceTimersByTime(500); + wheel.setDelay(1000); + wheel.submit("run-2", "data"); // 1s delay + + // t=1500: run-2 should have fired (submitted at t=500 with 1s delay) + vi.advanceTimersByTime(1100); + expect(dispatched).toEqual(["run-2"]); + + // t=3100: run-1 fires at its original 3s delay + vi.advanceTimersByTime(1500); + expect(dispatched).toEqual(["run-2", "run-1"]); + + wheel.stop(); + }); + + it("stop returns unprocessed items", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + wheel.submit("run-1", "data-1"); + wheel.submit("run-2", "data-2"); + wheel.submit("run-3", "data-3"); + + const remaining = wheel.stop(); + expect(dispatched).toEqual([]); + expect(wheel.size).toBe(0); + expect(remaining.length).toBe(3); + expect(remaining.map((r) => r.key).sort()).toEqual(["run-1", "run-2", "run-3"]); + expect(remaining.find((r) => r.key === "run-1")?.data).toBe("data-1"); + }); + + it("after stop, new submissions are silently dropped", () => { + const dispatched: string[] = []; + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + wheel.stop(); + + wheel.submit("run-late", "data"); + expect(dispatched).toEqual([]); + expect(wheel.size).toBe(0); + }); + + it("tracks size correctly through submit/cancel/dispatch", () => { + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: () => {}, + }); + + wheel.start(); + + wheel.submit("a", "data"); + wheel.submit("b", "data"); + expect(wheel.size).toBe(2); + + wheel.cancel("a"); + expect(wheel.size).toBe(1); + + vi.advanceTimersByTime(3100); + expect(wheel.size).toBe(0); + + wheel.stop(); + }); + + it("clamps delay to valid range", () => { + const dispatched: string[] = []; + + // Very small delay (should be at least 1 tick = 100ms) + const wheel = new TimerWheel({ + delayMs: 0, + onExpire: (item) => dispatched.push(item.key), + }); + + wheel.start(); + wheel.submit("run-1", "data"); + + vi.advanceTimersByTime(200); + expect(dispatched).toEqual(["run-1"]); + + wheel.stop(); + }); + + it("multiple cancel calls are safe", () => { + const wheel = new TimerWheel({ + delayMs: 3000, + onExpire: () => {}, + }); + + wheel.start(); + wheel.submit("run-1", "data"); + + expect(wheel.cancel("run-1")).toBe(true); + expect(wheel.cancel("run-1")).toBe(false); + + wheel.stop(); + }); +}); diff --git a/apps/supervisor/src/services/timerWheel.ts b/apps/supervisor/src/services/timerWheel.ts new file mode 100644 index 0000000000..cab5a5d7a2 --- /dev/null +++ b/apps/supervisor/src/services/timerWheel.ts @@ -0,0 +1,166 @@ +/** + * TimerWheel implements a hashed timer wheel for efficiently managing large numbers + * of delayed operations with O(1) submit, cancel, and per-item dispatch. + * + * Used by the supervisor to delay snapshot requests so that short-lived waitpoints + * (e.g. triggerAndWait that resolves in <3s) skip the snapshot entirely. + * + * The wheel is a ring buffer of slots. A single setInterval advances a cursor. + * When the cursor reaches a slot, all items in that slot are dispatched. + * + * Fixed capacity: 600 slots at 100ms tick = 60s max delay. + */ + +const TICK_MS = 100; +const NUM_SLOTS = 600; // 60s max delay at 100ms tick + +export type TimerWheelItem = { + key: string; + data: T; +}; + +export type TimerWheelOptions = { + /** Called when an item's delay expires. */ + onExpire: (item: TimerWheelItem) => void; + /** Delay in milliseconds before items fire. Clamped to [100, 60000]. */ + delayMs: number; +}; + +type Entry = { + key: string; + data: T; + slotIndex: number; +}; + +export class TimerWheel { + private slots: Set[]; + private entries: Map>; + private cursor: number; + private intervalId: ReturnType | null; + private onExpire: (item: TimerWheelItem) => void; + private delaySlots: number; + + constructor(opts: TimerWheelOptions) { + this.slots = Array.from({ length: NUM_SLOTS }, () => new Set()); + this.entries = new Map(); + this.cursor = 0; + this.intervalId = null; + this.onExpire = opts.onExpire; + this.delaySlots = Math.max(1, Math.min(NUM_SLOTS, Math.ceil(opts.delayMs / TICK_MS))); + } + + /** Start the timer wheel. Must be called before submitting items. */ + start(): void { + if (this.intervalId) return; + this.intervalId = setInterval(() => this.tick(), TICK_MS); + // Don't hold the process open just for the timer wheel + if (this.intervalId && typeof this.intervalId === "object" && "unref" in this.intervalId) { + this.intervalId.unref(); + } + } + + /** + * Stop the timer wheel and return all unprocessed items. + * The wheel keeps running normally during graceful shutdown - call stop() + * only when you're ready to tear down. Caller decides what to do with leftovers. + */ + stop(): TimerWheelItem[] { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + + const remaining: TimerWheelItem[] = []; + for (const [key, entry] of this.entries) { + remaining.push({ key, data: entry.data }); + } + + for (const slot of this.slots) { + slot.clear(); + } + this.entries.clear(); + + return remaining; + } + + /** + * Update the delay for future submissions. Already-queued items keep their original timing. + * Clamped to [TICK_MS, 60000ms]. + */ + setDelay(delayMs: number): void { + this.delaySlots = Math.max(1, Math.min(NUM_SLOTS, Math.ceil(delayMs / TICK_MS))); + } + + /** + * Submit an item to be dispatched after the configured delay. + * If an item with the same key already exists, it is replaced (dedup). + * No-op if the wheel is stopped. + */ + submit(key: string, data: T): void { + if (!this.intervalId) return; + + // Dedup: remove existing entry for this key + this.cancel(key); + + const slotIndex = (this.cursor + this.delaySlots) % NUM_SLOTS; + const entry: Entry = { key, data, slotIndex }; + + this.entries.set(key, entry); + this.slot(slotIndex).add(key); + } + + /** + * Cancel a pending item. Returns true if the item was found and removed. + */ + cancel(key: string): boolean { + const entry = this.entries.get(key); + if (!entry) return false; + + this.slot(entry.slotIndex).delete(key); + this.entries.delete(key); + return true; + } + + /** Look up a pending item without removing it. */ + peek(key: string): TimerWheelItem | undefined { + const entry = this.entries.get(key); + return entry ? { key, data: entry.data } : undefined; + } + + /** Number of pending items in the wheel. */ + get size(): number { + return this.entries.size; + } + + /** Whether the wheel is running. */ + get running(): boolean { + return this.intervalId !== null; + } + + /** Get a slot by index. The array is fully initialized so this always returns a Set. */ + private slot(index: number): Set { + const s = this.slots[index]; + if (!s) throw new Error(`TimerWheel: invalid slot index ${index}`); + return s; + } + + /** Advance the cursor and dispatch all items in the current slot. */ + private tick(): void { + this.cursor = (this.cursor + 1) % NUM_SLOTS; + const slot = this.slot(this.cursor); + + if (slot.size === 0) return; + + // Collect items to dispatch (copy keys since we mutate during iteration) + const keys = [...slot]; + slot.clear(); + + for (const key of keys) { + const entry = this.entries.get(key); + if (!entry) continue; + + this.entries.delete(key); + this.onExpire({ key, data: entry.data }); + } + } +} diff --git a/apps/supervisor/src/services/warmStartVerificationService.test.ts b/apps/supervisor/src/services/warmStartVerificationService.test.ts new file mode 100644 index 0000000000..4a5f58875c --- /dev/null +++ b/apps/supervisor/src/services/warmStartVerificationService.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, vi } from "vitest"; +import { setTimeout as sleep } from "node:timers/promises"; +import { WarmStartVerificationService } from "./warmStartVerificationService.js"; +import type { DequeuedMessage } from "@trigger.dev/core/v3"; +import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers"; + +// The TimerWheel ticks every 100ms, so a 1000ms delay (the env minimum) +// fires within ~1.1s. +const DELAY_MS = 1_000; +// Long enough that a pending verification would certainly have fired. +const SETTLE_MS = 1_600; + +const DEQUEUED_SNAPSHOT_ID = "snapshot_dequeued"; + +function makeMessage(runFriendlyId = "run_1"): DequeuedMessage { + return { + run: { friendlyId: runFriendlyId }, + snapshot: { friendlyId: DEQUEUED_SNAPSHOT_ID }, + } as unknown as DequeuedMessage; +} + +function createService(opts: { latestSnapshotId?: string; probeError?: boolean }) { + const getLatestSnapshot = vi.fn(async (_runId: string) => + opts.probeError + ? { success: false as const, error: "connection refused" } + : { + success: true as const, + data: { execution: { snapshot: { friendlyId: opts.latestSnapshotId } } }, + } + ); + + const createWorkload = vi.fn(async () => {}); + + const service = new WarmStartVerificationService({ + workerClient: { getLatestSnapshot } as unknown as SupervisorHttpClient, + delayMs: DELAY_MS, + createWorkload, + wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + }); + + return { service, getLatestSnapshot, createWorkload }; +} + +describe("WarmStartVerificationService", () => { + it("falls back to a cold create when the snapshot is unchanged", async () => { + const { service, createWorkload } = createService({ + latestSnapshotId: DEQUEUED_SNAPSHOT_ID, + }); + try { + const message = makeMessage(); + const timings = { warmStartCheckMs: 12 }; + service.schedule(message, timings); + + await vi.waitFor(() => expect(createWorkload).toHaveBeenCalledTimes(1), { + timeout: 3_000, + }); + expect(createWorkload).toHaveBeenCalledWith(message, timings); + } finally { + service.stop(); + } + }); + + it("does nothing when the snapshot has moved on (delivered)", async () => { + const { service, getLatestSnapshot, createWorkload } = createService({ + latestSnapshotId: "snapshot_executing", + }); + try { + service.schedule(makeMessage(), { warmStartCheckMs: 12 }); + + await vi.waitFor(() => expect(getLatestSnapshot).toHaveBeenCalledTimes(1), { + timeout: 3_000, + }); + await sleep(100); + expect(createWorkload).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("never falls back when the probe errors", async () => { + const { service, getLatestSnapshot, createWorkload } = createService({ probeError: true }); + try { + service.schedule(makeMessage(), { warmStartCheckMs: 12 }); + + await vi.waitFor(() => expect(getLatestSnapshot).toHaveBeenCalledTimes(1), { + timeout: 3_000, + }); + await sleep(100); + expect(createWorkload).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("cancel before the delay prevents the probe entirely", async () => { + const { service, getLatestSnapshot, createWorkload } = createService({ + latestSnapshotId: DEQUEUED_SNAPSHOT_ID, + }); + try { + service.schedule(makeMessage(), { warmStartCheckMs: 12 }); + + expect(service.cancel("run_1")).toBe(true); + + await sleep(SETTLE_MS); + expect(getLatestSnapshot).not.toHaveBeenCalled(); + expect(createWorkload).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("re-scheduling the same run replaces the pending verification", async () => { + const { service, getLatestSnapshot } = createService({ + latestSnapshotId: "snapshot_executing", + }); + try { + service.schedule(makeMessage(), { warmStartCheckMs: 1 }); + service.schedule(makeMessage(), { warmStartCheckMs: 2 }); + + await vi.waitFor(() => expect(getLatestSnapshot).toHaveBeenCalledTimes(1), { + timeout: 3_000, + }); + await sleep(SETTLE_MS); + expect(getLatestSnapshot).toHaveBeenCalledTimes(1); + } finally { + service.stop(); + } + }); +}); diff --git a/apps/supervisor/src/services/warmStartVerificationService.ts b/apps/supervisor/src/services/warmStartVerificationService.ts new file mode 100644 index 0000000000..ce0bbe4f16 --- /dev/null +++ b/apps/supervisor/src/services/warmStartVerificationService.ts @@ -0,0 +1,173 @@ +import pLimit from "p-limit"; +import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; +import type { DequeuedMessage } from "@trigger.dev/core/v3"; +import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers"; +import { tryCatch } from "@trigger.dev/core"; +import { TimerWheel } from "./timerWheel.js"; +import { emitOneShot, type WideEventOptions } from "../wideEvents/index.js"; + +const PROBE_CONCURRENCY_LIMIT = 10; + +export type WarmStartTimings = { + dequeueResponseMs?: number; + pollingIntervalMs?: number; + warmStartCheckMs: number; +}; + +type PendingVerification = { + message: DequeuedMessage; + timings: WarmStartTimings; +}; + +export type WarmStartVerificationServiceOptions = { + workerClient: SupervisorHttpClient; + /** How long after a warm-start hit to verify the runner acted on it. */ + delayMs: number; + /** Cold-creates the workload for a dispatched-but-lost run. */ + createWorkload: (message: DequeuedMessage, timings: WarmStartTimings) => Promise; + wideEventOpts: WideEventOptions; +}; + +/** + * Verifies that warm-start dispatches were actually acted on. + * + * Firestarter's `didWarmStart: true` means "response written to a socket", + * not "runner received it". A silently dead poller (no FIN - e.g. a VM torn + * down mid-poll) leaves the dispatched run stuck in PENDING_EXECUTING until + * the run engine's heartbeat redrive minutes later, burning a queue + * redelivery each time (TRI-10659). + * + * After a hit, the dequeued message is retained for `delayMs`, then the + * platform is asked for the run's latest snapshot. If it is still the exact + * snapshot we dequeued, no runner ever started the attempt - fall through to + * the regular cold-create path with the original message. Double-starts are + * impossible: `startRunAttempt` runs under a per-run lock and rejects stale + * snapshot ids, so if the original runner revives and races the fallback + * workload, exactly one wins and the loser exits before executing anything. + * + * On a probe ERROR we deliberately do nothing: the runner's attempt-start + * goes through nested retries, so during platform brownouts a healthy runner + * can legitimately act late - falling back on uncertainty would stampede + * duplicate workloads exactly when the platform is degraded. The heartbeat + * redrive remains the backstop for that case (and for supervisor restarts, + * which drop the in-memory timers). + */ +export class WarmStartVerificationService { + private readonly logger = new SimpleStructuredLogger("warm-start-verification"); + + private readonly timerWheel: TimerWheel; + private readonly probeLimit: ReturnType; + + private readonly workerClient: SupervisorHttpClient; + private readonly delayMs: number; + private readonly createWorkload: WarmStartVerificationServiceOptions["createWorkload"]; + private readonly wideEventOpts: WideEventOptions; + + constructor(opts: WarmStartVerificationServiceOptions) { + this.workerClient = opts.workerClient; + this.delayMs = opts.delayMs; + this.createWorkload = opts.createWorkload; + this.wideEventOpts = opts.wideEventOpts; + + this.probeLimit = pLimit(PROBE_CONCURRENCY_LIMIT); + this.timerWheel = new TimerWheel({ + delayMs: opts.delayMs, + onExpire: (item) => { + this.probeLimit(() => this.verify(item.data)).catch((error) => { + this.logger.error("Verification failed", { + runId: item.data.message.run.friendlyId, + error, + }); + }); + }, + }); + this.timerWheel.start(); + } + + /** Schedule delivery verification for a warm-start hit. */ + schedule(message: DequeuedMessage, timings: WarmStartTimings) { + this.timerWheel.submit(message.run.friendlyId, { message, timings }); + this.logger.debug("Verification scheduled", { + runId: message.run.friendlyId, + snapshotId: message.snapshot.friendlyId, + delayMs: this.delayMs, + }); + } + + /** + * Cancel a pending verification, e.g. when the runner connects to this + * supervisor. Purely an optimization: the matched runner often lives on a + * different node and connects to that node's supervisor, so most healthy + * deliveries are confirmed by the probe, not by this. + */ + cancel(runFriendlyId: string): boolean { + return this.timerWheel.cancel(runFriendlyId); + } + + /** Stop the timer wheel, dropping pending verifications. The run engine's + * heartbeat redrive covers anything dropped here. */ + stop() { + const remaining = this.timerWheel.stop(); + if (remaining.length > 0) { + this.logger.info("Stopped, dropped pending verifications", { count: remaining.length }); + } + } + + private async verify({ message, timings }: PendingVerification) { + const runFriendlyId = message.run.friendlyId; + + const result = await this.workerClient.getLatestSnapshot(runFriendlyId); + + if (!result.success) { + // Never fall back on uncertainty - see class docs. + this.emitOutcome(message, "probe_error", String(result.error)); + this.logger.warn("Verification probe failed, skipping", { + runId: runFriendlyId, + error: result.error, + }); + return; + } + + const latestSnapshotId = result.data.execution.snapshot.friendlyId; + + if (latestSnapshotId !== message.snapshot.friendlyId) { + // Something acted on the run (attempt started, or it was cancelled or + // requeued) - the dispatch is no longer ours to worry about. + this.emitOutcome(message, "delivered"); + return; + } + + this.emitOutcome(message, "fallback"); + this.logger.warn("Warm start dispatch was never acted on, cold starting", { + runId: runFriendlyId, + snapshotId: message.snapshot.friendlyId, + }); + + const [error] = await tryCatch(this.createWorkload(message, timings)); + if (error) { + this.logger.error("Fallback workload create failed", { + runId: runFriendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + private emitOutcome( + message: DequeuedMessage, + outcome: "delivered" | "fallback" | "probe_error", + error?: string + ) { + emitOneShot({ + ...this.wideEventOpts, + op: "warmstart.verify", + kind: "event", + populate: (state) => { + state.meta.run_id = message.run.friendlyId; + state.meta.snapshot_id = message.snapshot.friendlyId; + state.extras.outcome = outcome; + state.extras.delay_ms = this.delayMs; + if (error) state.extras.error = error; + }, + }); + } +} diff --git a/apps/supervisor/src/util.ts b/apps/supervisor/src/util.ts index 4fcda27b2a..d14dd99bfe 100644 --- a/apps/supervisor/src/util.ts +++ b/apps/supervisor/src/util.ts @@ -14,6 +14,18 @@ export function getDockerHostDomain() { return isMacOS || isWindows ? "host.docker.internal" : "localhost"; } +/** Extract the W3C traceparent string from an untyped trace context record */ +export function extractTraceparent(traceContext?: Record): string | undefined { + if ( + traceContext && + "traceparent" in traceContext && + typeof traceContext.traceparent === "string" + ) { + return traceContext.traceparent; + } + return undefined; +} + export function getRunnerId(runId: string, attemptNumber?: number) { const parts = ["runner", runId.replace("run_", "")]; @@ -23,3 +35,10 @@ export function getRunnerId(runId: string, attemptNumber?: number) { return parts.join("-"); } + +/** Derive a unique runnerId for a restore cycle using the checkpoint suffix */ +export function getRestoreRunnerId(runFriendlyId: string, checkpointId: string) { + const runIdShort = runFriendlyId.replace("run_", ""); + const checkpointSuffix = checkpointId.slice(-8); + return `runner-${runIdShort}-${checkpointSuffix}`; +} diff --git a/apps/supervisor/src/wideEvents/baggage.test.ts b/apps/supervisor/src/wideEvents/baggage.test.ts new file mode 100644 index 0000000000..0579533345 --- /dev/null +++ b/apps/supervisor/src/wideEvents/baggage.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from "vitest"; +import { encodeBaggage } from "./baggage.js"; + +describe("encodeBaggage", () => { + it("returns empty string for an empty map", () => { + expect(encodeBaggage({})).toBe(""); + }); + + it("encodes a single entry as k=v", () => { + expect(encodeBaggage({ run_id: "run-1" })).toBe("run_id=run-1"); + }); + + it("sorts keys for stable output across hops", () => { + expect(encodeBaggage({ b: "2", a: "1", c: "3" })).toBe("a=1,b=2,c=3"); + }); + + it("skips empty keys and empty values", () => { + expect(encodeBaggage({ "": "v", k: "", real: "x" })).toBe("real=x"); + }); + + it("truncates values longer than the cap", () => { + const long = "x".repeat(1024); + const got = encodeBaggage({ k: long }); + const value = got.slice("k=".length); + expect(value.length).toBe(256); + }); + + it("caps multibyte values by UTF-8 bytes, not code units", () => { + const long = "ใ‚".repeat(512); // 3 UTF-8 bytes each + const got = encodeBaggage({ k: long }); + const value = got.slice("k=".length); + expect(Buffer.byteLength(value, "utf8")).toBeLessThanOrEqual(256); + }); + + it("caps the number of entries", () => { + const meta: Record = {}; + for (let i = 0; i < 50; i++) { + // Sortable two-digit keys so we know which 32 survive. + meta[`k${String(i).padStart(2, "0")}`] = "v"; + } + const got = encodeBaggage(meta); + expect(got.split(",").length).toBe(32); + }); +}); diff --git a/apps/supervisor/src/wideEvents/baggage.ts b/apps/supervisor/src/wideEvents/baggage.ts new file mode 100644 index 0000000000..7750ac7930 --- /dev/null +++ b/apps/supervisor/src/wideEvents/baggage.ts @@ -0,0 +1,45 @@ +/** + * W3C Baggage (https://www.w3.org/TR/baggage/) encoding for outbound peer + * calls. Serialises a State's `meta` map into a `Baggage` header value so + * the downstream service auto-stamps the same labels onto its own wide + * events - even on early-error paths that bail before parsing the request + * body. + * + * Outbound discipline: only call this on peer-to-peer hops within the trust + * boundary. External-endpoint calls (image registries, cloud-provider + * APIs, third-party webhooks) must not include the Baggage header. + */ + +import { truncateUtf8 } from "./truncate.js"; + +/** + * Cap the number of entries serialised onto the header. A misbehaving + * caller's `meta` map shouldn't blow up downstream event width. + */ +const MAX_BAGGAGE_ENTRIES = 32; + +/** + * Cap each value's length. Defense against an upstream that stuffs + * unbounded payloads into a meta value. + */ +const MAX_BAGGAGE_VALUE_BYTES = 256; + +/** + * Encode a `meta` map as a Baggage header value (`k1=v1,k2=v2`). Keys are + * sorted for stable output across hops; an empty input yields the empty + * string so the caller can skip emitting the header entirely. + */ +export function encodeBaggage(meta: Record): string { + const entries = Object.entries(meta).filter(([k, v]) => k && v); + if (entries.length === 0) return ""; + + entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + + const out: string[] = []; + for (const [k, raw] of entries) { + if (out.length >= MAX_BAGGAGE_ENTRIES) break; + const v = truncateUtf8(raw, MAX_BAGGAGE_VALUE_BYTES); + out.push(`${k}=${v}`); + } + return out.join(","); +} diff --git a/apps/supervisor/src/wideEvents/context.ts b/apps/supervisor/src/wideEvents/context.ts new file mode 100644 index 0000000000..a89859c270 --- /dev/null +++ b/apps/supervisor/src/wideEvents/context.ts @@ -0,0 +1,14 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type { State } from "./state.js"; + +/** + * AsyncLocalStorage threading per-operation `State` through the call stack. + * Wrappers enter a state via `wideEventStorage.run(state, () => fn())` and + * any code in the async call tree retrieves it via `fromContext()`. + */ +export const wideEventStorage = new AsyncLocalStorage(); + +/** Returns the State attached to the current async context, or null. */ +export function fromContext(): State | null { + return wideEventStorage.getStore() ?? null; +} diff --git a/apps/supervisor/src/wideEvents/emit.test.ts b/apps/supervisor/src/wideEvents/emit.test.ts new file mode 100644 index 0000000000..0daefa6487 --- /dev/null +++ b/apps/supervisor/src/wideEvents/emit.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect } from "vitest"; +import { emit, EmitMessage } from "./emit.js"; +import { newState } from "./new.js"; + +function captureEmit(state: Parameters[0]): Record { + const captured: string[] = []; + const origWrite = process.stdout.write; + process.stdout.write = ((chunk: unknown) => { + captured.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + try { + emit(state); + } finally { + process.stdout.write = origWrite; + } + expect(captured).toHaveLength(1); + const line = captured[0]; + if (!line) throw new Error("no captured line"); + return JSON.parse(line) as Record; +} + +describe("emit", () => { + it("emits a single line with the stable message + request_id", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 200; + s.ok = true; + s.durationMs = 5; + const out = captureEmit(s); + expect(out.msg).toBe(EmitMessage); + expect(out.request_id).toBe(s.requestId); + expect(out.service).toBe("supervisor"); + expect(out.ok).toBe(true); + expect(out.status).toBe(200); + expect(out.duration_ms).toBe(5); + }); + + it("emits start_time as an ISO timestamp set by newState", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 200; + s.ok = true; + const out = captureEmit(s); + expect(typeof out.start_time).toBe("string"); + // Microsecond-precision RFC3339 (6 fractional digits), parseable as a date. + expect(out.start_time).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/); + expect(Number.isNaN(new Date(out.start_time as string).getTime())).toBe(false); + }); + + it("omits start_time when unset", () => { + const s = newState({ service: "supervisor", env: {} }); + delete s.startTime; + s.statusCode = 200; + s.ok = true; + const out = captureEmit(s); + expect(out).not.toHaveProperty("start_time"); + }); + + it("omits empty optional fields", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 200; + s.ok = true; + const out = captureEmit(s); + expect(out).not.toHaveProperty("trace_id"); + expect(out).not.toHaveProperty("version"); + expect(out).not.toHaveProperty("commit_sha"); + expect(out).not.toHaveProperty("error.code"); + }); + + it("flattens meta keys as meta.", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 200; + s.ok = true; + s.meta.run_id = "run_abc"; + s.meta.deployment_id = "dep_xyz"; + const out = captureEmit(s); + expect(out["meta.run_id"]).toBe("run_abc"); + expect(out["meta.deployment_id"]).toBe("dep_xyz"); + expect(out).not.toHaveProperty("meta"); + }); + + it("flattens phases as phase..", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 200; + s.ok = true; + s.phases.push({ name: "warm_start", durationMs: 12, ok: true, attempts: 1 }); + s.phases.push({ + name: "workload_create", + durationMs: 3, + ok: false, + attempts: 2, + errorCode: "Error", + errorMsg: "boom", + sub: { create_ms: 1 }, + }); + const out = captureEmit(s); + expect(out["phase.warm_start.duration_ms"]).toBe(12); + expect(out["phase.warm_start.ok"]).toBe(true); + expect(out["phase.warm_start.attempts"]).toBe(1); + expect(out["phase.workload_create.duration_ms"]).toBe(3); + expect(out["phase.workload_create.ok"]).toBe(false); + expect(out["phase.workload_create.attempts"]).toBe(2); + expect(out["phase.workload_create.error_code"]).toBe("Error"); + expect(out["phase.workload_create.error_message"]).toBe("boom"); + expect(out["phase.workload_create.create_ms"]).toBe(1); + }); + + it("includes error.code/message/kind when state.error is set", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 500; + s.error = { code: "InternalError", message: "kaboom", kind: "internal" }; + const out = captureEmit(s); + expect(out["error.code"]).toBe("InternalError"); + expect(out["error.message"]).toBe("kaboom"); + expect(out["error.kind"]).toBe("internal"); + }); + + it("truncates very long error messages", () => { + const s = newState({ service: "supervisor", env: {} }); + s.error = { code: "Big", message: "x".repeat(2000), kind: "internal" }; + const out = captureEmit(s); + expect((out["error.message"] as string).length).toBe(512); + }); + + it("flattens extras at the top level", () => { + const s = newState({ service: "supervisor", env: {} }); + s.statusCode = 200; + s.ok = true; + s.extras.route = "/health"; + s.extras["dispatch.result"] = "hit"; + const out = captureEmit(s); + expect(out.route).toBe("/health"); + expect(out["dispatch.result"]).toBe("hit"); + }); +}); diff --git a/apps/supervisor/src/wideEvents/emit.ts b/apps/supervisor/src/wideEvents/emit.ts new file mode 100644 index 0000000000..bfb03ad36c --- /dev/null +++ b/apps/supervisor/src/wideEvents/emit.ts @@ -0,0 +1,84 @@ +import type { State } from "./state.js"; +import { truncateUtf8 } from "./truncate.js"; + +/** + * Stable slog message string for every wide event. Downstream filters (jq, + * Axiom queries, Vector pipelines) pin to this constant. The `service` field + * disambiguates which service emitted it. + */ +export const EmitMessage = "wide_event"; + +const MAX_ERROR_MSG_BYTES = 512; + +/** + * Serializes a State as a single flat-keyed JSON line on stdout. Keys are + * flat (no nested objects) to keep jq filtering and Axiom indexing cheap. + * Empty optional fields are omitted. + */ +export function emit(state: State): void { + // Best-effort: an observability failure (serialization, stdout write) must + // never break or mask the caller's operation. Every call site relies on this. + try { + const out: Record = { + msg: EmitMessage, + request_id: state.requestId, + }; + + if (state.traceId) out.trace_id = state.traceId; + appendIfSet(out, "start_time", state.startTime); + appendIfSet(out, "service", state.service); + appendIfSet(out, "version", state.version); + appendIfSet(out, "commit_sha", state.commitSha); + appendIfSet(out, "region", state.region); + appendIfSet(out, "node_id", state.nodeId); + + appendIfSet(out, "op", state.op); + appendIfSet(out, "kind", state.kind); + + out.ok = state.ok; + if (state.statusCode !== 0) out.status = state.statusCode; + out.duration_ms = state.durationMs; + + if (state.error) { + appendIfSet(out, "error.code", state.error.code); + appendIfSet(out, "error.message", truncateUtf8(state.error.message, MAX_ERROR_MSG_BYTES)); + appendIfSet(out, "error.kind", state.error.kind); + } + + for (const [k, v] of Object.entries(state.meta)) { + out["meta." + k] = v; + } + + for (const p of state.phases) { + const prefix = "phase." + p.name + "."; + out[prefix + "duration_ms"] = p.durationMs; + out[prefix + "ok"] = p.ok; + out[prefix + "attempts"] = p.attempts; + if (p.errorCode) out[prefix + "error_code"] = p.errorCode; + if (p.errorMsg) out[prefix + "error_message"] = p.errorMsg; + if (p.sub) { + for (const [sk, sv] of Object.entries(p.sub)) { + out[prefix + sk] = sv; + } + } + } + + for (const [k, v] of Object.entries(state.extras)) { + out[k] = v; + } + + process.stdout.write(JSON.stringify(out) + "\n"); + } catch (err) { + try { + process.stderr.write( + `wide_event_emit_failed: ${err instanceof Error ? err.message : String(err)}\n` + ); + } catch { + // last resort - drop the event rather than throw + } + } +} + +function appendIfSet(out: Record, key: string, value: string | undefined): void { + if (value) out[key] = value; +} diff --git a/apps/supervisor/src/wideEvents/index.ts b/apps/supervisor/src/wideEvents/index.ts new file mode 100644 index 0000000000..6e61d85896 --- /dev/null +++ b/apps/supervisor/src/wideEvents/index.ts @@ -0,0 +1,24 @@ +/** + * Wide-event observability surface for the supervisor. One flat-keyed JSON + * line per natural unit of work (HTTP request, dequeue iteration, socket + * lifecycle event). Events join across services via `trace_id` (parsed from + * the inbound W3C `traceparent`) and `meta.run_id`. + * + * Off by default behind a kill switch - the dispatch hotpath runs at high + * QPS, so logging pressure must be cleanly removable. + */ +export { type Env, isValidRequestId, newState, type NewStateOptions } from "./new.js"; +export { emit, EmitMessage } from "./emit.js"; +export { parseTraceId } from "./traceparent.js"; +export { fromContext, wideEventStorage } from "./context.js"; +export { type PhaseOpt, recordPhase, recordPhaseSince, timePhase } from "./record.js"; +export { + emitOneShot, + runWideEvent, + setExtra, + setMeta, + type WideEventLifecycleOptions, + type WideEventOptions, +} from "./middleware.js"; +export type { ErrorInfo, PhaseRecord, State } from "./state.js"; +export { encodeBaggage } from "./baggage.js"; diff --git a/apps/supervisor/src/wideEvents/middleware.test.ts b/apps/supervisor/src/wideEvents/middleware.test.ts new file mode 100644 index 0000000000..a431df4ada --- /dev/null +++ b/apps/supervisor/src/wideEvents/middleware.test.ts @@ -0,0 +1,206 @@ +import { describe, it, expect } from "vitest"; +import { fromContext } from "./context.js"; +import { emitOneShot, runWideEvent, setMeta } from "./middleware.js"; + +function captureStdout(fn: () => Promise | unknown): Promise { + const captured: string[] = []; + const orig = process.stdout.write; + process.stdout.write = ((chunk: unknown) => { + captured.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + return Promise.resolve(fn()) + .finally(() => { + process.stdout.write = orig; + }) + .then(() => captured); +} + +describe("runWideEvent", () => { + it("emits one event with ok=true when no statusCode is set", async () => { + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: true, op: "test", route: "/x", method: "POST" }, + async () => undefined + ); + }); + expect(lines).toHaveLength(1); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev.ok).toBe(true); + expect(ev.service).toBe("supervisor"); + expect(ev.route).toBe("/x"); + expect(ev.method).toBe("POST"); + expect(typeof ev.duration_ms).toBe("number"); + expect(typeof ev.request_id).toBe("string"); + }); + + it("derives ok from statusCode set via finalize", async () => { + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: true, op: "test" }, + async () => undefined, + (state) => { + state.statusCode = 200; + } + ); + }); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev.ok).toBe(true); + expect(ev.status).toBe(200); + }); + + it("treats 4xx as ok=false", async () => { + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: true, op: "test" }, + async () => undefined, + (state) => { + state.statusCode = 400; + } + ); + }); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev.ok).toBe(false); + expect(ev.status).toBe(400); + }); + + it("emits ok=false with error.kind=internal on throw", async () => { + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: true, op: "test" }, + async () => { + throw new Error("boom"); + } + ).catch(() => undefined); + }); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev.ok).toBe(false); + expect(ev.status).toBe(500); + expect(ev["error.kind"]).toBe("internal"); + expect(ev["error.message"]).toBe("boom"); + }); + + it("threads state through AsyncLocalStorage", async () => { + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: true, op: "test" }, + async () => { + setMeta(fromContext(), "run_id", "run_abc"); + } + ); + }); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev["meta.run_id"]).toBe("run_abc"); + expect(ev.ok).toBe(true); + }); + + it("picks up inbound traceparent for trace_id", async () => { + const tp = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: true, op: "test", traceparent: tp }, + async () => undefined + ); + }); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev.trace_id).toBe("4bf92f3577b34da6a3ce929d0e0e4736"); + }); + + it("honours setup() to attach meta and extras before fn runs", async () => { + const lines = await captureStdout(async () => { + await runWideEvent( + { + service: "supervisor", + env: {}, + enabled: true, + op: "test", + setup: (state) => { + state.meta.run_id = "run_abc"; + state.extras.iteration = "dequeue"; + }, + }, + async () => undefined + ); + }); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev["meta.run_id"]).toBe("run_abc"); + expect(ev.iteration).toBe("dequeue"); + }); + + it("short-circuits to pass-through when enabled=false", async () => { + let seenState: ReturnType = null; + const lines = await captureStdout(async () => { + await runWideEvent( + { service: "supervisor", env: {}, enabled: false, op: "test" }, + async () => { + seenState = fromContext(); + } + ); + }); + expect(lines).toHaveLength(0); + expect(seenState).toBe(null); + }); + + it("isolates state across concurrent invocations", async () => { + const lines = await captureStdout(async () => { + await Promise.all( + ["a", "b", "c"].map((tag) => + runWideEvent({ service: "supervisor", env: {}, enabled: true, op: "test" }, async () => { + const s = fromContext(); + if (!s) throw new Error("no state"); + s.meta.tag = tag; + await new Promise((r) => setTimeout(r, 5)); + expect(s.meta.tag).toBe(tag); + }) + ) + ); + }); + const tags = lines.map((l) => (JSON.parse(l) as Record)["meta.tag"]); + expect(tags.sort()).toEqual(["a", "b", "c"]); + }); +}); + +describe("emitOneShot", () => { + it("emits a single event with populated meta when enabled", async () => { + const lines = await captureStdout(() => { + emitOneShot({ + service: "supervisor", + env: {}, + enabled: true, + op: "test", + populate: (s) => { + s.meta.run_id = "run_abc"; + s.extras.event = "run:start"; + }, + }); + }); + expect(lines).toHaveLength(1); + const line = lines[0]; + if (!line) throw new Error("no line"); + const ev = JSON.parse(line) as Record; + expect(ev.ok).toBe(true); + expect(ev["meta.run_id"]).toBe("run_abc"); + expect(ev.event).toBe("run:start"); + }); + + it("emits nothing when disabled", async () => { + const lines = await captureStdout(() => { + emitOneShot({ service: "supervisor", env: {}, enabled: false, op: "test" }); + }); + expect(lines).toHaveLength(0); + }); +}); diff --git a/apps/supervisor/src/wideEvents/middleware.ts b/apps/supervisor/src/wideEvents/middleware.ts new file mode 100644 index 0000000000..034c136414 --- /dev/null +++ b/apps/supervisor/src/wideEvents/middleware.ts @@ -0,0 +1,132 @@ +import { emit } from "./emit.js"; +import { newState, type Env } from "./new.js"; +import { wideEventStorage } from "./context.js"; +import type { State } from "./state.js"; + +/** Options common to every wide-event lifecycle. */ +export type WideEventOptions = { + service: string; + env: Env; + /** + * Kill switch. When false, lifecycles degenerate into transparent + * pass-through - no State allocation, no AsyncLocalStorage run, no emit. + * Important for the dispatch hotpath where logging pressure must be + * cleanly removable. + */ + enabled: boolean; +}; + +/** Per-invocation options layered on top of `WideEventOptions`. */ +export type WideEventLifecycleOptions = WideEventOptions & { + /** Operation discriminator (`instance.create`, `dequeue`, ...). Required. */ + op: string; + /** Event shape: `inbound` | `outbound` | `event` | `scheduled`. Optional. */ + kind?: string; + /** Route template (HTTP only) captured into `extras.route`. */ + route?: string; + /** HTTP method captured into `extras.method`. */ + method?: string; + /** Inbound W3C traceparent (HTTP header, queue message field). */ + traceparent?: string; + /** Inbound request id (e.g. `x-request-id` header). */ + inboundRequestId?: string; + /** Runs after the state is built, before the wrapped fn. Use to attach meta. */ + setup?: (state: State) => void; +}; + +/** + * Runs `fn` inside an AsyncLocalStorage state and emits one wide event on + * completion or error. `finalize` runs after `fn` returns but before emit - + * use it to read out-of-band outcome info (e.g. `res.statusCode` for an HTTP + * route) and assign to `state.statusCode`. The wrapper computes `ok` from + * `statusCode` if it's set; otherwise it defaults to true on success. + * + * Returns the original `fn` result. When `enabled=false`, `fn` runs unchanged + * with no event emitted. + */ +export async function runWideEvent( + opts: WideEventLifecycleOptions, + fn: () => Promise | T, + finalize?: (state: State) => void +): Promise { + if (!opts.enabled) { + return fn(); + } + + const state = newState({ + service: opts.service, + env: opts.env, + inboundRequestId: opts.inboundRequestId, + traceparent: opts.traceparent, + op: opts.op, + kind: opts.kind, + }); + if (opts.route) state.extras.route = opts.route; + if (opts.method) state.extras.method = opts.method; + + const start = performance.now(); + try { + if (opts.setup) opts.setup(state); + const result = await wideEventStorage.run(state, () => Promise.resolve(fn())); + state.durationMs = Math.round(performance.now() - start); + if (finalize) finalize(state); + if (state.statusCode !== 0) { + state.ok = state.statusCode >= 200 && state.statusCode < 300; + } else { + state.ok = true; + } + emit(state); + return result; + } catch (err) { + state.durationMs = Math.round(performance.now() - start); + const e = err instanceof Error ? err : new Error(String(err)); + if (state.statusCode === 0) state.statusCode = 500; + state.ok = false; + state.error = { + code: e.name || "Error", + message: e.message, + kind: "internal", + }; + emit(state); + throw err; + } +} + +/** + * One-shot wide event with no wrapped operation. Use for socket lifecycle + * events (`run:start`, `run:stop`) where there is no surrounding async unit + * of work to time. `populate` runs synchronously to attach meta/extras + * before emit. + */ +export function emitOneShot( + opts: WideEventOptions & { + op: string; + kind?: string; + traceparent?: string; + populate?: (state: State) => void; + } +): void { + if (!opts.enabled) return; + const state = newState({ + service: opts.service, + env: opts.env, + traceparent: opts.traceparent, + op: opts.op, + kind: opts.kind, + }); + if (opts.populate) opts.populate(state); + state.ok = true; + emit(state); +} + +/** Convenience accessor for in-handler meta mutation. */ +export function setMeta(state: State | null, key: string, value: string): void { + if (!state) return; + state.meta[key] = value; +} + +/** Convenience for free-form fields (did_warm_start, dispatch.result, ...). */ +export function setExtra(state: State | null, key: string, value: unknown): void { + if (!state) return; + state.extras[key] = value; +} diff --git a/apps/supervisor/src/wideEvents/new.test.ts b/apps/supervisor/src/wideEvents/new.test.ts new file mode 100644 index 0000000000..476c49c3d0 --- /dev/null +++ b/apps/supervisor/src/wideEvents/new.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from "vitest"; +import { isValidRequestId, newState } from "./new.js"; + +describe("isValidRequestId", () => { + it("accepts visible ASCII", () => { + expect(isValidRequestId("req-abc-123_456.7")).toBe(true); + expect(isValidRequestId("a")).toBe(true); + }); + + it("rejects empty string", () => { + expect(isValidRequestId("")).toBe(false); + }); + + it("rejects overlong strings (>128 bytes)", () => { + expect(isValidRequestId("a".repeat(128))).toBe(true); + expect(isValidRequestId("a".repeat(129))).toBe(false); + }); + + it("rejects whitespace, newlines, control chars", () => { + expect(isValidRequestId("has space")).toBe(false); + expect(isValidRequestId("has\ttab")).toBe(false); + expect(isValidRequestId("has\nnewline")).toBe(false); + expect(isValidRequestId("\x00null")).toBe(false); + }); + + it("rejects high-bit / non-ASCII", () => { + expect(isValidRequestId("cafรฉ")).toBe(false); + expect(isValidRequestId("a\x7f")).toBe(false); + }); +}); + +describe("newState", () => { + const env = { version: "1.0.0", commitSha: "abc123", region: "us-east-1", nodeId: "node-1" }; + + it("populates service identity from env", () => { + const s = newState({ service: "supervisor", env }); + expect(s.service).toBe("supervisor"); + expect(s.version).toBe("1.0.0"); + expect(s.commitSha).toBe("abc123"); + expect(s.region).toBe("us-east-1"); + expect(s.nodeId).toBe("node-1"); + }); + + it("mints a fresh request id when none provided", () => { + const s = newState({ service: "test", env: {} }); + expect(s.requestId).toMatch(/^req-[0-9a-f]{32}$/); + }); + + it("honours a valid inbound request id", () => { + const s = newState({ service: "test", env: {}, inboundRequestId: "trace-abc-123" }); + expect(s.requestId).toBe("trace-abc-123"); + }); + + it("rejects unsafe inbound request id and mints a fresh one", () => { + const s = newState({ service: "test", env: {}, inboundRequestId: "has space" }); + expect(s.requestId).toMatch(/^req-[0-9a-f]{32}$/); + }); + + it("parses traceparent into traceId and preserves the raw header", () => { + const tp = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + const s = newState({ service: "test", env: {}, traceparent: tp }); + expect(s.traceId).toBe("4bf92f3577b34da6a3ce929d0e0e4736"); + expect(s.traceparent).toBe(tp); + }); + + it("leaves traceId empty when no traceparent provided", () => { + const s = newState({ service: "test", env: {} }); + expect(s.traceId).toBe(""); + expect(s.traceparent).toBe(""); + }); + + it("initialises empty meta/extras/phases", () => { + const s = newState({ service: "test", env: {} }); + expect(s.meta).toEqual({}); + expect(s.extras).toEqual({}); + expect(s.phases).toEqual([]); + expect(s.ok).toBe(false); + expect(s.statusCode).toBe(0); + expect(s.durationMs).toBe(0); + }); +}); diff --git a/apps/supervisor/src/wideEvents/new.ts b/apps/supervisor/src/wideEvents/new.ts new file mode 100644 index 0000000000..7a4dba8a09 --- /dev/null +++ b/apps/supervisor/src/wideEvents/new.ts @@ -0,0 +1,96 @@ +import { randomBytes } from "node:crypto"; +import { parseTraceId } from "./traceparent.js"; +import type { State } from "./state.js"; + +const MAX_REQUEST_ID_LEN = 128; + +/** + * Validates an inbound request id. Non-empty, no longer than 128 bytes, + * composed entirely of visible ASCII (0x21..0x7E). Rejects newlines, control + * characters, whitespace, DEL, high-bit bytes - any of which could poison the + * log pipeline if echoed back verbatim. + */ +export function isValidRequestId(s: string): boolean { + if (s.length === 0 || s.length > MAX_REQUEST_ID_LEN) return false; + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c < 0x21 || c > 0x7e) return false; + } + return true; +} + +/** + * Service-level identity that's constant for the lifetime of the process. + * Populated once at startup, copied into every State. + */ +export type Env = { + version?: string; + commitSha?: string; + region?: string; + nodeId?: string; +}; + +export type NewStateOptions = { + service: string; + env: Env; + /** Optional inbound request id (e.g. from `x-request-id`). If unsafe or absent, a fresh `req-` is minted. */ + inboundRequestId?: string; + /** Optional inbound W3C traceparent (HTTP header, queue message field). */ + traceparent?: string; + /** Operation discriminator. Dotted `noun.verb`. Defaults to empty (set later). */ + op?: string; + /** Event shape: `inbound` | `outbound` | `event` | `scheduled`. Defaults to empty. */ + kind?: string; +}; + +/** + * Builds a State for a wide-event lifecycle. + * + * - requestId: honours `inboundRequestId` if present and safe; otherwise + * mints a fresh `req-` id. + * - traceId: parsed from the provided traceparent (graceful empty if + * absent or malformed). + * - traceparent: preserved verbatim for downstream propagation. + */ +export function newState(opts: NewStateOptions): State { + const traceparent = opts.traceparent ?? ""; + const inbound = opts.inboundRequestId ?? ""; + const requestId = isValidRequestId(inbound) ? inbound : newRequestId(); + + return { + startTime: nowRfc3339(), + requestId, + traceId: parseTraceId(traceparent), + traceparent, + service: opts.service, + version: opts.env.version, + commitSha: opts.env.commitSha, + region: opts.env.region, + nodeId: opts.env.nodeId, + op: opts.op ?? "", + kind: opts.kind ?? "", + meta: {}, + phases: [], + ok: false, + statusCode: 0, + durationMs: 0, + extras: {}, + }; +} + +function newRequestId(): string { + return "req-" + randomBytes(16).toString("hex"); +} + +/** + * Current wall-clock time as an RFC3339 string with microsecond precision. + * `Date.toISOString()` only has millisecond resolution, which is too coarse to + * order multiple wide events emitted within the same millisecond. + * `performance.timeOrigin + performance.now()` gives a sub-millisecond wall-clock + * reading; we append the microsecond digits to the millisecond ISO string. + */ +function nowRfc3339(): string { + const ms = performance.timeOrigin + performance.now(); + const micros = Math.floor((ms % 1) * 1000); // microseconds within the millisecond (0..999) + return new Date(ms).toISOString().slice(0, -1) + String(micros).padStart(3, "0") + "Z"; +} diff --git a/apps/supervisor/src/wideEvents/record.test.ts b/apps/supervisor/src/wideEvents/record.test.ts new file mode 100644 index 0000000000..beeb0fff22 --- /dev/null +++ b/apps/supervisor/src/wideEvents/record.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from "vitest"; +import { fromContext, wideEventStorage } from "./context.js"; +import { recordPhase, recordPhaseSince, timePhase } from "./record.js"; +import { newState } from "./new.js"; +import type { State } from "./state.js"; + +function makeState(): State { + return newState({ service: "test", env: {} }); +} + +describe("recordPhase", () => { + it("appends a successful phase", () => { + const s = makeState(); + recordPhase(s, "lookup", performance.now() - 50, undefined); + expect(s.phases).toHaveLength(1); + const phase = s.phases[0]; + if (!phase) throw new Error("missing phase"); + expect(phase.name).toBe("lookup"); + expect(phase.ok).toBe(true); + expect(phase.attempts).toBe(1); + expect(phase.durationMs).toBeGreaterThanOrEqual(45); + }); + + it("appends a failed phase with error code/message", () => { + const s = makeState(); + recordPhase(s, "dispatch", performance.now(), new Error("nope")); + const phase = s.phases[0]; + if (!phase) throw new Error("missing phase"); + expect(phase.ok).toBe(false); + expect(phase.errorCode).toBe("Error"); + expect(phase.errorMsg).toBe("nope"); + }); + + it("truncates very long error messages", () => { + const s = makeState(); + recordPhase(s, "x", performance.now(), new Error("y".repeat(2000))); + const phase = s.phases[0]; + if (!phase) throw new Error("missing phase"); + expect(phase.errorMsg?.length).toBe(512); + }); + + it("honours opts.attempts", () => { + const s = makeState(); + recordPhase(s, "retry", performance.now(), undefined, { attempts: 3 }); + expect(s.phases[0]?.attempts).toBe(3); + }); + + it("attaches sub-timings", () => { + const s = makeState(); + recordPhase(s, "complex", performance.now(), undefined, { sub: { setup_ms: 10, work_ms: 5 } }); + expect(s.phases[0]?.sub).toEqual({ setup_ms: 10, work_ms: 5 }); + }); + + it("is a no-op when state is null", () => { + expect(() => recordPhase(null, "x", performance.now(), undefined)).not.toThrow(); + }); +}); + +describe("timePhase + AsyncLocalStorage threading", () => { + it("records via fromContext on success", async () => { + const s = makeState(); + const value = await wideEventStorage.run(s, () => timePhase("work", async () => 42)); + expect(value).toBe(42); + expect(s.phases).toHaveLength(1); + expect(s.phases[0]?.ok).toBe(true); + }); + + it("records via fromContext on error and rethrows", async () => { + const s = makeState(); + await expect( + wideEventStorage.run(s, () => + timePhase("work", async () => { + throw new Error("boom"); + }) + ) + ).rejects.toThrow("boom"); + expect(s.phases).toHaveLength(1); + expect(s.phases[0]?.ok).toBe(false); + expect(s.phases[0]?.errorMsg).toBe("boom"); + }); + + it("runs fn unchanged when no state on context", async () => { + const value = await timePhase("work", async () => "ok"); + expect(value).toBe("ok"); + }); +}); + +describe("recordPhaseSince", () => { + it("records using a caller-captured start time", async () => { + const s = makeState(); + await wideEventStorage.run(s, async () => { + const start = performance.now(); + await new Promise((r) => setTimeout(r, 10)); + recordPhaseSince("spanning", start, undefined); + }); + expect(s.phases).toHaveLength(1); + expect(s.phases[0]?.durationMs).toBeGreaterThanOrEqual(8); + }); +}); + +describe("fromContext", () => { + it("returns null when no state attached", () => { + expect(fromContext()).toBe(null); + }); + + it("returns the state when inside wideEventStorage.run", () => { + const s = makeState(); + wideEventStorage.run(s, () => { + expect(fromContext()).toBe(s); + }); + }); +}); diff --git a/apps/supervisor/src/wideEvents/record.ts b/apps/supervisor/src/wideEvents/record.ts new file mode 100644 index 0000000000..b7b59089a0 --- /dev/null +++ b/apps/supervisor/src/wideEvents/record.ts @@ -0,0 +1,82 @@ +import { fromContext } from "./context.js"; +import type { PhaseRecord, State } from "./state.js"; +import { truncateUtf8 } from "./truncate.js"; + +const MAX_ERROR_MSG_BYTES = 512; + +/** Optional knobs for a phase record. */ +export type PhaseOpt = { + /** Attempt count for the phase (default 1). */ + attempts?: number; + /** Sub-timings to fold into `phase..`. */ + sub?: Record; +}; + +/** + * Appends a phase outcome to `state.phases`. Safe to call on success + * (`err === undefined`) and error paths. `errorMsg` is truncated to 512 bytes + * to keep the wide event compact. No-op if state is null. + */ +export function recordPhase( + state: State | null, + name: string, + startMs: number, + err: Error | undefined, + opts: PhaseOpt = {} +): void { + if (!state) return; + const p: PhaseRecord = { + name, + durationMs: Math.round(performance.now() - startMs), + ok: err === undefined, + attempts: opts.attempts ?? 1, + }; + if (err) { + p.errorCode = err.name || "Error"; + p.errorMsg = truncateUtf8(err.message, MAX_ERROR_MSG_BYTES); + } + if (opts.sub) p.sub = opts.sub; + state.phases.push(p); +} + +/** + * Runs `fn` and appends a phase outcome to the State attached to the current + * async context. If no state is on context (test paths, background work), + * `fn` runs unchanged. The phase is recorded on both success and error paths + * so failed phases still appear in the wide event with duration_ms + + * error_code. + */ +export async function timePhase( + name: string, + fn: () => Promise | T, + opts: PhaseOpt = {} +): Promise { + const start = performance.now(); + try { + const result = await fn(); + recordPhase(fromContext(), name, start, undefined, opts); + return result; + } catch (err) { + recordPhase(fromContext(), name, start, asError(err), opts); + throw err; + } +} + +/** + * Appends a phase outcome to the State attached to the current async context + * using a `startMs` captured by the caller. Use when the phase boundary spans + * multiple calls with intermediate error handling that can't fit inside a + * single `timePhase` closure. Nil-state safe. + */ +export function recordPhaseSince( + name: string, + startMs: number, + err: Error | undefined, + opts: PhaseOpt = {} +): void { + recordPhase(fromContext(), name, startMs, err, opts); +} + +function asError(e: unknown): Error { + return e instanceof Error ? e : new Error(String(e)); +} diff --git a/apps/supervisor/src/wideEvents/state.ts b/apps/supervisor/src/wideEvents/state.ts new file mode 100644 index 0000000000..dece3a3f5f --- /dev/null +++ b/apps/supervisor/src/wideEvents/state.ts @@ -0,0 +1,84 @@ +/** + * Per-event accumulator backing a single wide event. The supervisor emits one + * flat-keyed JSON line per natural unit of work (dequeue iteration, HTTP + * request, socket lifecycle event). Optional fields are omitted on emit so + * events stay compact. + */ +export type State = { + /** + * Wall-clock time the event began, as an ISO-8601 string. Emitted as + * `start_time` so log collection orders events by when work started rather + * than by the collector's ingestion time. + */ + startTime?: string; + + // Cross-stack correlation. + requestId: string; + traceId: string; + /** + * Raw inbound W3C `traceparent`, preserved verbatim so outbound calls can + * propagate the same trace context without losing the parent span-id. + * Empty when no inbound traceparent was set. + */ + traceparent: string; + + // Service identity (set by `newState` from Env). + service: string; + version?: string; + commitSha?: string; + region?: string; + nodeId?: string; + + /** + * Operation discriminator. Dotted `noun.verb` (e.g. `instance.create`, + * `snapshot.dispatch`). Low cardinality - bounded set per service, not + * unbounded. Empty allowed during construction but expected to be set + * before emit. + */ + op: string; + + /** + * Event shape. `inbound` for received requests, `outbound` for outgoing + * calls, `event` for ambient occurrences with no meaningful duration, + * `scheduled` for timer-driven work. Empty allowed; omitted from emit + * when empty. + */ + kind: string; + + // Caller-attached opaque metadata, flattened to `meta.` on emit. + meta: Record; + + // Per-phase outcomes, in completion order. + phases: PhaseRecord[]; + + // Top-level outcome (set after the wrapped operation returns). + ok: boolean; + statusCode: number; + durationMs: number; + error?: ErrorInfo; + + // Free-form ad-hoc additions (route, method, did_warm_start, ...). + extras: Record; +}; + +/** + * Single named phase outcome. Retries collapse into `attempts > 1` with the + * last error reflected in errorCode/errorMsg. + */ +export type PhaseRecord = { + name: string; + durationMs: number; + ok: boolean; + attempts: number; + errorCode?: string; + errorMsg?: string; + sub?: Record; +}; + +/** Top-level error summary for a failed operation. */ +export type ErrorInfo = { + code: string; + message: string; + /** Coarse classification - "client" | "upstream" | "internal" | "timeout". */ + kind: string; +}; diff --git a/apps/supervisor/src/wideEvents/traceparent.test.ts b/apps/supervisor/src/wideEvents/traceparent.test.ts new file mode 100644 index 0000000000..85ed31c3b6 --- /dev/null +++ b/apps/supervisor/src/wideEvents/traceparent.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest"; +import { parseTraceId } from "./traceparent.js"; + +describe("parseTraceId", () => { + const validTraceId = "4bf92f3577b34da6a3ce929d0e0e4736"; + const validHeader = `00-${validTraceId}-00f067aa0ba902b7-01`; + + it("extracts the trace-id from a valid W3C traceparent", () => { + expect(parseTraceId(validHeader)).toBe(validTraceId); + }); + + it("returns empty string for empty/null/undefined input", () => { + expect(parseTraceId("")).toBe(""); + expect(parseTraceId(null)).toBe(""); + expect(parseTraceId(undefined)).toBe(""); + }); + + it("returns empty for wrong segment count", () => { + expect(parseTraceId("00-abc-def")).toBe(""); + expect(parseTraceId("00-abc-def-01-extra")).toBe(""); + }); + + it("returns empty for non-zero version byte", () => { + expect(parseTraceId(`01-${validTraceId}-00f067aa0ba902b7-01`)).toBe(""); + }); + + it("returns empty for wrong-length trace-id", () => { + expect(parseTraceId("00-abc-00f067aa0ba902b7-01")).toBe(""); + }); + + it("returns empty for non-hex trace-id", () => { + expect(parseTraceId("00-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-00f067aa0ba902b7-01")).toBe(""); + }); + + it("returns empty for all-zero trace-id", () => { + expect(parseTraceId("00-00000000000000000000000000000000-00f067aa0ba902b7-01")).toBe(""); + }); + + it("accepts uppercase hex", () => { + const tid = "4BF92F3577B34DA6A3CE929D0E0E4736"; + expect(parseTraceId(`00-${tid}-00f067aa0ba902b7-01`)).toBe(tid); + }); +}); diff --git a/apps/supervisor/src/wideEvents/traceparent.ts b/apps/supervisor/src/wideEvents/traceparent.ts new file mode 100644 index 0000000000..9e84294f06 --- /dev/null +++ b/apps/supervisor/src/wideEvents/traceparent.ts @@ -0,0 +1,39 @@ +/** + * Extracts the trace-id from a W3C `traceparent` header. Returns "" when the + * header is absent, malformed, or carries an all-zero trace-id. + * + * Format: `---` + * version : 2 hex chars, must be "00" + * trace-id: 32 hex chars, non-zero + * span-id : 16 hex chars (not validated - we only need trace-id) + * flags : 2 hex chars (not validated) + */ +export function parseTraceId(header: string | null | undefined): string { + if (!header) return ""; + const parts = header.split("-"); + if (parts.length !== 4) return ""; + if (parts[0] !== "00") return ""; + const tid = parts[1]; + if (!tid || tid.length !== 32) return ""; + if (!isHex(tid)) return ""; + if (isAllZero(tid)) return ""; + return tid; +} + +function isHex(s: string): boolean { + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + const isDigit = c >= 0x30 && c <= 0x39; + const isLower = c >= 0x61 && c <= 0x66; + const isUpper = c >= 0x41 && c <= 0x46; + if (!isDigit && !isLower && !isUpper) return false; + } + return true; +} + +function isAllZero(s: string): boolean { + for (let i = 0; i < s.length; i++) { + if (s.charCodeAt(i) !== 0x30) return false; + } + return true; +} diff --git a/apps/supervisor/src/wideEvents/truncate.test.ts b/apps/supervisor/src/wideEvents/truncate.test.ts new file mode 100644 index 0000000000..4eb272f1e6 --- /dev/null +++ b/apps/supervisor/src/wideEvents/truncate.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from "vitest"; +import { truncateUtf8 } from "./truncate.js"; + +describe("truncateUtf8", () => { + it("returns short ASCII unchanged", () => { + expect(truncateUtf8("hello", 512)).toBe("hello"); + }); + + it("truncates ASCII to the byte cap", () => { + expect(truncateUtf8("x".repeat(1024), 256)).toBe("x".repeat(256)); + }); + + it("never exceeds the byte cap for multibyte input", () => { + // "ใ‚" is 3 UTF-8 bytes; 200 of them = 600 bytes. + const got = truncateUtf8("ใ‚".repeat(200), 256); + expect(Buffer.byteLength(got, "utf8")).toBeLessThanOrEqual(256); + }); + + it("does not split a multibyte sequence", () => { + // 256 / 3 bytes = 85 whole chars (255 bytes), the 86th would overflow. + expect(truncateUtf8("ใ‚".repeat(200), 256)).toBe("ใ‚".repeat(85)); + }); + + it("does not split a surrogate pair", () => { + // "๐Ÿ˜€" is 2 UTF-16 units / 4 UTF-8 bytes; only one fits under a 5-byte cap. + const got = truncateUtf8("๐Ÿ˜€๐Ÿ˜€", 5); + expect(got).toBe("๐Ÿ˜€"); + expect(Buffer.byteLength(got, "utf8")).toBe(4); + }); +}); diff --git a/apps/supervisor/src/wideEvents/truncate.ts b/apps/supervisor/src/wideEvents/truncate.ts new file mode 100644 index 0000000000..417735fe77 --- /dev/null +++ b/apps/supervisor/src/wideEvents/truncate.ts @@ -0,0 +1,20 @@ +/** + * Truncate `value` to at most `maxBytes` UTF-8 bytes without splitting a + * multi-byte sequence or surrogate pair. Plain `.slice()` counts UTF-16 code + * units, so multibyte text can blow past a byte cap and cutting mid-pair + * leaves a lone surrogate that downstream JSON / Postgres consumers reject. + */ +export function truncateUtf8(value: string, maxBytes: number): string { + if (Buffer.byteLength(value, "utf8") <= maxBytes) return value; + + let bytes = 0; + let end = 0; + // `for..of` yields whole code points, so a surrogate pair is never split. + for (const ch of value) { + const size = Buffer.byteLength(ch, "utf8"); + if (bytes + size > maxBytes) break; + bytes += size; + end += ch.length; + } + return value.slice(0, end); +} diff --git a/apps/supervisor/src/workloadManager/compute.test.ts b/apps/supervisor/src/workloadManager/compute.test.ts new file mode 100644 index 0000000000..95a9ca2f3b --- /dev/null +++ b/apps/supervisor/src/workloadManager/compute.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { ComputeClientError } from "@internal/compute"; +import { isRetryableCreateError, runnerNameForAttempt } from "./compute.js"; + +describe("runnerNameForAttempt", () => { + it("keeps the unsuffixed name for the first attempt", () => { + expect(runnerNameForAttempt("runner-abc123", 1)).toBe("runner-abc123"); + }); + + it("suffixes retry attempts deterministically", () => { + expect(runnerNameForAttempt("runner-abc123", 2)).toBe("runner-abc123-r2"); + expect(runnerNameForAttempt("runner-abc123", 3)).toBe("runner-abc123-r3"); + }); +}); + +describe("isRetryableCreateError", () => { + it("retries statuses where the create definitely did not commit", () => { + expect(isRetryableCreateError(new ComputeClientError(500, "tap busy", "http://gw"))).toBe(true); + expect(isRetryableCreateError(new ComputeClientError(503, "no placement", "http://gw"))).toBe( + true + ); + }); + + it("does not retry lost-response statuses (create may have committed)", () => { + expect(isRetryableCreateError(new ComputeClientError(502, "bad gateway", "http://gw"))).toBe( + false + ); + expect( + isRetryableCreateError(new ComputeClientError(504, "gateway timeout", "http://gw")) + ).toBe(false); + }); + + it("does not retry 4xx responses", () => { + expect(isRetryableCreateError(new ComputeClientError(400, "bad request", "http://gw"))).toBe( + false + ); + expect(isRetryableCreateError(new ComputeClientError(409, "conflict", "http://gw"))).toBe( + false + ); + }); + + it("does not retry timeouts (instance may still be provisioning)", () => { + expect(isRetryableCreateError(new DOMException("timed out", "TimeoutError"))).toBe(false); + }); + + it("retries network-level fetch failures", () => { + expect(isRetryableCreateError(new TypeError("fetch failed"))).toBe(true); + }); + + it("does not retry unknown errors", () => { + expect(isRetryableCreateError(new Error("something else"))).toBe(false); + expect(isRetryableCreateError("string error")).toBe(false); + }); +}); diff --git a/apps/supervisor/src/workloadManager/compute.ts b/apps/supervisor/src/workloadManager/compute.ts new file mode 100644 index 0000000000..89360437dd --- /dev/null +++ b/apps/supervisor/src/workloadManager/compute.ts @@ -0,0 +1,513 @@ +import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; +import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic"; +import { flattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes"; +import { + type WorkloadManager, + type WorkloadManagerCreateOptions, + type WorkloadManagerOptions, +} from "./types.js"; +import { ComputeClient, ComputeClientError, stripImageDigest } from "@internal/compute"; +import { setTimeout as sleep } from "node:timers/promises"; +import { extractTraceparent, getRunnerId } from "../util.js"; +import type { OtlpTraceService } from "../services/otlpTraceService.js"; +import { tryCatch } from "@trigger.dev/core"; +import { encodeBaggage, fromContext } from "../wideEvents/index.js"; + +const DEFAULT_CREATE_MAX_ATTEMPTS = 3; +const DEFAULT_CREATE_RETRY_BASE_DELAY_MS = 250; + +/** + * TEMPORARY (TRI-10293): a failed create can leave its instance name + * registered gateway/fcrun-side until async cleanup runs, so a same-name + * retry can 409 against our own residue. Until the gateway cleans up + * failed-create registrations properly, retry attempts get a deterministic + * suffix. Attempt 1 keeps the unsuffixed name so the non-retry path is + * unchanged; the suffixed name flows into both the instance name and + * TRIGGER_RUNNER_ID, which downstream flows treat as one opaque + * self-reported token. Only attempts following a ComputeClientError are + * suffixed - network-failure retries keep the same name on purpose, because + * the gateway's name-collision 409 is their safety net against + * double-creating an instance whose create response was lost. + */ +export function runnerNameForAttempt(runnerId: string, attempt: number): string { + return attempt === 1 ? runnerId : `${runnerId}-r${attempt}`; +} + +/** + * Whether a failed instance create is worth retrying. Only statuses where + * the create definitely did NOT commit are retried: 500 means the agent or + * fcrun returned a create error (e.g. a netns slot holding the tap busy, a + * full node disk - placement may differ on retry), 503 means the gateway + * had nowhere to place it. 502/504 are excluded: the gateway emits those + * when it fails to reach the node or read its response, which can happen + * AFTER the agent committed the create - and the gateway only records the + * instance name on a clean 201, so a same-name retry would miss the + * collision check and could double-create the VM on another node. 4xx won't + * heal on retry, and timeouts may still be provisioning. Network-level + * fetch failures are safe: if the gateway processed the create, its name + * index is populated and the retry 409s harmlessly. + */ +export function isRetryableCreateError(error: unknown): boolean { + if (error instanceof ComputeClientError) { + return error.status === 500 || error.status === 503; + } + if (error instanceof DOMException && error.name === "TimeoutError") { + return false; + } + // Network-level fetch failures (gateway briefly unreachable) + return error instanceof TypeError; +} + +type ComputeWorkloadManagerOptions = WorkloadManagerOptions & { + gateway: { + url: string; + authToken?: string; + timeoutMs: number; + }; + snapshots: { + enabled: boolean; + delayMs: number; + dispatchLimit: number; + callbackUrl: string; + }; + tracing?: OtlpTraceService; + runner: { + instanceName: string; + otelEndpoint: string; + prettyLogs: boolean; + sendRunDebugLogs: boolean; + }; + createRetry?: { + maxAttempts: number; + baseDelayMs: number; + }; +}; + +export class ComputeWorkloadManager implements WorkloadManager { + private readonly logger = new SimpleStructuredLogger("compute-workload-manager"); + private readonly compute: ComputeClient; + private readonly createMaxAttempts: number; + private readonly createRetryBaseDelayMs: number; + + constructor(private opts: ComputeWorkloadManagerOptions) { + this.createMaxAttempts = opts.createRetry?.maxAttempts ?? DEFAULT_CREATE_MAX_ATTEMPTS; + this.createRetryBaseDelayMs = + opts.createRetry?.baseDelayMs ?? DEFAULT_CREATE_RETRY_BASE_DELAY_MS; + + if (opts.workloadApiDomain) { + this.logger.warn("โš ๏ธ Custom workload API domain", { + domain: opts.workloadApiDomain, + }); + } + + this.compute = new ComputeClient({ + gatewayUrl: opts.gateway.url, + authToken: opts.gateway.authToken, + timeoutMs: opts.gateway.timeoutMs, + // Forward the current wide-event scope's traceparent + request_id so the + // downstream service continues the same trace and joins its own wide + // events to ours. Additionally serialize caller-supplied meta labels + // into the W3C Baggage header so the downstream service auto-stamps + // them even on early-error paths that bail before parsing the body. + // When called outside a wide-event scope (or when wide events are + // disabled), `fromContext` returns undefined and propagation is skipped. + getPropagationHeaders: () => { + const state = fromContext(); + if (!state) return {}; + const headers: Record = { "x-request-id": state.requestId }; + if (state.traceparent) { + headers.traceparent = state.traceparent; + } + const baggage = encodeBaggage(state.meta); + if (baggage) { + headers.baggage = baggage; + } + return headers; + }, + }); + } + + get snapshotsEnabled(): boolean { + return this.opts.snapshots.enabled; + } + + get snapshotDelayMs(): number { + return this.opts.snapshots.delayMs; + } + + get snapshotDispatchLimit(): number { + return this.opts.snapshots.dispatchLimit; + } + + get traceSpansEnabled(): boolean { + return !!this.opts.tracing; + } + + async create(opts: WorkloadManagerCreateOptions) { + const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber); + + const envVars: Record = { + OTEL_EXPORTER_OTLP_ENDPOINT: this.opts.runner.otelEndpoint, + TRIGGER_DEQUEUED_AT_MS: String(opts.dequeuedAt.getTime()), + TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()), + TRIGGER_ENV_ID: opts.envId, + TRIGGER_DEPLOYMENT_ID: opts.deploymentToken ?? opts.deploymentFriendlyId, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: opts.deploymentFriendlyId, + TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion, + TRIGGER_RUN_ID: opts.runFriendlyId, + TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId, + TRIGGER_SUPERVISOR_API_PROTOCOL: this.opts.workloadApiProtocol, + TRIGGER_SUPERVISOR_API_PORT: String(this.opts.workloadApiPort), + TRIGGER_SUPERVISOR_API_DOMAIN: this.opts.workloadApiDomain ?? "", + TRIGGER_WORKER_INSTANCE_NAME: this.opts.runner.instanceName, + TRIGGER_RUNNER_ID: runnerId, + TRIGGER_MACHINE_CPU: String(opts.machine.cpu), + TRIGGER_MACHINE_MEMORY: String(opts.machine.memory), + PRETTY_LOGS: String(this.opts.runner.prettyLogs), + TRIGGER_SEND_RUN_DEBUG_LOGS: String(this.opts.runner.sendRunDebugLogs), + }; + + if (this.opts.warmStartUrl) { + envVars.TRIGGER_WARM_START_URL = this.opts.warmStartUrl; + } + + if (this.snapshotsEnabled && this.opts.metadataUrl) { + envVars.TRIGGER_METADATA_URL = this.opts.metadataUrl; + } + + if (this.opts.heartbeatIntervalSeconds) { + envVars.TRIGGER_HEARTBEAT_INTERVAL_SECONDS = String(this.opts.heartbeatIntervalSeconds); + } + + if (this.opts.snapshotPollIntervalSeconds) { + envVars.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS = String( + this.opts.snapshotPollIntervalSeconds + ); + } + + if (this.opts.additionalEnvVars) { + Object.assign(envVars, this.opts.additionalEnvVars); + } + + // Strip image digest - resolve by tag, not digest + const imageRef = stripImageDigest(opts.image); + + // Labels forwarded to the compute provider for network-policy selection. + // `org` is always set so every run carries its org identity. + const labels: Record = { + org: opts.orgId, + }; + + // Wide event: single canonical log line emitted in finally + const event: Record = { + // High-cardinality identifiers + runId: opts.runFriendlyId, + runnerId, + envId: opts.envId, + envType: opts.envType, + orgId: opts.orgId, + projectId: opts.projectId, + deploymentVersion: opts.deploymentVersion, + machine: opts.machine.name, + // Environment + instanceName: this.opts.runner.instanceName, + // Supervisor timing + dequeueResponseMs: opts.dequeueResponseMs, + pollingIntervalMs: opts.pollingIntervalMs, + warmStartCheckMs: opts.warmStartCheckMs, + // Request + image: imageRef, + }; + + const startMs = performance.now(); + + try { + const createRequest = { + name: runnerId, + image: imageRef, + env: envVars, + cpu: opts.machine.cpu, + memory_gb: opts.machine.memory, + metadata: { + runId: opts.runFriendlyId, + envId: opts.envId, + envType: opts.envType, + orgId: opts.orgId, + projectId: opts.projectId, + deploymentVersion: opts.deploymentVersion, + machine: opts.machine.name, + }, + ...(Object.keys(labels).length > 0 ? { labels } : {}), + }; + + // Retry transient placement failures instead of abandoning the run: a + // swallowed create error leaves the run waiting for the run engine's + // PENDING_EXECUTING timeout (minutes) before it is redriven, while a + // retried create typically succeeds in under a second (TRI-10293). + let error: unknown; + let data: Awaited> | null | undefined; + let attempt = 1; + // Set after a ComputeClientError: the failed create may have left its + // name registered, so subsequent attempts use a suffixed name. + let suffixAttempts = false; + for (; attempt <= this.createMaxAttempts; attempt++) { + const attemptRunnerId = suffixAttempts ? runnerNameForAttempt(runnerId, attempt) : runnerId; + [error, data] = await tryCatch( + this.compute.instances.create( + attemptRunnerId === runnerId + ? createRequest + : { + ...createRequest, + name: attemptRunnerId, + env: { ...envVars, TRIGGER_RUNNER_ID: attemptRunnerId }, + } + ) + ); + + if (!error) { + event.runnerId = attemptRunnerId; + break; + } + + if (error instanceof ComputeClientError) { + suffixAttempts = true; + } + + this.logger.warn("create instance attempt failed", { + runnerId: attemptRunnerId, + attempt, + error: error instanceof Error ? error.message : String(error), + }); + + if (!isRetryableCreateError(error) || attempt === this.createMaxAttempts) break; + await sleep(this.createRetryBaseDelayMs * attempt); + } + event.createAttempts = attempt; + + if (error || !data) { + event.error = error instanceof Error ? error.message : String(error); + event.errorType = + error instanceof DOMException && error.name === "TimeoutError" ? "timeout" : "fetch"; + // Intentional: errors are captured in the wide event, not thrown. This matches + // the Docker/K8s managers. The run will eventually time out if scheduling fails. + return; + } + + event.instanceId = data.id; + event.ok = true; + + // Parse timing data from compute response (optional - requires gateway timing flag) + if (data._timing) { + event.timing = data._timing; + } + + this.#emitProvisionSpan(opts, startMs, data._timing); + } finally { + event.durationMs = Math.round(performance.now() - startMs); + event.ok ??= false; + this.logger.debug("create instance", event); + } + } + + async snapshot(opts: { runnerId: string; metadata: Record }): Promise { + const [error] = await tryCatch( + this.compute.instances.snapshot(opts.runnerId, { + callback: { + url: this.opts.snapshots.callbackUrl, + metadata: opts.metadata, + }, + }) + ); + + if (error) { + this.logger.error("snapshot request failed", { + runnerId: opts.runnerId, + error: error instanceof Error ? error.message : String(error), + }); + return false; + } + + this.logger.debug("snapshot request accepted", { runnerId: opts.runnerId }); + return true; + } + + async deleteInstance(runnerId: string): Promise { + const [error] = await tryCatch(this.compute.instances.delete(runnerId)); + + if (error) { + this.logger.error("delete instance failed", { + runnerId, + error: error instanceof Error ? error.message : String(error), + }); + return false; + } + + this.logger.debug("delete instance success", { runnerId }); + return true; + } + + #emitProvisionSpan(opts: WorkloadManagerCreateOptions, startMs: number, timing?: unknown) { + if (!this.traceSpansEnabled) return; + + const parsed = parseTraceparent(extractTraceparent(opts.traceContext)); + if (!parsed) return; + + const endMs = performance.now(); + const now = Date.now(); + const provisionStartEpochMs = now - (endMs - startMs); + const endEpochMs = now; + + // Span starts at dequeue time so events (dequeue) render in the thin-line section + // before "Started". The actual provision call time is in provisionStartEpochMs. + // Subtract 1ms so compute span always sorts before the attempt span (same dequeue time) + const startEpochMs = opts.dequeuedAt.getTime() - 1; + + const spanAttributes: Record = { + "compute.type": "create", + "compute.provision_start_ms": provisionStartEpochMs, + ...(timing + ? (flattenAttributes(timing, "compute") as Record) + : {}), + }; + + if (opts.dequeueResponseMs !== undefined) { + spanAttributes["supervisor.dequeue_response_ms"] = opts.dequeueResponseMs; + } + if (opts.warmStartCheckMs !== undefined) { + spanAttributes["supervisor.warm_start_check_ms"] = opts.warmStartCheckMs; + } + + // Use the platform API URL, not the runner OTLP endpoint (which may be a VM gateway IP) + this.opts.tracing?.emit({ + traceId: parsed.traceId, + parentSpanId: parsed.spanId, + spanName: "compute.provision", + startTimeMs: startEpochMs, + endTimeMs: endEpochMs, + resourceAttributes: { + "ctx.environment.id": opts.envId, + "ctx.organization.id": opts.orgId, + "ctx.project.id": opts.projectId, + "ctx.run.id": opts.runFriendlyId, + }, + spanAttributes, + }); + } + + async restore(opts: { + snapshotId: string; + runnerId: string; + runFriendlyId: string; + snapshotFriendlyId: string; + machine: { cpu: number; memory: number }; + // Trace context for OTel span emission + traceContext?: Record; + envId?: string; + orgId?: string; + projectId?: string; + hasPrivateLink?: boolean; + dequeuedAt?: Date; + }): Promise { + const metadata: Record = { + TRIGGER_RUNNER_ID: opts.runnerId, + TRIGGER_RUN_ID: opts.runFriendlyId, + TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId, + TRIGGER_SUPERVISOR_API_PROTOCOL: this.opts.workloadApiProtocol, + TRIGGER_SUPERVISOR_API_PORT: String(this.opts.workloadApiPort), + TRIGGER_SUPERVISOR_API_DOMAIN: this.opts.workloadApiDomain ?? "", + TRIGGER_WORKER_INSTANCE_NAME: this.opts.runner.instanceName, + }; + + // Resupply labels on restore (the provider doesn't persist them across a + // snapshot). orgId is optional on the restore opts type, so guard it. + const labels: Record = {}; + if (opts.orgId) { + labels.org = opts.orgId; + } + + this.logger.verbose("restore request body", { + snapshotId: opts.snapshotId, + runnerId: opts.runnerId, + }); + + const startMs = performance.now(); + + const [error] = await tryCatch( + this.compute.snapshots.restore(opts.snapshotId, { + name: opts.runnerId, + metadata, + cpu: opts.machine.cpu, + memory_gb: opts.machine.memory, + ...(Object.keys(labels).length > 0 ? { labels } : {}), + }) + ); + + const durationMs = Math.round(performance.now() - startMs); + + if (error) { + this.logger.error("restore request failed", { + snapshotId: opts.snapshotId, + runnerId: opts.runnerId, + error: error instanceof Error ? error.message : String(error), + durationMs, + }); + return false; + } + + this.logger.debug("restore request success", { + snapshotId: opts.snapshotId, + runnerId: opts.runnerId, + durationMs, + }); + + this.#emitRestoreSpan(opts, startMs); + + return true; + } + + #emitRestoreSpan( + opts: { + snapshotId: string; + runnerId: string; + runFriendlyId: string; + traceContext?: Record; + envId?: string; + orgId?: string; + projectId?: string; + dequeuedAt?: Date; + }, + startMs: number + ) { + if (!this.traceSpansEnabled) return; + + const parsed = parseTraceparent(extractTraceparent(opts.traceContext)); + if (!parsed || !opts.envId || !opts.orgId || !opts.projectId) return; + + const endMs = performance.now(); + const now = Date.now(); + const restoreStartEpochMs = now - (endMs - startMs); + const endEpochMs = now; + + // Subtract 1ms so restore span always sorts before the attempt span + const startEpochMs = (opts.dequeuedAt?.getTime() ?? restoreStartEpochMs) - 1; + + this.opts.tracing?.emit({ + traceId: parsed.traceId, + parentSpanId: parsed.spanId, + spanName: "compute.restore", + startTimeMs: startEpochMs, + endTimeMs: endEpochMs, + resourceAttributes: { + "ctx.environment.id": opts.envId, + "ctx.organization.id": opts.orgId, + "ctx.project.id": opts.projectId, + "ctx.run.id": opts.runFriendlyId, + }, + spanAttributes: { + "compute.type": "restore", + "compute.snapshot_id": opts.snapshotId, + }, + }); + } +} diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts index d6651d325a..c8049873b6 100644 --- a/apps/supervisor/src/workloadManager/docker.ts +++ b/apps/supervisor/src/workloadManager/docker.ts @@ -62,7 +62,7 @@ export class DockerWorkloadManager implements WorkloadManager { } async create(opts: WorkloadManagerCreateOptions) { - this.logger.log("create()", { opts }); + this.logger.verbose("create()", { opts }); const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber); @@ -72,7 +72,9 @@ export class DockerWorkloadManager implements WorkloadManager { `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`, `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, `TRIGGER_ENV_ID=${opts.envId}`, - `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`, + `TRIGGER_DEPLOYMENT_ID=${opts.deploymentToken ?? opts.deploymentFriendlyId}`, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + `TRIGGER_DEPLOYMENT_FRIENDLY_ID=${opts.deploymentFriendlyId}`, `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`, `TRIGGER_RUN_ID=${opts.runFriendlyId}`, `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`, @@ -84,6 +86,7 @@ export class DockerWorkloadManager implements WorkloadManager { `TRIGGER_MACHINE_CPU=${opts.machine.cpu}`, `TRIGGER_MACHINE_MEMORY=${opts.machine.memory}`, `PRETTY_LOGS=${env.RUNNER_PRETTY_LOGS}`, + `TRIGGER_SEND_RUN_DEBUG_LOGS=${env.SEND_RUN_DEBUG_LOGS}`, ]; if (this.opts.warmStartUrl) { diff --git a/apps/supervisor/src/workloadManager/ecrAuth.ts b/apps/supervisor/src/workloadManager/ecrAuth.ts index 33e98f6319..851249898b 100644 --- a/apps/supervisor/src/workloadManager/ecrAuth.ts +++ b/apps/supervisor/src/workloadManager/ecrAuth.ts @@ -1,7 +1,7 @@ import { ECRClient, GetAuthorizationTokenCommand } from "@aws-sdk/client-ecr"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; import { tryCatch } from "@trigger.dev/core"; -import Docker from "dockerode"; +import type Docker from "dockerode"; interface ECRTokenCache { token: string; diff --git a/apps/supervisor/src/workloadManager/kubernetes.test.ts b/apps/supervisor/src/workloadManager/kubernetes.test.ts new file mode 100644 index 0000000000..bb15c23e9f --- /dev/null +++ b/apps/supervisor/src/workloadManager/kubernetes.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { + BLOCK_IO_URING_SECCOMP_PROFILE, + nodetypeNodeSelector, + runPodTolerations, + withBlockIoUringSeccompProfile, +} from "./kubernetesPodSpec.js"; + +const basePodSpec = { + restartPolicy: "Never" as const, + automountServiceAccountToken: false, + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + fsGroup: 1000, + }, +}; + +describe("nodetypeNodeSelector", () => { + it("omits the nodeSelector entirely when the label is empty or unset", () => { + for (const label of ["", undefined]) { + expect(nodetypeNodeSelector(label)).toEqual({}); + } + }); + + it("pins to nodetype=