From 0e7f4904dece81431789147451f9e8cb313cdc92 Mon Sep 17 00:00:00 2001 From: imtahirabatool Date: Thu, 12 Nov 2026 20:59:34 +0500 Subject: [PATCH] engine: resume a run after a crash --- README.md | 21 ++- runner/scripts/engine.ts | 77 ++++++-- runner/src/engine/engine.ts | 283 +++++++++++++++++++---------- runner/src/engine/run-store.ts | 59 ++++++ runner/src/index.ts | 3 +- runner/src/runner.ts | 33 +++- runner/test/engine-recover.test.ts | 256 ++++++++++++++++++++++++++ runner/test/engine.test.ts | 12 +- runner/test/runner.test.ts | 31 +++- 9 files changed, 641 insertions(+), 134 deletions(-) create mode 100644 runner/src/engine/run-store.ts create mode 100644 runner/test/engine-recover.test.ts diff --git a/README.md b/README.md index 255c069..1f75e2f 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ When it ends: ## Status -Week 1 (Oct 19 to 23, 2026), plus the first part of week 2. +Through week 4 (Nov 2026): the runner, the sandbox, the engine and crash resume. | Piece | File | State | | --- | --- | --- | @@ -82,7 +82,9 @@ Week 1 (Oct 19 to 23, 2026), plus the first part of week 2. | `approval.requested` after `task.finished` | [runner/src/runner.ts](runner/src/runner.ts) | Done (contracts v0.2.0). Real and fake runners, never after a stop or a failure | | Repo memory: notes read before a run, a note added after | [runner/src/memory/](runner/src/memory/) | Done. `MemoryStore` with an in-memory stub and a file store. Stopped and failed runs add nothing, and a memory error never fails a run | | Repo profile: language, package manager, test command, Node version | [runner/src/memory/profile.ts](runner/src/memory/profile.ts) | Done. Read from files in the worktree, no model call. The sandbox uses it to pick the image and decide whether to install | -| Sandbox: run a repo's tests in a locked Docker box | [runner/src/runtime/](runner/src/runtime/) | Done. `runInSandbox()`: no network, 1 GB with no swap, 1 CPU, 256 processes, read-only repo and system, non-root, your timezone, 60 s. npm only. Nothing calls it yet: the engine's test step will | +| Sandbox: run a repo's tests in a locked Docker box | [runner/src/runtime/](runner/src/runtime/) | Done. `runInSandbox()`: no network, 1 GB with no swap, 1 CPU, 256 processes, read-only repo and system, non-root, your timezone, 60 s. npm only. The engine's test step calls it | +| Engine: analyze → plan → implement → test → review | [runner/src/engine/](runner/src/engine/) | Done. `createEngine()` is a `Runner` for `agent: "codehands"`. Claude Code does implement through the runner, the sandbox does test, and one model call each does analyze, plan and review. Once the change exists, nothing after it throws it away: failing tests or a failed review still end in `awaiting_approval` | +| Crash resume: kill the daemon, it finishes anyway | [runner/src/engine/run-store.ts](runner/src/engine/run-store.ts) | Done. The engine saves `run.json` before every event. After a restart, `engine.recover()` lists the runs a crash cut off, and `resume()` carries on from the first step that didn't finish: no step is paid for twice, Claude Code's own session is continued, `seq` never repeats, and `approve` waits for test and review | Next, from the [design](https://github.com/devweekends/codehands-docs#who-owns-what): a model per phase, naming provider errors as one `provider.status` event (`rate_limited`, `auth_expired`, `usage_limit`, `subscription_inactive`, `down`), memory and user preferences, and two models on one task. @@ -119,7 +121,10 @@ const { branch, commit } = await runner.approve(sessionId, "Fix the date bug"); | `createFakeRunner({ delayMs? })` | Emits 12 scripted events, 300 ms apart, then `approval.requested`, and supports every method including stop and resume. Costs nothing, needs no repo. Build against it, and keep it as the demo backup | | `EventSequence` | Hands out events with the next `seq` | | `SessionStore`, `SessionRecord` | Session records with the legal-move check from the contracts. Saved as files when given a root, otherwise in memory (the fake runner) | -| `RecoverableRunner`, `RecoveryReport` | What `createRunner` returns: the contract's `Runner` plus `recover()`, and what `recover()` reports | +| `RecoverableRunner`, `RecoveryReport`, `RecoverOptions` | What `createRunner` and `createEngine` return: the contract's `Runner` plus `recover()`, `worktree()` and `status()`. `recover({ keep })` keeps a running session its owner will continue as `stopped` instead of failing it, and lists it in `report.resumable` | +| `createEngine(options?)`, `EngineOptions`, `modelFor`, `DEFAULT_MODEL` | The engine. `runner`, `model`, `sandbox`, `memory` and `runsRoot` can each be swapped, which is how the tests run it for $0. `modelFor(o, step)`: the step's own model, then the run's, then the default | +| `RunStore`, `RunState` | The engine's `run.json`: each finished step's result, the run's status, and the last `seq` sent | +| `createClaudeModel()`, `createCannedModel(answers)`, `ModelClient` | Who answers analyze, plan and review: `claude -p` with no tools, or fixed text for tests and `--free` | | `branchName(prompt, sessionId)`, `slugify(text)` | `codehands/fix-the-failing-test-3f2a9c1e` | | `MemoryStore`, `createMemoryStub()`, `createFileMemory(root?)` | Repo memory: `notes`, `addNote`, `profile`, `saveProfile`. The stub keeps it in a `Map` for tests | | `formatNote`, `runNote`, `trimNotes`, `notesForPrompt` | How a note is written, built from a run, trimmed to about 200 lines, and capped at 6,000 characters for the prompt | @@ -135,6 +140,8 @@ The daemon owns the real database. The runner keeps only what it needs to finish When the daemon starts it should call `recover()` once, before anything else. Sessions still marked `queued` or `running` become `failed`, and their ids come back in `report.failed`. Failed sessions lose their worktree, along with anything uncommitted in it, because `failed` is final and nothing else would remove it. Their branch goes too, unless the agent committed to it. `awaiting_approval` and `stopped` sessions are left alone. A worktree that can't be removed comes back in `report.cleanupErrors` and is tried again next time. +**With the engine (Midhat):** on startup call `engine.recover()`, not `runner.recover()` (the engine calls the runner's itself), then `resume()` every id in `report.resumable`. Each resume is a normal event stream: store its events like any run's. + ### How Claude Code's output becomes events Each stdout line from `claude -p --output-format stream-json --verbose` is one JSON object. The parser in [claude-code.ts](runner/src/agents/claude-code.ts) maps it like this: @@ -168,6 +175,8 @@ Some details that matter: | `~/.codehands/memory//notes.md` | Per-repo notes, newest last, trimmed to about 200 lines by dropping the oldest whole notes | | `~/.codehands/memory//profile.json` | The repo profile from the latest run | | `~/.codehands/sessions/.json` | What the runner needs to finish a session after a restart | +| `~/.codehands/runs//run.json` | The engine's saved state: each finished step's result, the status and the last `seq`. Written before every event, through a temp file, so a crash never leaves half of it | +| `~/.codehands/runs//-.json` | One file per engine step: what went in, what came out, model, cost and time | | `~/.codehands/runs//test-.json` | Each sandbox test run: passed, exit code, timed out, out of memory, the last 64 KB of output | | Docker containers `ch--` | One per sandbox run, removed right after. `-install` on the end for the install box | | `codehands/-` | The task's branch, in your repo | @@ -219,6 +228,8 @@ Run these in `runner/`. | `pnpm sandbox [repoPath] [--tz ]` | Runs a repo's tests in the locked box once and prints the result. Defaults to the fixture. `--tz UTC` shows why the box needs your timezone | No | | `pnpm fake` | Prints the fake runner's events, one JSON object per line. This is exactly what the daemon receives | No | | `pnpm try [repoPath] [prompt] [--approve \| --reject]` | One real run through the runner, one line per event, then the diff stat, then approve, reject or keep | Yes, capped at $0.50 | +| `pnpm engine [repoPath] [prompt] [--free | --canned] [--wrong-fix] [--slow] [--approve | --reject]` | One run through the engine, one event per line. `--free` costs nothing: canned thinking, the recorded Claude run as implement, and the real Docker box. `--canned` pays only for implement. `--wrong-fix` shows failing tests. `--slow` waits 10 s in every step so you can kill it halfway | Only without `--free` | +| `pnpm engine --recover [--free | --canned] [--approve | --reject]` | What the daemon does after a restart: finds the runs a crash cut off and finishes them. Use the same flags the run started with | Only without `--free` | | `pnpm build` | Compiles to `dist/` | No | | `pnpm typecheck` | Type check only | No | @@ -236,7 +247,9 @@ The scripts in `runner/scripts/`: | File | Covers | | --- | --- | | [claude-parser.test.ts](runner/test/claude-parser.test.ts) | The whole recorded run replays in order; each mapping above; blocked commands; failed tests hidden behind exit code 0; error results; broken lines; `raw` on every event | -| [runner.test.ts](runner/test/runner.test.ts) | A worktree on a new branch with your repo untouched; clean failures for a repo with no commits, an agent with no result line, a missing binary and an unsupported agent; diff with new files; approve commits on the branch and leaves `main` alone; reject deletes the branch; nothing to approve; stop kills the agent and its children, keeps the worktree and refuses a session that is not running; resume continues the numbering, passes the prompt, can be stopped again and refuses a session that is not stopped; `approval.requested` right after `task.finished`; memory: the second run gets the first run's note, the profile is saved, and failed or stopped runs add no note; after a restart: approve, reject, diff and resume still work, and `recover()` fails interrupted sessions, removes their worktrees, keeps branches with commits, leaves live sessions alone, and reports what it can't clean or read | +| [runner.test.ts](runner/test/runner.test.ts) | A worktree on a new branch with your repo untouched; clean failures for a repo with no commits, an agent with no result line, a missing binary and an unsupported agent; diff with new files; approve commits on the branch and leaves `main` alone; reject deletes the branch; nothing to approve; stop kills the agent and its children, keeps the worktree and refuses a session that is not running; resume continues the numbering, passes the prompt, can be stopped again and refuses a session that is not stopped; `approval.requested` right after `task.finished`; memory: the second run gets the first run's note, the profile is saved, and failed or stopped runs add no note; after a restart: approve, reject, diff and resume still work, and `recover()` fails interrupted sessions, removes their worktrees, keeps branches with commits, leaves live sessions alone, and reports what it can't clean or read; `recover({ keep })` keeps a session as `stopped` and it resumes; `status()` | +| [engine.test.ts](runner/test/engine.test.ts) | The five steps in order ending in `awaiting_approval`; Claude Code's events tagged with the step; each step gets the one before; model per step; a file per step and `run.json`; wrong agent and a repo with no commits; analyze, implement, tests, sandbox and review failing, with the change kept once it exists; Claude Code's own failure says why; approve waits for the whole run; reject; stop during implement and resume; tests run in the worktree with the change | +| [engine-recover.test.ts](runner/test/engine-recover.test.ts) | A crash at every point: during analyze, after plan, during implement (Claude Code's session continued), right after Claude Code finished, during review, between `task.finished` and `approval.requested`. Each time no finished step runs again and `seq` carries on. Approve refused until the run finishes; finished and failed runs not resumed; a run stopped before the restart still resumes; a second resume refused; `RunStore` never reads outside its folder | | [session-store.test.ts](runner/test/session-store.test.ts) | A new store finds what the old one saved; the pid is never saved; moves are still checked after loading; ids that aren't plain names never reach the disk; no root means memory only | | [fake-runner.test.ts](runner/test/fake-runner.test.ts) | Event order and `seq`; approve only after finishing; stop then resume with `seq` continuing; no `approval.requested` after a stop | | [memory.test.ts](runner/test/memory.test.ts) | The stub keeps notes in order and repos apart; trimming drops whole notes and keeps the newest; the prompt cap keeps the newest end; a note's summary stays on one line; the file store reads back from disk and treats missing files as empty | diff --git a/runner/scripts/engine.ts b/runner/scripts/engine.ts index a9ae14a..412b17a 100644 --- a/runner/scripts/engine.ts +++ b/runner/scripts/engine.ts @@ -1,37 +1,50 @@ -// pnpm engine [repoPath] [prompt] [--free | --canned] [--wrong-fix] [--approve | --reject]: one run through the engine, one event per line. +// pnpm engine [repoPath] [prompt] [--free | --canned] [--wrong-fix] [--slow] [--approve | --reject]: one run, one event per line. +// pnpm engine --recover [--free | --canned] [--approve | --reject]: what the daemon does after a restart. +// It finds the runs a crash cut off and finishes them. // --canned answers analyze, plan and review from canned text, so only the implement step costs money. // --free costs nothing: canned thinking, your recorded Claude run as implement (it also makes the real // one-line fix), and the real Docker box. Add --wrong-fix to see what failing tests look like. +// --slow waits 10 seconds in every thinking step and before Claude Code answers, so you can kill it halfway. // Without --approve or --reject it keeps the worktree. Run scripts/reset-fixture.sh afterwards. import { randomUUID } from "node:crypto"; import { homedir } from "node:os"; import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; -import { createCannedModel, createEngine, createRunner } from "../src/index.js"; +import type { RunnerEvent } from "@codehands/contracts"; +import { createCannedModel, createEngine, createRunner, type ModelClient } from "../src/index.js"; const flags = process.argv.slice(2).filter((a) => a.startsWith("--")); const [repoPath = join(homedir(), "codehands-fixtures", "date-bug"), prompt = "Fix the failing test. Run npm test to check."] = process.argv.slice(2).filter((a) => !a.startsWith("--")); const free = flags.includes("--free"); +const slow = flags.includes("--slow"); const canned = createCannedModel({ analyze: "The test expects the day of the month in UTC. src/date.js probably reads it in local time.", plan: "1. Read src/date.js and test/date.test.js\n2. Read the day in UTC\n3. Run npm test", review: `VERDICT: ${flags.includes("--wrong-fix") ? "concerns" : "approve"}\n(canned review: no model was asked)`, }); +const thinking: ModelClient = slow ? { complete: async (r) => (await sleep(10_000), canned.complete(r)) } : canned; // --free: instead of Claude Code, node prints your recorded run. First it makes the change the // recorded run made: getDate() becomes getUTCDate() in src/date.js. --wrong-fix only adds a comment. +// It prints task.started at once and the rest after --slow's 10 seconds: the window to kill it in implement. +// The same stand-in answers a resume, so a run cut off during implement also finishes for free. const recording = fileURLToPath(new URL("../test/fixtures/spike-1.jsonl", import.meta.url)); const [from, to] = flags.includes("--wrong-fix") ? ["getDate();", "getDate(); // the day"] : ["getDate()", "getUTCDate()"]; -const replay = `const fs=require("fs"),f="src/date.js";if(fs.existsSync(f))fs.writeFileSync(f,fs.readFileSync(f,"utf8").replace(${JSON.stringify(from)},${JSON.stringify(to)}));process.stdout.write(fs.readFileSync(process.argv[1],"utf8"))`; -const runner = free ? createRunner({ agentCommand: () => ({ file: process.execPath, args: ["-e", replay, recording] }) }) : undefined; +const replay = `const fs=require("fs"),f="src/date.js",lines=fs.readFileSync(process.argv[1],"utf8").split("\\n"); +process.stdout.write(lines.slice(0,2).join("\\n")+"\\n"); +setTimeout(()=>{ + if(fs.existsSync(f))fs.writeFileSync(f,fs.readFileSync(f,"utf8").replace(${JSON.stringify(from)},${JSON.stringify(to)})); + process.stdout.write(lines.slice(2).join("\\n")); +},${slow ? 10_000 : 0});`; +const standIn = () => ({ file: process.execPath, args: ["-e", replay, recording] }); +const runner = free ? createRunner({ agentCommand: standIn, resumeCommand: standIn }) : undefined; -const engine = createEngine({ runner, model: free || flags.includes("--canned") ? canned : undefined }); -const sessionId = randomUUID(); +const engine = createEngine({ runner, model: free || flags.includes("--canned") ? thinking : undefined }); -let finished = false; -for await (const e of engine.start({ sessionId, repoPath, prompt, agent: "codehands", model: "sonnet", budgetUsd: 0.5 })) { +function print(e: RunnerEvent) { const p = e.payload; const step = typeof p.step === "string" ? `[${p.step}]` : ""; const detail = p.reason ?? p.command ?? p.path ?? p.model ?? (typeof p.text === "string" ? p.text.slice(0, 70) : ""); @@ -39,19 +52,43 @@ for await (const e of engine.start({ sessionId, repoPath, prompt, agent: "codeha : e.type === "test.result" && p.step === "test" ? ` ran=${p.ran} passed=${p.passed}${p.error ? ` ${p.error}` : ""}` : e.type === "task.finished" ? ` verdict=${p.verdict} $${Number(p.total_cost_usd).toFixed(4)}` : ""; console.log(`${String(e.seq).padStart(3)} ${e.type.padEnd(17)} ${step.padEnd(12)} ${String(detail).replace(/\s+/g, " ")}${extra}`); - if (e.type === "task.finished") finished = true; } -if (!finished) process.exit(1); -console.log(`\nsession: ${sessionId}`); -console.log(`step files: ${join(homedir(), ".codehands", "runs", sessionId)}`); -if (free) console.log("cost: $0. The cost above is the recorded run's, replayed"); -if (flags.includes("--approve")) { - const { branch, commit } = await engine.approve(sessionId, prompt); - console.log(`approved: ${branch} @ ${commit.slice(0, 7)}`); -} else if (flags.includes("--reject")) { - await engine.reject(sessionId); - console.log("rejected: worktree and branch deleted"); +// Prints a run's events, then approves, rejects or keeps it. false when it didn't finish. +async function watch(sessionId: string, events: AsyncIterable): Promise { + let finished = false; + for await (const e of events) { + print(e); + if (e.type === "task.finished") finished = true; + } + if (!finished) return false; + console.log(`\nsession: ${sessionId}`); + console.log(`step files: ${join(homedir(), ".codehands", "runs", sessionId)}`); + if (free) console.log("cost: $0. The cost above is the recorded run's, replayed"); + if (flags.includes("--approve")) { + const { branch, commit } = await engine.approve(sessionId, prompt); + console.log(`approved: ${branch} @ ${commit.slice(0, 7)}`); + } else if (flags.includes("--reject")) { + await engine.reject(sessionId); + console.log("rejected: worktree and branch deleted"); + } else { + console.log("kept: the worktree is still there, see `git worktree list` in the repo"); + } + return true; +} + +if (flags.includes("--recover")) { + if (slow) console.log(`pid ${process.pid} · slow: kill me halfway too, then run --recover again\n`); + const report = await engine.recover(); + console.log(`recover: ${report.resumable.length} to resume, ${report.failed.length} failed, ${report.cleanupErrors.length} cleanup errors`); + let ok = true; + for (const sessionId of report.resumable) { + console.log(`\nresuming ${sessionId}`); + ok = (await watch(sessionId, engine.resume(sessionId))) && ok; + } + if (!ok) process.exit(1); } else { - console.log("kept: the worktree is still there, see `git worktree list` in the repo"); + const sessionId = randomUUID(); + console.log(`pid ${process.pid} · session ${sessionId}${slow ? " · slow: kill me halfway, then run with --recover" : ""}\n`); + if (!(await watch(sessionId, engine.start({ sessionId, repoPath, prompt, agent: "codehands", model: "sonnet", budgetUsd: 0.5 })))) process.exit(1); } \ No newline at end of file diff --git a/runner/src/engine/engine.ts b/runner/src/engine/engine.ts index c5c25f8..289ddf8 100644 --- a/runner/src/engine/engine.ts +++ b/runner/src/engine/engine.ts @@ -7,15 +7,18 @@ import { baseCommit, git } from "../git/worktree.js"; import { createFileMemory, repoKey } from "../memory/notes.js"; import { detectRepoProfile } from "../memory/profile.js"; import { notesForPrompt, type MemoryStore } from "../memory/store.js"; -import { createRunner, type RecoverableRunner } from "../runner.js"; +import { createRunner, type RecoverableRunner, type RecoveryReport } from "../runner.js"; import { runInSandbox, type SandboxOptions, type SandboxResult } from "../runtime/sandbox.js"; import { createClaudeModel, type ModelClient } from "./model.js"; -import { analyzePrompt, implementPrompt, parseVerdict, planPrompt, reviewPrompt, type Prompt } from "./prompts.js"; +import { analyzePrompt, implementPrompt, MAX_TEST_OUTPUT, parseVerdict, planPrompt, reviewPrompt, type Prompt } from "./prompts.js"; +import { RunStore, type RunState } from "./run-store.js"; // Engine v1: analyze → plan → implement → test → review → awaiting approval. // The same Runner interface as the runner, so the daemon picks it for agent "codehands". // One rule: once the change exists, nothing after it throws it away. Tests that can't run // and a review that fails still end in awaiting approval, with the reason in the events. +// Task 5: every event saves run.json first, so after a crash recover() + resume() carry on +// from the first step that didn't finish. export const DEFAULT_MODEL = "claude-sonnet-5-5"; const SAFE_ID = /^[\w-]+$/; // the session id names a folder @@ -25,18 +28,13 @@ export interface EngineOptions { model?: ModelClient; // answers analyze, plan and review. Default createClaudeModel() sandbox?: (o: SandboxOptions) => Promise; // the test step. Default runInSandbox memory?: MemoryStore; // what earlier runs learned. Default createFileMemory() - runsRoot?: string; // step files. Default ~/.codehands/runs + runsRoot?: string; // step files and run.json. Default ~/.codehands/runs } -interface EngineRun { - options: StartOptions; +// A run this process is driving: its saved state, plus the event counter. +interface Run { + state: RunState; seq: EventSequence; - step?: StepName; // the step running now - stopped: boolean; // stopped during implement, waiting for resume() - done: boolean; // approval.requested went out - plan: string; - files: number; // step files written so far: 1-analyze.json, 2-plan.json … - costUsd: number; } type Outcome = { ok: true; text: string } | { ok: false; reason: string }; @@ -51,38 +49,60 @@ export function createEngine({ memory = createFileMemory(), runsRoot = join(homedir(), ".codehands", "runs"), }: EngineOptions = {}): RecoverableRunner { - const runs = new Map(); + const store = new RunStore(runsRoot); + // Runs whose events are flowing in this process. Saved as running but not in here = cut off by a crash. + const live = new Map(); - function get(sessionId: string): EngineRun { - const run = runs.get(sessionId); - if (!run) throw new Error(`unknown engine session ${sessionId}`); - return run; + function stateOf(sessionId: string): RunState { + const state = live.get(sessionId)?.state ?? store.load(sessionId); + if (!state) throw new Error(`unknown engine session ${sessionId}`); + return state; } - const emit = (run: EngineRun, type: RunnerEventType, payload: Record = {}, raw?: unknown) => - run.seq.next(type, payload, raw); + // For error messages: what the run is doing now. + function now(sessionId: string, s: RunState): string { + if (s.status === "stopped") return "stopped"; + if (s.status !== "running") return "not running"; + return live.has(sessionId) ? s.step ?? "starting" : "cut off by a restart, resume it"; + } + + // Saves the state before the event goes out. After a crash, seq carries on from here, + // and every step whose result is in the state is never paid for twice. + function emit(run: Run, type: RunnerEventType, payload: Record = {}, raw?: unknown): RunnerEvent { + const event = run.seq.next(type, payload, raw); + run.state.lastSeq = run.seq.last; + store.save(run.state); + return event; + } - // Every step leaves a file: what went in, what came out, cost and time. Week 4 resumes from these. + function fail(run: Run, step: StepName | undefined, reason: string): RunnerEvent { + run.state.status = "failed"; + return emit(run, "task.failed", { reason: step ? `${step}: ${reason}` : reason, step }); + } + + // Every step leaves a file: what went in, what came out, cost and time. // A step file must never fail a run, so errors are swallowed. - async function saveStep(run: EngineRun, step: StepName, record: Record): Promise { - run.files += 1; - const dir = join(runsRoot, run.options.sessionId); + async function saveStep(run: Run, step: StepName, record: Record): Promise { + run.state.files += 1; + const dir = join(runsRoot, run.state.options.sessionId); try { await mkdir(dir, { recursive: true }); - await writeFile(join(dir, `${run.files}-${step}.json`), JSON.stringify({ step, ...record }, null, 2)); + await writeFile(join(dir, `${run.state.files}-${step}.json`), JSON.stringify({ step, ...record }, null, 2)); } catch { } } // analyze, plan and review: one model call each, with no tools. - async function* think(run: EngineRun, step: StepName, p: Prompt): AsyncGenerator { - run.step = step; - const m = modelFor(run.options, step); + // keep() puts the answer in the state before step.finished goes out, so it's saved with it. + async function* think(run: Run, step: StepName, p: Prompt, keep: (text: string) => void): AsyncGenerator { + run.state.step = step; + const m = modelFor(run.state.options, step); yield emit(run, "step.started", { step, model: m }); const started = Date.now(); try { const reply = await model.complete({ step, model: m, ...p }); - run.costUsd += reply.costUsd; + run.state.costUsd += reply.costUsd; await saveStep(run, step, { ok: true, model: m, durationMs: Date.now() - started, costUsd: reply.costUsd, input: p, output: reply.text }); + keep(reply.text); yield emit(run, "message", { step, text: reply.text }); yield emit(run, "step.finished", { step, ok: true, costUsd: reply.costUsd }); return { ok: true, text: reply.text }; @@ -97,10 +117,9 @@ export function createEngine({ // Claude Code does the work in the runner's worktree. Its events go out as they arrive, renumbered // and tagged with the step. The runner's own task.started, task.finished and approval.requested // stay inside: the engine sends its own when the whole run is done. - async function* implement(run: EngineRun, inner: AsyncIterable, prompt: string): AsyncGenerator { - run.step = "implement"; - run.stopped = false; - const m = modelFor(run.options, "implement"); + async function* implement(run: Run, inner: AsyncIterable, prompt: string): AsyncGenerator { + run.state.step = "implement"; + const m = modelFor(run.state.options, "implement"); yield emit(run, "step.started", { step: "implement", model: m }); const started = Date.now(); for await (const e of inner) { @@ -109,33 +128,47 @@ export function createEngine({ yield emit(run, "message", { step: "implement", text: "Claude Code started" }, e.raw); } else if (e.type === "task.finished") { const costUsd = Number(e.payload.total_cost_usd ?? 0); - run.costUsd += costUsd; + run.state.costUsd += costUsd; + run.state.implemented = true; await saveStep(run, "implement", { ok: true, model: m, durationMs: Date.now() - started, costUsd, input: prompt, output: e.payload.result, diffStat: e.payload.diffStat }); yield emit(run, "step.finished", { step: "implement", ok: true, costUsd }); return "done"; } else if (e.type === "task.stopped") { - run.stopped = true; + run.state.status = "stopped"; yield emit(run, "step.finished", { step: "implement", ok: false }); yield emit(run, "task.stopped", { step: "implement" }); return "stopped"; } else if (e.type === "task.failed") { - await saveStep(run, "implement", { ok: false, model: m, durationMs: Date.now() - started, input: prompt, error: e.payload.reason }); + // The runner's own failures have a reason. Claude Code's (an error result) have only result and subtype. + const reason = String(e.payload.reason ?? e.payload.result ?? e.payload.subtype ?? "Claude Code failed"); + await saveStep(run, "implement", { ok: false, model: m, durationMs: Date.now() - started, input: prompt, error: reason }); yield emit(run, "step.finished", { step: "implement", ok: false }); - yield emit(run, "task.failed", { ...e.payload, step: "implement" }); + run.state.status = "failed"; + yield emit(run, "task.failed", { ...e.payload, reason: `implement: ${reason}`, step: "implement" }); return "failed"; } else { yield emit(run, e.type, { ...e.payload, step: "implement" }, e.raw); } } - yield emit(run, "task.failed", { reason: "implement ended without a result", step: "implement" }); + yield fail(run, "implement", "Claude Code ended without a result"); return "failed"; } - // test and review, then the end of the run. Shared by start() and resume(). - async function* finish(run: EngineRun): AsyncGenerator { - const { sessionId } = run.options; + // Where Claude Code's part comes from: a new session, the stopped one, or none at all + // when it finished just before a restart. undefined means implement is already done. + function implementStream(run: Run, prompt: string, resumePrompt?: string): AsyncIterable | undefined { + const o = run.state.options; + const status = runner.status(o.sessionId); + if (status === undefined) return runner.start({ ...o, agent: "claude-code", prompt, model: modelFor(o, "implement") }); + if (status === "stopped") return runner.resume(o.sessionId, resumePrompt); + if (status === "awaiting_approval") return undefined; + throw new Error(`can't continue: Claude Code's session is ${status}`); + } - run.step = "test"; + // The tests, in the Task 3 box, in the worktree with the change. + async function* test(run: Run): AsyncGenerator { + const { sessionId } = run.state.options; + run.state.step = "test"; yield emit(run, "step.started", { step: "test" }); const started = Date.now(); const worktree = runner.worktree(sessionId); @@ -148,99 +181,155 @@ export function createEngine({ result = { command: "", image: "", passed: false, exitCode: null, timedOut: false, oomKilled: false, durationMs: 0, output: "", outputTruncated: false, error: "sandbox.failed", message: (err as Error).message }; } const ran = !result.error; - const { output: _output, ...summary } = result; // the output itself is in result.resultFile + const { output, ...summary } = result; // the whole output is in result.resultFile + run.state.tests = { ...result, output: output.slice(-MAX_TEST_OUTPUT) }; // what review needs, if it runs after a restart await saveStep(run, "test", { ok: ran, durationMs: Date.now() - started, result: summary }); yield emit(run, "test.result", { step: "test", ran, ...summary }); yield emit(run, "step.finished", { step: "test", ok: ran }); + } - const { patch, stat } = await runner.diff(sessionId); - const review = yield* think(run, "review", reviewPrompt(run.options.prompt, run.plan, patch, result)); - const verdict = review.ok ? parseVerdict(review.text) : "unknown"; - - run.step = undefined; - run.done = true; + // Review, then the end of the run. + async function* finish(run: Run): AsyncGenerator { + const s = run.state; + const tests = s.tests!; + const { patch, stat } = await runner.diff(s.options.sessionId); + if (!s.review) { + const review = yield* think(run, "review", reviewPrompt(s.options.prompt, s.plan!, patch, tests), (text) => { + s.review = { verdict: parseVerdict(text), text }; + }); + if (!review.ok) s.review = { verdict: "unknown", text: `review failed: ${review.reason}` }; + } + const { verdict, text } = s.review!; + s.step = undefined; yield emit(run, "task.finished", { diffStat: stat, verdict, - review: review.ok ? review.text : `review failed: ${review.reason}`, - tests: { ran, passed: result.passed }, - total_cost_usd: run.costUsd, + review: text, + tests: { ran: !tests.error, passed: tests.passed }, + total_cost_usd: s.costUsd, }); + s.status = "awaiting_approval"; yield emit(run, "approval.requested", { diffStat: stat, verdict }); } - const fail = (run: EngineRun, step: StepName | undefined, reason: string) => - emit(run, "task.failed", { reason: step ? `${step}: ${reason}` : reason, step }); - - return { - async *start(o) { - if (runs.has(o.sessionId)) throw new Error(`session ${o.sessionId} already exists`); - const run: EngineRun = { options: o, seq: new EventSequence(o.sessionId), stopped: false, done: false, plan: "", files: 0, costUsd: 0 }; - runs.set(o.sessionId, run); - - if (o.agent !== "codehands") { - yield fail(run, undefined, `the engine runs agent "codehands", not "${o.agent}"`); - return; - } - if (!SAFE_ID.test(o.sessionId)) { - yield fail(run, undefined, `bad session id: ${o.sessionId}`); - return; - } - yield emit(run, "task.started", { agent: "codehands", prompt: o.prompt }); - - try { + // The five steps, skipping every step whose result is already in the state. + // start(), resume() after a stop, and resume() after a crash all come through here. + async function* drive(run: Run, resumePrompt?: string): AsyncGenerator { + const s = run.state; + const o = s.options; + live.set(o.sessionId, run); + try { + if (s.analysis === undefined) { // Check the repo before paying for any model call: implement needs a commit to branch from. await baseCommit(o.repoPath); const profile = await detectRepoProfile(o.repoPath); const files = (await git(o.repoPath, ["ls-files"]).catch(() => "")).split("\n").filter(Boolean); const notes = await memory.notes(await repoKey(o.repoPath)).then(notesForPrompt).catch(() => undefined); + const r = yield* think(run, "analyze", analyzePrompt(o.prompt, profile, files, notes), (t) => { s.analysis = t; }); + if (!r.ok) return void (yield fail(run, "analyze", r.reason)); + } + if (s.plan === undefined) { + const r = yield* think(run, "plan", planPrompt(o.prompt, s.analysis!), (t) => { s.plan = t; }); + if (!r.ok) return void (yield fail(run, "plan", r.reason)); + } + if (!s.implemented) { + s.step = "implement"; + const prompt = implementPrompt(o.prompt, s.plan!); + const inner = implementStream(run, prompt, resumePrompt); + if (inner) { + const outcome = yield* implement(run, inner, prompt); + if (outcome !== "done") return; + } else { + s.implemented = true; // Claude Code finished just before the restart + yield emit(run, "step.finished", { step: "implement", ok: true }); + } + } + if (!s.tests) yield* test(run); + yield* finish(run); + } catch (err) { + yield fail(run, s.step, (err as Error).message); + } finally { + live.delete(o.sessionId); + } + } - const analysis = yield* think(run, "analyze", analyzePrompt(o.prompt, profile, files, notes)); - if (!analysis.ok) return void (yield fail(run, "analyze", analysis.reason)); - - const plan = yield* think(run, "plan", planPrompt(o.prompt, analysis.text)); - if (!plan.ok) return void (yield fail(run, "plan", plan.reason)); - run.plan = plan.text; - - const prompt = implementPrompt(o.prompt, plan.text); - const outcome = yield* implement(run, runner.start({ ...o, agent: "claude-code", prompt, model: modelFor(o, "implement") }), prompt); - if (outcome === "done") yield* finish(run); - } catch (err) { - yield fail(run, run.step, (err as Error).message); + return { + async *start(o) { + // Checked before anything is saved: a bad id must never name a folder. + if (o.agent !== "codehands" || !SAFE_ID.test(o.sessionId)) { + const reason = o.agent !== "codehands" ? `the engine runs agent "codehands", not "${o.agent}"` : `bad session id: ${o.sessionId}`; + yield new EventSequence(o.sessionId).next("task.failed", { reason }); + return; } + if (live.has(o.sessionId) || store.load(o.sessionId)) throw new Error(`session ${o.sessionId} already exists`); + const run: Run = { state: { options: o, status: "running", lastSeq: 0, files: 0, costUsd: 0 }, seq: new EventSequence(o.sessionId) }; + live.set(o.sessionId, run); + yield emit(run, "task.started", { agent: "codehands", prompt: o.prompt }); + yield* drive(run); }, // v1 can only stop while Claude Code works. analyze, plan and review are single short calls. async stop(sessionId) { - const run = get(sessionId); - if (run.step !== "implement" || run.stopped) { - throw new Error(`the engine can only stop during implement (now: ${run.stopped ? "stopped" : run.step ?? "not running"})`); + const s = stateOf(sessionId); + if (!live.has(sessionId) || s.status !== "running" || s.step !== "implement") { + throw new Error(`the engine can only stop during implement (now: ${now(sessionId, s)})`); } await runner.stop(sessionId); }, + // Carries on a run that was stopped, or one a crash cut off (recover() lists those). + // Steps that finished are skipped. seq carries on from the last event sent. async *resume(sessionId, prompt) { - const run = get(sessionId); - if (!run.stopped) throw new Error(`session ${sessionId} is not stopped`); - try { - const outcome = yield* implement(run, runner.resume(sessionId, prompt), prompt ?? "Continue."); - if (outcome === "done") yield* finish(run); - } catch (err) { - yield fail(run, run.step, (err as Error).message); - } + const s = stateOf(sessionId); + if (live.has(sessionId)) throw new Error(`session ${sessionId} is already running`); + const cutOff = s.status === "running"; + if (s.status !== "stopped" && !cutOff) throw new Error(`session ${sessionId} is not stopped`); + const run: Run = { state: s, seq: new EventSequence(sessionId, s.lastSeq) }; + live.set(sessionId, run); // now, so a second resume() is refused + s.status = "running"; + if (cutOff) yield emit(run, "message", { step: s.step, text: `Picked up after a restart, from ${s.step ?? "the start"}` }); + yield* drive(run, prompt); }, diff: (sessionId) => runner.diff(sessionId), - // The runner already allows approve after implement. The engine waits for test and review. + // The runner already allows approve after implement. The engine waits for test and review, + // even after a restart: run.json says whether review finished. async approve(sessionId, message) { - const run = runs.get(sessionId); - if (run && !run.done) throw new Error(`the engine hasn't finished ${sessionId} yet (now: ${run.stopped ? "stopped" : run.step ?? "not running"})`); + const s = live.get(sessionId)?.state ?? store.load(sessionId); + if (s && s.status !== "awaiting_approval") throw new Error(`the engine hasn't finished ${sessionId} yet (now: ${now(sessionId, s)})`); return runner.approve(sessionId, message); }, reject: (sessionId) => runner.reject(sessionId), - recover: () => runner.recover(), + + // Call once when the daemon starts, instead of runner.recover(): it calls that itself. + // resumable: engine runs a crash cut off, for the daemon to resume(). Their Claude Code + // session, if implement was running, is kept as stopped instead of failed. + async recover() { + const resumable: string[] = []; + const unreadable: RecoveryReport["cleanupErrors"] = []; + for (const sessionId of store.ids()) { + try { + if (!live.has(sessionId) && store.load(sessionId)?.status === "running") resumable.push(sessionId); + } catch (err) { + unreadable.push({ sessionId, reason: (err as Error).message }); + } + } + const report = await runner.recover({ keep: (id) => resumable.includes(id) }); + return { failed: report.failed, cleanupErrors: [...unreadable, ...report.cleanupErrors], resumable }; + }, + worktree: (sessionId) => runner.worktree(sessionId), + + // The run's own status until it's waiting for approval. After that the runner's, which knows approved and rejected. + status(sessionId) { + try { + const s = live.get(sessionId)?.state ?? store.load(sessionId); + return s && s.status !== "awaiting_approval" ? s.status : runner.status(sessionId); + } catch { + return undefined; + } + }, }; } \ No newline at end of file diff --git a/runner/src/engine/run-store.ts b/runner/src/engine/run-store.ts new file mode 100644 index 0000000..e83527b --- /dev/null +++ b/runner/src/engine/run-store.ts @@ -0,0 +1,59 @@ +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { StartOptions, StepName } from "@codehands/contracts"; +import type { SandboxResult } from "../runtime/sandbox.js"; +import type { Verdict } from "./prompts.js"; + +// Everything the engine needs to carry on after a restart: what each finished step gave, +// and where the run stands. One file per run, next to its step files: +// //run.json. Saved before every event goes out. +export interface RunState { + options: StartOptions; + status: "running" | "stopped" | "awaiting_approval" | "failed"; + step?: StepName; // the step running now + analysis?: string; // set when analyze finished + plan?: string; // set when plan finished + implemented?: boolean; // Claude Code finished: the change is in the worktree + tests?: SandboxResult; // the test step's result, output cut short + review?: { verdict: Verdict; text: string }; // set when review finished, or failed + lastSeq: number; // the last event sent, so seq carries on + files: number; // step files written so far + costUsd: number; +} + +// Session ids name folders, so one like ../../x must never reach the disk. +const SAFE_ID = /^[\w-]+$/; + +export class RunStore { + constructor(private readonly root: string) { } + + // Writes to a temp file first, so a crash mid-write can't leave half a state. + save(state: RunState): void { + const dir = join(this.root, state.options.sessionId); + mkdirSync(dir, { recursive: true }); + const file = join(dir, "run.json"); + writeFileSync(`${file}.tmp`, JSON.stringify(state)); + renameSync(`${file}.tmp`, file); + } + + // undefined when the session isn't the engine's: a Claude Code run, or an unknown id. + load(sessionId: string): RunState | undefined { + if (!SAFE_ID.test(sessionId)) return undefined; + let text: string; + try { + text = readFileSync(join(this.root, sessionId, "run.json"), "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw err; + } + return JSON.parse(text) as RunState; + } + + // Every run with a state file, including ones from before a restart. + ids(): string[] { + if (!existsSync(this.root)) return []; + return readdirSync(this.root, { withFileTypes: true }) + .filter((d) => d.isDirectory() && existsSync(join(this.root, d.name, "run.json"))) + .map((d) => d.name); + } +} \ No newline at end of file diff --git a/runner/src/index.ts b/runner/src/index.ts index 3585217..db2f539 100644 --- a/runner/src/index.ts +++ b/runner/src/index.ts @@ -1,4 +1,4 @@ -export { createRunner, type RecoverableRunner, type RecoveryReport, type RunnerOptions } from "./runner.js"; +export { createRunner, type RecoverableRunner, type RecoveryReport, type RecoverOptions, type RunnerOptions } from "./runner.js"; export { createFakeRunner, type FakeRunnerOptions } from "./fake-runner.js"; export { EventSequence } from "./events.js"; export { SessionStore, type SessionRecord } from "./session-store.js"; @@ -11,3 +11,4 @@ export { dockerArgs, installArgs, dockerAvailable, hostTimeZone, DEFAULT_LIMITS, export { createEngine, modelFor, DEFAULT_MODEL, type EngineOptions } from "./engine/engine.js"; export { createClaudeModel, createCannedModel, claudeModelArgs, parseClaudeJson, type ModelClient, type ModelRequest, type ModelReply, type ClaudeModelOptions } from "./engine/model.js"; export { analyzePrompt, planPrompt, implementPrompt, reviewPrompt, parseVerdict, type Prompt, type Verdict } from "./engine/prompts.js"; +export { RunStore, type RunState } from "./engine/run-store.js"; \ No newline at end of file diff --git a/runner/src/runner.ts b/runner/src/runner.ts index 226eda4..01335d1 100644 --- a/runner/src/runner.ts +++ b/runner/src/runner.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { createInterface } from "node:readline"; -import type { Runner, RunnerEvent, StartOptions } from "@codehands/contracts"; +import type { Runner, RunnerEvent, SessionStatus, StartOptions } from "@codehands/contracts"; import { execa } from "execa"; import { claudeArgs, claudeResumeArgs, newClaudeParseState, parseClaudeLine } from "./agents/claude-code.js"; import { EventSequence } from "./events.js"; @@ -29,14 +29,23 @@ export interface RunnerOptions { // What recover() did. failed: sessions a restart interrupted, for the daemon to mark failed too. // cleanupErrors: sessions it couldn't clean up. They stay on disk and the next recover() tries again. +// resumable: interrupted sessions that were kept instead, for the daemon to resume(). export interface RecoveryReport { failed: string[]; cleanupErrors: { sessionId: string; reason: string }[]; + resumable: string[]; +} + +export interface RecoverOptions { + // A running session this says yes to is kept as stopped, with its worktree, instead of failed. + // The engine uses it for its implement step: it continues that session after the restart. + keep?: (sessionId: string) => boolean; } export interface RecoverableRunner extends Runner { - recover(): Promise; + recover(o?: RecoverOptions): Promise; worktree(sessionId: string): string | undefined; + status(sessionId: string): SessionStatus | undefined; } const claudeCommand = (o: StartOptions, notes?: string) => ({ file: "claude", args: claudeArgs(o, notes) }); @@ -210,15 +219,29 @@ export function createRunner({ return undefined; } }, + // Where the session stands, or undefined when there is no such session. + status(sessionId) { + try { + return store.get(sessionId).status; + } catch { + return undefined; + } + }, // Call once when the daemon starts, before it starts or commands anything. - // A session still marked queued or running lost its agent, so it becomes failed. + // A session still marked queued or running lost its agent, so it becomes failed, + // unless keep() says its owner will continue it: then it becomes stopped, worktree and all. // awaiting_approval and stopped sessions are left alone: approve, reject and resume still work. - async recover() { - const report: RecoveryReport = { failed: [], cleanupErrors: [] }; + async recover({ keep = () => false }: RecoverOptions = {}) { + const report: RecoveryReport = { failed: [], cleanupErrors: [], resumable: [] }; for (const sessionId of store.ids()) { if (exits.has(sessionId)) continue; // its agent is alive in this process try { const record = store.get(sessionId); + if (record.status === "running" && record.worktreePath && keep(sessionId)) { + store.transition(sessionId, "stopped"); + report.resumable.push(sessionId); + continue; + } if (record.status === "queued" || record.status === "running") { store.transition(sessionId, "failed"); report.failed.push(sessionId); diff --git a/runner/test/engine-recover.test.ts b/runner/test/engine-recover.test.ts new file mode 100644 index 0000000..6ee1ceb --- /dev/null +++ b/runner/test/engine-recover.test.ts @@ -0,0 +1,256 @@ +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { RunnerEvent, StartOptions } from "@codehands/contracts"; +import { afterAll, describe, expect, it } from "vitest"; +import { + createCannedModel, createEngine, createMemoryStub, createRunner, RunStore, + type SandboxOptions, type SandboxResult, +} from "../src/index.js"; + +// Crash tests. A "crash" here: we stop reading a run's events and leave it hanging, the way a +// killed daemon leaves its files. Then a new engine and runner on the same folders, like the +// restarted daemon, call recover() and resume(). Nothing in the first engine runs again. +const replay = fileURLToPath(new URL("./helpers/replay-agent.mjs", import.meta.url)); +const spike = fileURLToPath(new URL("./fixtures/spike-1.jsonl", import.meta.url)); +const tmp = mkdtempSync(join(tmpdir(), "codehands-recover-")); +afterAll(() => rmSync(tmp, { recursive: true, force: true })); + +// Claude Code that dies early: prints task.started, then exits with no result. +const dies = join(tmp, "dies.jsonl"); +writeFileSync(dies, readFileSync(spike, "utf8").split("\n").slice(0, 2).join("\n") + "\n"); + +const git = (cwd: string, ...args: string[]) => + execFileSync("git", ["-c", "user.name=test", "-c", "user.email=test@example.com", "-c", "core.autocrlf=false", ...args], { cwd, encoding: "utf8" }).trim(); + +function makeRepo(): string { + const repo = mkdtempSync(join(tmp, "repo-")); + git(repo, "init", "-q", "-b", "main"); + writeFileSync(join(repo, "a.txt"), "hello\n"); + writeFileSync(join(repo, "package.json"), JSON.stringify({ name: "r", private: true, scripts: { test: "node -e 0" } })); + git(repo, "add", "-A"); + git(repo, "commit", "-q", "-m", "first"); + return repo; +} + +const options = (repoPath: string): StartOptions => + ({ sessionId: randomUUID(), repoPath, prompt: "Fix the failing test", agent: "codehands" }); + +// The folders a daemon keeps between restarts. +const folders = () => ({ + sessionRoot: mkdtempSync(join(tmp, "sessions-")), + worktreeRoot: join(tmp, "worktrees"), + runsRoot: mkdtempSync(join(tmp, "runs-")), +}); +type Folders = ReturnType; + +// One daemon's engine. calls counts what it paid for: model steps, Claude Code starts and resumes, test runs. +function daemon(f: Folders, agent: "edit" | "dies" = "edit") { + const calls = { claudeStart: 0, claudeResume: 0, tests: 0 }; + const model = createCannedModel({ analyze: "ANALYSIS", plan: "1. PLAN", review: "VERDICT: approve" }); + const runner = createRunner({ + sessionRoot: f.sessionRoot, + worktreeRoot: f.worktreeRoot, + memory: createMemoryStub(), + agentCommand: () => { + calls.claudeStart += 1; + return { file: process.execPath, args: agent === "dies" ? [replay, dies] : [replay, spike, "0", "edit"] }; + }, + resumeCommand: () => { + calls.claudeResume += 1; + return { file: process.execPath, args: [replay, spike, "0", "edit"] }; + }, + }); + const sandbox = async (o: SandboxOptions): Promise => { + calls.tests += 1; + return { command: "npm test", image: "node:22-slim", passed: true, exitCode: 0, timedOut: false, oomKilled: false, durationMs: 900, output: `# pass 1 in ${o.worktree}`, outputTruncated: false }; + }; + const engine = createEngine({ runner, model, sandbox, memory: createMemoryStub(), runsRoot: f.runsRoot }); + return { engine, model, calls }; +} + +// Reads events until crashAt says so, then never asks for another: the process "died" there. +async function crashAfter(run: AsyncIterable, crashAt: (e: RunnerEvent) => boolean): Promise { + const it = run[Symbol.asyncIterator](); + const events: RunnerEvent[] = []; + for (let r = await it.next(); !r.done; r = await it.next()) { + events.push(r.value); + if (crashAt(r.value)) break; + } + return events; +} + +async function drain(run: AsyncIterable): Promise { + const events: RunnerEvent[] = []; + for await (const e of run) events.push(e); + return events; +} + +const is = (type: string, step?: string) => (e: RunnerEvent) => e.type === type && (step === undefined || e.payload.step === step); +const steps = (events: RunnerEvent[]) => events.filter((e) => e.type.startsWith("step.")).map((e) => `${e.type === "step.started" ? ">" : "<"}${e.payload.step}`); + +// Crash a run at crashAt, restart, recover, resume. Returns both halves. +async function crashAndResume(crashAt: (e: RunnerEvent) => boolean, agent: "edit" | "dies" = "edit") { + const f = folders(); + const o = options(makeRepo()); + const first = daemon(f, agent); + const before = await crashAfter(first.engine.start(o), crashAt); + + const second = daemon(f); + const report = await second.engine.recover(); + const after = await drain(second.engine.resume(o.sessionId)); + return { f, o, first, second, report, before, after, all: [...before, ...after] }; +} + +describe("engine after a crash", () => { + it("recover lists the cut-off run, and resume finishes it with seq carrying on", async () => { + const { o, report, after, all } = await crashAndResume(is("step.started", "plan")); + expect(report).toEqual({ failed: [], cleanupErrors: [], resumable: [o.sessionId] }); + expect(after[0]).toMatchObject({ type: "message", payload: { text: "Picked up after a restart, from plan" } }); + expect(all.map((e) => e.seq)).toEqual(all.map((_, i) => i + 1)); + expect(after.at(-1)).toMatchObject({ type: "approval.requested", payload: { verdict: "approve" } }); + }); + + it("crash during analyze: analyze runs again", async () => { + const { second, after } = await crashAndResume(is("step.started", "analyze")); + expect(second.model.calls.map((c) => c.step)).toEqual(["analyze", "plan", "review"]); + expect(steps(after)[0]).toBe(">analyze"); + }); + + it("crash after plan: analyze and plan aren't paid for twice, and implement gets the saved plan", async () => { + const { second, after } = await crashAndResume(is("step.finished", "plan")); + expect(second.model.calls.map((c) => c.step)).toEqual(["review"]); + expect(steps(after)[0]).toBe(">implement"); + expect(second.calls.claudeStart).toBe(1); + }); + + it("crash during implement: Claude Code's session is continued, not started over", async () => { + const { o, second, report, after } = await crashAndResume((e) => e.type === "message" && e.payload.text === "Claude Code started", "dies"); + expect(report.resumable).toEqual([o.sessionId]); + expect(second.calls).toMatchObject({ claudeStart: 0, claudeResume: 1 }); + expect(steps(after)).toEqual([">implement", "test", "review", " { + const f = folders(); + const o = options(makeRepo()); + await crashAfter(daemon(f).engine.start(o), is("step.finished", "implement")); + // The runner had finished, but the engine died before it saved that. + const store = new RunStore(f.runsRoot); + store.save({ ...store.load(o.sessionId)!, implemented: undefined }); + + const second = daemon(f); + await second.engine.recover(); + const after = await drain(second.engine.resume(o.sessionId)); + expect(second.calls).toMatchObject({ claudeStart: 0, claudeResume: 0, tests: 1 }); + expect(steps(after)).toEqual(["test", "review", " { + const { second, after } = await crashAndResume(is("step.started", "review")); + expect(second.calls.tests).toBe(0); + expect(second.model.calls.map((c) => c.step)).toEqual(["review"]); + expect(second.model.calls[0]!.prompt).toContain("The tests passed"); + expect(after.at(-1)!.type).toBe("approval.requested"); + }); + + it("crash between task.finished and approval.requested: resume sends both again", async () => { + const { second, after } = await crashAndResume(is("task.finished")); + expect(second.model.calls).toHaveLength(0); + expect(after.map((e) => e.type)).toEqual(["message", "task.finished", "approval.requested"]); + }); +}); + +describe("engine recover and approve after a restart", () => { + it("approve is refused until the cut-off run finishes", async () => { + const f = folders(); + const o = options(makeRepo()); + await crashAfter(daemon(f).engine.start(o), is("step.started", "test")); + + const second = daemon(f); + await second.engine.recover(); + await expect(second.engine.approve(o.sessionId, "too early")).rejects.toThrow(/hasn't finished .* cut off by a restart/); + expect(second.engine.status(o.sessionId)).toBe("running"); + await drain(second.engine.resume(o.sessionId)); + expect(second.engine.status(o.sessionId)).toBe("awaiting_approval"); + const { branch, commit } = await second.engine.approve(o.sessionId, "Fix the failing test"); + expect(git(o.repoPath, "rev-parse", branch)).toBe(commit); + expect(second.engine.status(o.sessionId)).toBe("approved"); + }); + + it("recover leaves finished and failed runs alone, and a second recover finds nothing", async () => { + const f = folders(); + const first = daemon(f); + const done = options(makeRepo()); + await drain(first.engine.start(done)); + const failed = options(join(tmp, "not-a-repo")); + await drain(first.engine.start(failed)); + const cut = options(makeRepo()); + await crashAfter(first.engine.start(cut), is("step.started", "implement")); + + const second = daemon(f); + expect((await second.engine.recover()).resumable).toEqual([cut.sessionId]); + await drain(second.engine.resume(cut.sessionId)); + expect((await second.engine.recover()).resumable).toEqual([]); + }); + + it("a run stopped before the restart resumes like any stopped run", async () => { + const f = folders(); + const o = options(makeRepo()); + const pidFile = join(tmp, `child-${o.sessionId}.pid`); + const runner = createRunner({ + sessionRoot: f.sessionRoot, worktreeRoot: f.worktreeRoot, memory: createMemoryStub(), + agentCommand: () => ({ file: process.execPath, args: [replay, spike, "0", "hang", pidFile] }), + }); + const engine = createEngine({ runner, model: createCannedModel({}), memory: createMemoryStub(), runsRoot: f.runsRoot, sandbox: async () => { throw new Error("never reached: stopped during implement"); } }); + let stopping: Promise | undefined; + for await (const e of engine.start(o)) { + if (e.type === "message" && e.payload.text === "Claude Code started") stopping = engine.stop(o.sessionId); + } + await stopping; + + const second = daemon(f); + expect((await second.engine.recover()).resumable).toEqual([]); // stopped is not cut off + const after = await drain(second.engine.resume(o.sessionId, "Carry on.")); + expect(second.calls.claudeResume).toBe(1); + expect(after.at(-1)!.type).toBe("approval.requested"); + }); + + it("refuses a second resume of the same run, and a resume of a run nobody knows", async () => { + const f = folders(); + const o = options(makeRepo()); + await crashAfter(daemon(f).engine.start(o), is("step.started", "analyze")); + const second = daemon(f); + await second.engine.recover(); + const it1 = second.engine.resume(o.sessionId)[Symbol.asyncIterator](); + await it1.next(); + await expect(drain(second.engine.resume(o.sessionId))).rejects.toThrow(/already running/); + await expect(drain(second.engine.resume(randomUUID()))).rejects.toThrow(/unknown engine session/); + for (let r = await it1.next(); !r.done; r = await it1.next()); + }); +}); + +describe("RunStore", () => { + it("saves and loads a run, and lists only folders with a run.json", () => { + const root = mkdtempSync(join(tmp, "store-")); + const store = new RunStore(root); + const o = options("x"); + store.save({ options: o, status: "running", lastSeq: 3, files: 1, costUsd: 0, analysis: "A" }); + expect(store.load(o.sessionId)).toMatchObject({ status: "running", lastSeq: 3, analysis: "A" }); + writeFileSync(join(root, "stray.txt"), ""); + expect(store.ids()).toEqual([o.sessionId]); + }); + + it("says undefined for an unknown id, and never reads outside its folder", () => { + const store = new RunStore(mkdtempSync(join(tmp, "store-"))); + expect(store.load(randomUUID())).toBeUndefined(); + expect(store.load("../../etc")).toBeUndefined(); + expect(new RunStore(join(tmp, "nothing-here")).ids()).toEqual([]); + }); +}); \ No newline at end of file diff --git a/runner/test/engine.test.ts b/runner/test/engine.test.ts index c34220a..bb76fa0 100644 --- a/runner/test/engine.test.ts +++ b/runner/test/engine.test.ts @@ -133,7 +133,7 @@ describe("engine.start", () => { const o = options(makeRepo()); await drain(engine.start(o)); const dir = join(runsRoot, o.sessionId); - expect(readdirSync(dir).sort()).toEqual(["1-analyze.json", "2-plan.json", "3-implement.json", "4-test.json", "5-review.json"]); + expect(readdirSync(dir).sort()).toEqual(["1-analyze.json", "2-plan.json", "3-implement.json", "4-test.json", "5-review.json", "run.json"]); expect(JSON.parse(readFileSync(join(dir, "2-plan.json"), "utf8"))).toMatchObject({ step: "plan", ok: true, output: "1. PLAN" }); expect(JSON.parse(readFileSync(join(dir, "3-implement.json"), "utf8")).diffStat).toMatch(/2 files changed/); }); @@ -167,7 +167,15 @@ describe("engine when something goes wrong", () => { const { engine } = setup({ runner: { agentCommand: () => ({ file: process.execPath, args: [replay, empty, "1"] }) } }); const events = await drain(engine.start(options(makeRepo()))); expect(steps(events).at(-1)).toBe(" { + const failed = join(tmp, "claude-error.jsonl"); + writeFileSync(failed, JSON.stringify({ type: "result", subtype: "error_during_execution", is_error: true, result: "No conversation found" }) + "\n"); + const { engine } = setup({ runner: { agentCommand: () => ({ file: process.execPath, args: [replay, failed, "1"] }) } }); + const events = await drain(engine.start(options(makeRepo()))); + expect(events.at(-1)).toMatchObject({ type: "task.failed", payload: { step: "implement", reason: "implement: No conversation found" } }); }); it("tests fail: that's a result, and the run still ends awaiting approval", async () => { diff --git a/runner/test/runner.test.ts b/runner/test/runner.test.ts index 4f0af03..0b4afa3 100644 --- a/runner/test/runner.test.ts +++ b/runner/test/runner.test.ts @@ -340,11 +340,11 @@ describe("runner after a restart", () => { markRunning(roots.sessionRoot, o.sessionId); const restarted = newRunner(roots); - expect(await restarted.recover()).toEqual({ failed: [o.sessionId], cleanupErrors: [] }); + expect(await restarted.recover()).toEqual({ failed: [o.sessionId], cleanupErrors: [], resumable: [] }); expect(existsSync(worktree)).toBe(false); expect(git(repo, "branch", "--list", branchOf(o))).toBe(""); await expect(restarted.approve(o.sessionId, "msg")).rejects.toThrow(/no worktree/); - expect(await restarted.recover()).toEqual({ failed: [], cleanupErrors: [] }); + expect(await restarted.recover()).toEqual({ failed: [], cleanupErrors: [], resumable: [] }); }); it("recover keeps a branch the agent committed to", async () => { @@ -371,7 +371,7 @@ describe("runner after a restart", () => { const worktree = worktreeOf(repo, o.sessionId)!; expect(existsSync(worktree)).toBe(true); - expect(await newRunner(roots).recover()).toEqual({ failed: [], cleanupErrors: [] }); + expect(await newRunner(roots).recover()).toEqual({ failed: [], cleanupErrors: [], resumable: [] }); expect(existsSync(worktree)).toBe(false); }); @@ -380,7 +380,7 @@ describe("runner after a restart", () => { const waiting = await finished(roots); const stopped = await stoppedRun(roots); - expect(await newRunner(roots).recover()).toEqual({ failed: [], cleanupErrors: [] }); + expect(await newRunner(roots).recover()).toEqual({ failed: [], cleanupErrors: [], resumable: [] }); expect(existsSync(worktreeOf(waiting.repo, waiting.o.sessionId)!)).toBe(true); expect(existsSync(worktreeOf(stopped.repo, stopped.o.sessionId)!)).toBe(true); }); @@ -398,7 +398,7 @@ describe("runner after a restart", () => { expect(existsSync(worktree)).toBe(true); git(repo, "worktree", "unlock", worktree); - expect(await newRunner(roots).recover()).toEqual({ failed: [], cleanupErrors: [] }); + expect(await newRunner(roots).recover()).toEqual({ failed: [], cleanupErrors: [], resumable: [] }); expect(existsSync(worktree)).toBe(false); }); @@ -413,6 +413,27 @@ describe("runner after a restart", () => { expect(report.cleanupErrors.map((e) => e.sessionId)).toEqual(["broken"]); expect(worktreeOf(repo, o.sessionId)).toBeUndefined(); }); + + it("recover keeps an interrupted session that keep() claims, and it resumes", async () => { + const roots = sessionRootOf(); + const { repo, o } = await stoppedRun(roots); + const worktree = worktreeOf(repo, o.sessionId)!; + markRunning(roots.sessionRoot, o.sessionId); + + const restarted = newRunner({ ...roots, resumeCommand: replayAgent(spike, 0, "edit") }); + expect(await restarted.recover({ keep: (id) => id === o.sessionId })).toEqual({ failed: [], cleanupErrors: [], resumable: [o.sessionId] }); + expect(restarted.status(o.sessionId)).toBe("stopped"); + expect(existsSync(worktree)).toBe(true); + const resumed = await drain(restarted.resume(o.sessionId)); + expect(resumed.at(-1)!.type).toBe("approval.requested"); + }); + + it("status says where a session stands, and undefined for one it doesn't know", async () => { + const roots = sessionRootOf(); + const { o } = await finished(roots); + expect(newRunner(roots).status(o.sessionId)).toBe("awaiting_approval"); + expect(newRunner(roots).status(randomUUID())).toBeUndefined(); + }); }); describe("runner memory", () => {