diff --git a/runner/package.json b/runner/package.json index 07216d8..d66ae8e 100644 --- a/runner/package.json +++ b/runner/package.json @@ -19,7 +19,8 @@ "test": "vitest run", "fake": "tsx scripts/fake.ts", "try": "tsx scripts/try.ts", - "sandbox": "tsx scripts/sandbox.ts" + "sandbox": "tsx scripts/sandbox.ts", + "engine": "tsx scripts/engine.ts" }, "dependencies": { "@codehands/contracts": "file:../../codehands-contracts", diff --git a/runner/scripts/engine.ts b/runner/scripts/engine.ts new file mode 100644 index 0000000..a9ae14a --- /dev/null +++ b/runner/scripts/engine.ts @@ -0,0 +1,57 @@ +// pnpm engine [repoPath] [prompt] [--free | --canned] [--wrong-fix] [--approve | --reject]: one run through the engine, one event per line. +// --canned answers analyze, plan and review from canned text, so only the implement step costs money. +// --free costs nothing: canned thinking, your recorded Claude run as implement (it also makes the real +// one-line fix), and the real Docker box. Add --wrong-fix to see what failing tests look like. +// Without --approve or --reject it keeps the worktree. Run scripts/reset-fixture.sh afterwards. +import { randomUUID } from "node:crypto"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createCannedModel, createEngine, createRunner } from "../src/index.js"; + +const flags = process.argv.slice(2).filter((a) => a.startsWith("--")); +const [repoPath = join(homedir(), "codehands-fixtures", "date-bug"), prompt = "Fix the failing test. Run npm test to check."] = + process.argv.slice(2).filter((a) => !a.startsWith("--")); +const free = flags.includes("--free"); + +const canned = createCannedModel({ + analyze: "The test expects the day of the month in UTC. src/date.js probably reads it in local time.", + plan: "1. Read src/date.js and test/date.test.js\n2. Read the day in UTC\n3. Run npm test", + review: `VERDICT: ${flags.includes("--wrong-fix") ? "concerns" : "approve"}\n(canned review: no model was asked)`, +}); + +// --free: instead of Claude Code, node prints your recorded run. First it makes the change the +// recorded run made: getDate() becomes getUTCDate() in src/date.js. --wrong-fix only adds a comment. +const recording = fileURLToPath(new URL("../test/fixtures/spike-1.jsonl", import.meta.url)); +const [from, to] = flags.includes("--wrong-fix") ? ["getDate();", "getDate(); // the day"] : ["getDate()", "getUTCDate()"]; +const replay = `const fs=require("fs"),f="src/date.js";if(fs.existsSync(f))fs.writeFileSync(f,fs.readFileSync(f,"utf8").replace(${JSON.stringify(from)},${JSON.stringify(to)}));process.stdout.write(fs.readFileSync(process.argv[1],"utf8"))`; +const runner = free ? createRunner({ agentCommand: () => ({ file: process.execPath, args: ["-e", replay, recording] }) }) : undefined; + +const engine = createEngine({ runner, model: free || flags.includes("--canned") ? canned : undefined }); +const sessionId = randomUUID(); + +let finished = false; +for await (const e of engine.start({ sessionId, repoPath, prompt, agent: "codehands", model: "sonnet", budgetUsd: 0.5 })) { + const p = e.payload; + const step = typeof p.step === "string" ? `[${p.step}]` : ""; + const detail = p.reason ?? p.command ?? p.path ?? p.model ?? (typeof p.text === "string" ? p.text.slice(0, 70) : ""); + const extra = e.type === "step.finished" ? ` ok=${p.ok}${p.costUsd ? ` $${Number(p.costUsd).toFixed(4)}` : ""}` + : e.type === "test.result" && p.step === "test" ? ` ran=${p.ran} passed=${p.passed}${p.error ? ` ${p.error}` : ""}` + : e.type === "task.finished" ? ` verdict=${p.verdict} $${Number(p.total_cost_usd).toFixed(4)}` : ""; + console.log(`${String(e.seq).padStart(3)} ${e.type.padEnd(17)} ${step.padEnd(12)} ${String(detail).replace(/\s+/g, " ")}${extra}`); + if (e.type === "task.finished") finished = true; +} +if (!finished) process.exit(1); + +console.log(`\nsession: ${sessionId}`); +console.log(`step files: ${join(homedir(), ".codehands", "runs", sessionId)}`); +if (free) console.log("cost: $0. The cost above is the recorded run's, replayed"); +if (flags.includes("--approve")) { + const { branch, commit } = await engine.approve(sessionId, prompt); + console.log(`approved: ${branch} @ ${commit.slice(0, 7)}`); +} else if (flags.includes("--reject")) { + await engine.reject(sessionId); + console.log("rejected: worktree and branch deleted"); +} else { + console.log("kept: the worktree is still there, see `git worktree list` in the repo"); +} \ No newline at end of file diff --git a/runner/src/engine/engine.ts b/runner/src/engine/engine.ts new file mode 100644 index 0000000..c5c25f8 --- /dev/null +++ b/runner/src/engine/engine.ts @@ -0,0 +1,246 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { RunnerEvent, RunnerEventType, StartOptions, StepName } from "@codehands/contracts"; +import { EventSequence } from "../events.js"; +import { baseCommit, git } from "../git/worktree.js"; +import { createFileMemory, repoKey } from "../memory/notes.js"; +import { detectRepoProfile } from "../memory/profile.js"; +import { notesForPrompt, type MemoryStore } from "../memory/store.js"; +import { createRunner, type RecoverableRunner } from "../runner.js"; +import { runInSandbox, type SandboxOptions, type SandboxResult } from "../runtime/sandbox.js"; +import { createClaudeModel, type ModelClient } from "./model.js"; +import { analyzePrompt, implementPrompt, parseVerdict, planPrompt, reviewPrompt, type Prompt } from "./prompts.js"; + +// Engine v1: analyze → plan → implement → test → review → awaiting approval. +// The same Runner interface as the runner, so the daemon picks it for agent "codehands". +// One rule: once the change exists, nothing after it throws it away. Tests that can't run +// and a review that fails still end in awaiting approval, with the reason in the events. + +export const DEFAULT_MODEL = "claude-sonnet-5-5"; +const SAFE_ID = /^[\w-]+$/; // the session id names a folder + +export interface EngineOptions { + runner?: RecoverableRunner; // does implement, diff, approve and reject. Default createRunner() + model?: ModelClient; // answers analyze, plan and review. Default createClaudeModel() + sandbox?: (o: SandboxOptions) => Promise; // the test step. Default runInSandbox + memory?: MemoryStore; // what earlier runs learned. Default createFileMemory() + runsRoot?: string; // step files. Default ~/.codehands/runs +} + +interface EngineRun { + options: StartOptions; + seq: EventSequence; + step?: StepName; // the step running now + stopped: boolean; // stopped during implement, waiting for resume() + done: boolean; // approval.requested went out + plan: string; + files: number; // step files written so far: 1-analyze.json, 2-plan.json … + costUsd: number; +} + +type Outcome = { ok: true; text: string } | { ok: false; reason: string }; + +// Which model a step uses: its own, then the run's, then the default. +export const modelFor = (o: StartOptions, step: StepName) => o.models?.[step] ?? o.model ?? DEFAULT_MODEL; + +export function createEngine({ + runner = createRunner(), + model = createClaudeModel(), + sandbox = runInSandbox, + memory = createFileMemory(), + runsRoot = join(homedir(), ".codehands", "runs"), +}: EngineOptions = {}): RecoverableRunner { + const runs = new Map(); + + function get(sessionId: string): EngineRun { + const run = runs.get(sessionId); + if (!run) throw new Error(`unknown engine session ${sessionId}`); + return run; + } + + const emit = (run: EngineRun, type: RunnerEventType, payload: Record = {}, raw?: unknown) => + run.seq.next(type, payload, raw); + + // Every step leaves a file: what went in, what came out, cost and time. Week 4 resumes from these. + // A step file must never fail a run, so errors are swallowed. + async function saveStep(run: EngineRun, step: StepName, record: Record): Promise { + run.files += 1; + const dir = join(runsRoot, run.options.sessionId); + try { + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, `${run.files}-${step}.json`), JSON.stringify({ step, ...record }, null, 2)); + } catch { } + } + + // analyze, plan and review: one model call each, with no tools. + async function* think(run: EngineRun, step: StepName, p: Prompt): AsyncGenerator { + run.step = step; + const m = modelFor(run.options, step); + yield emit(run, "step.started", { step, model: m }); + const started = Date.now(); + try { + const reply = await model.complete({ step, model: m, ...p }); + run.costUsd += reply.costUsd; + await saveStep(run, step, { ok: true, model: m, durationMs: Date.now() - started, costUsd: reply.costUsd, input: p, output: reply.text }); + yield emit(run, "message", { step, text: reply.text }); + yield emit(run, "step.finished", { step, ok: true, costUsd: reply.costUsd }); + return { ok: true, text: reply.text }; + } catch (err) { + const reason = (err as Error).message; + await saveStep(run, step, { ok: false, model: m, durationMs: Date.now() - started, input: p, error: reason }); + yield emit(run, "step.finished", { step, ok: false }); + return { ok: false, reason }; + } + } + + // Claude Code does the work in the runner's worktree. Its events go out as they arrive, renumbered + // and tagged with the step. The runner's own task.started, task.finished and approval.requested + // stay inside: the engine sends its own when the whole run is done. + async function* implement(run: EngineRun, inner: AsyncIterable, prompt: string): AsyncGenerator { + run.step = "implement"; + run.stopped = false; + const m = modelFor(run.options, "implement"); + yield emit(run, "step.started", { step: "implement", model: m }); + const started = Date.now(); + for await (const e of inner) { + if (e.type === "approval.requested") continue; + if (e.type === "task.started") { + yield emit(run, "message", { step: "implement", text: "Claude Code started" }, e.raw); + } else if (e.type === "task.finished") { + const costUsd = Number(e.payload.total_cost_usd ?? 0); + run.costUsd += costUsd; + await saveStep(run, "implement", { ok: true, model: m, durationMs: Date.now() - started, costUsd, input: prompt, output: e.payload.result, diffStat: e.payload.diffStat }); + yield emit(run, "step.finished", { step: "implement", ok: true, costUsd }); + return "done"; + } else if (e.type === "task.stopped") { + run.stopped = true; + yield emit(run, "step.finished", { step: "implement", ok: false }); + yield emit(run, "task.stopped", { step: "implement" }); + return "stopped"; + } else if (e.type === "task.failed") { + await saveStep(run, "implement", { ok: false, model: m, durationMs: Date.now() - started, input: prompt, error: e.payload.reason }); + yield emit(run, "step.finished", { step: "implement", ok: false }); + yield emit(run, "task.failed", { ...e.payload, step: "implement" }); + return "failed"; + } else { + yield emit(run, e.type, { ...e.payload, step: "implement" }, e.raw); + } + } + yield emit(run, "task.failed", { reason: "implement ended without a result", step: "implement" }); + return "failed"; + } + + // test and review, then the end of the run. Shared by start() and resume(). + async function* finish(run: EngineRun): AsyncGenerator { + const { sessionId } = run.options; + + run.step = "test"; + yield emit(run, "step.started", { step: "test" }); + const started = Date.now(); + const worktree = runner.worktree(sessionId); + let result: SandboxResult; + try { + if (!worktree) throw new Error("the runner has no worktree for this session"); + result = await sandbox({ sessionId, worktree, profile: await detectRepoProfile(worktree), runsRoot }); + } catch (err) { + // runInSandbox never throws, but a stand-in might. + result = { command: "", image: "", passed: false, exitCode: null, timedOut: false, oomKilled: false, durationMs: 0, output: "", outputTruncated: false, error: "sandbox.failed", message: (err as Error).message }; + } + const ran = !result.error; + const { output: _output, ...summary } = result; // the output itself is in result.resultFile + await saveStep(run, "test", { ok: ran, durationMs: Date.now() - started, result: summary }); + yield emit(run, "test.result", { step: "test", ran, ...summary }); + yield emit(run, "step.finished", { step: "test", ok: ran }); + + const { patch, stat } = await runner.diff(sessionId); + const review = yield* think(run, "review", reviewPrompt(run.options.prompt, run.plan, patch, result)); + const verdict = review.ok ? parseVerdict(review.text) : "unknown"; + + run.step = undefined; + run.done = true; + yield emit(run, "task.finished", { + diffStat: stat, + verdict, + review: review.ok ? review.text : `review failed: ${review.reason}`, + tests: { ran, passed: result.passed }, + total_cost_usd: run.costUsd, + }); + yield emit(run, "approval.requested", { diffStat: stat, verdict }); + } + + const fail = (run: EngineRun, step: StepName | undefined, reason: string) => + emit(run, "task.failed", { reason: step ? `${step}: ${reason}` : reason, step }); + + return { + async *start(o) { + if (runs.has(o.sessionId)) throw new Error(`session ${o.sessionId} already exists`); + const run: EngineRun = { options: o, seq: new EventSequence(o.sessionId), stopped: false, done: false, plan: "", files: 0, costUsd: 0 }; + runs.set(o.sessionId, run); + + if (o.agent !== "codehands") { + yield fail(run, undefined, `the engine runs agent "codehands", not "${o.agent}"`); + return; + } + if (!SAFE_ID.test(o.sessionId)) { + yield fail(run, undefined, `bad session id: ${o.sessionId}`); + return; + } + yield emit(run, "task.started", { agent: "codehands", prompt: o.prompt }); + + try { + // Check the repo before paying for any model call: implement needs a commit to branch from. + await baseCommit(o.repoPath); + const profile = await detectRepoProfile(o.repoPath); + const files = (await git(o.repoPath, ["ls-files"]).catch(() => "")).split("\n").filter(Boolean); + const notes = await memory.notes(await repoKey(o.repoPath)).then(notesForPrompt).catch(() => undefined); + + const analysis = yield* think(run, "analyze", analyzePrompt(o.prompt, profile, files, notes)); + if (!analysis.ok) return void (yield fail(run, "analyze", analysis.reason)); + + const plan = yield* think(run, "plan", planPrompt(o.prompt, analysis.text)); + if (!plan.ok) return void (yield fail(run, "plan", plan.reason)); + run.plan = plan.text; + + const prompt = implementPrompt(o.prompt, plan.text); + const outcome = yield* implement(run, runner.start({ ...o, agent: "claude-code", prompt, model: modelFor(o, "implement") }), prompt); + if (outcome === "done") yield* finish(run); + } catch (err) { + yield fail(run, run.step, (err as Error).message); + } + }, + + // v1 can only stop while Claude Code works. analyze, plan and review are single short calls. + async stop(sessionId) { + const run = get(sessionId); + if (run.step !== "implement" || run.stopped) { + throw new Error(`the engine can only stop during implement (now: ${run.stopped ? "stopped" : run.step ?? "not running"})`); + } + await runner.stop(sessionId); + }, + + async *resume(sessionId, prompt) { + const run = get(sessionId); + if (!run.stopped) throw new Error(`session ${sessionId} is not stopped`); + try { + const outcome = yield* implement(run, runner.resume(sessionId, prompt), prompt ?? "Continue."); + if (outcome === "done") yield* finish(run); + } catch (err) { + yield fail(run, run.step, (err as Error).message); + } + }, + + diff: (sessionId) => runner.diff(sessionId), + + // The runner already allows approve after implement. The engine waits for test and review. + async approve(sessionId, message) { + const run = runs.get(sessionId); + if (run && !run.done) throw new Error(`the engine hasn't finished ${sessionId} yet (now: ${run.stopped ? "stopped" : run.step ?? "not running"})`); + return runner.approve(sessionId, message); + }, + + reject: (sessionId) => runner.reject(sessionId), + recover: () => runner.recover(), + worktree: (sessionId) => runner.worktree(sessionId), + }; +} \ No newline at end of file diff --git a/runner/src/engine/model.ts b/runner/src/engine/model.ts new file mode 100644 index 0000000..228a297 --- /dev/null +++ b/runner/src/engine/model.ts @@ -0,0 +1,89 @@ +import { tmpdir } from "node:os"; +import type { StepName } from "@codehands/contracts"; +import { execa } from "execa"; + +// What the engine needs from a model: text in, text out, and what it cost. +// Tania's model code plugs in here later. Tests use createCannedModel(). +export interface ModelRequest { + step: StepName; + model: string; + system: string; + prompt: string; +} + +export interface ModelReply { + text: string; + costUsd: number; +} + +export interface ModelClient { + complete(req: ModelRequest): Promise; +} + +export interface ClaudeModelOptions { + budgetUsd?: number; // per call, default 0.2 + // Which program to run. Tests swap in a script that prints a canned reply. + command?: (args: string[]) => { file: string; args: string[] }; +} + +// One plain answer from Claude Code: no tools, our own system prompt, nothing saved. +// --tools "" means the model can only think and write, never touch files. +export function claudeModelArgs(req: ModelRequest, budgetUsd: number): string[] { + return [ + "-p", req.prompt, + "--output-format", "json", + "--tools", "", + "--system-prompt", req.system, + "--model", req.model, + "--strict-mcp-config", + "--no-session-persistence", + "--max-budget-usd", String(budgetUsd), + ]; +} + +// Runs in the temp folder, so no CLAUDE.md from a repo is read into the answer. +export function createClaudeModel({ budgetUsd = 0.2, command = (args) => ({ file: "claude", args }) }: ClaudeModelOptions = {}): ModelClient { + return { + async complete(req) { + const { file, args } = command(claudeModelArgs(req, budgetUsd)); + // execa can still throw, before it even starts: when the program isn't found, Windows + // falls back to cmd.exe, which refuses a prompt with newlines in it. + let r; + try { + r = await execa(file, args, { cwd: tmpdir(), reject: false, stdin: "ignore", timeout: 5 * 60_000, windowsHide: true }); + } catch (err) { + throw new Error(`claude could not start: ${(err as Error).message.split("\n")[0]}`); + } + return parseClaudeJson(String(r.stdout ?? ""), String(r.stderr ?? ""), r.exitCode); + }, + }; +} + +// --output-format json prints one object, the same shape as the `result` line in stream-json. +export function parseClaudeJson(stdout: string, stderr: string, exitCode?: number): ModelReply { + let o: { result?: unknown; is_error?: boolean; subtype?: string; total_cost_usd?: number }; + try { + o = JSON.parse(stdout); + } catch { + throw new Error(`claude gave no answer (exit ${exitCode ?? "?"}): ${stderr.trim().slice(-300) || stdout.trim().slice(-300)}`); + } + if (o.is_error || typeof o.result !== "string") { + throw new Error(`claude failed: ${typeof o.result === "string" ? o.result : o.subtype ?? "no result"}`); + } + return { text: o.result, costUsd: Number(o.total_cost_usd ?? 0) }; +} + +// Canned answers per step, for tests and free demos. An Error makes that step fail. +// calls keeps every request, so a test can check what each step was asked. +export function createCannedModel(answers: Partial>): ModelClient & { calls: ModelRequest[] } { + const calls: ModelRequest[] = []; + return { + calls, + async complete(req) { + calls.push(req); + const answer = answers[req.step]; + if (answer instanceof Error) throw answer; + return { text: answer ?? `(canned ${req.step})`, costUsd: 0 }; + }, + }; +} \ No newline at end of file diff --git a/runner/src/engine/prompts.ts b/runner/src/engine/prompts.ts new file mode 100644 index 0000000..2059538 --- /dev/null +++ b/runner/src/engine/prompts.ts @@ -0,0 +1,67 @@ +import type { RepoProfile } from "../memory/profile.js"; +import type { SandboxResult } from "../runtime/sandbox.js"; + +// What each step asks. Pure functions, so a test can read every prompt without a model. +// Caps keep each prompt well under Windows' 32,767-character command line. + +export const MAX_FILES = 200; +export const MAX_PATCH = 12_000; +export const MAX_TEST_OUTPUT = 3_000; + +export interface Prompt { + system: string; + prompt: string; +} + +const tail = (text: string, max: number) => (text.length > max ? `…(cut, last ${max} characters)\n${text.slice(-max)}` : text); +const head = (text: string, max: number) => (text.length > max ? `${text.slice(0, max)}\n…(cut at ${max} characters)` : text); + +export function analyzePrompt(task: string, profile: RepoProfile, files: string[], notes?: string): Prompt { + const list = files.slice(0, MAX_FILES).join("\n") + (files.length > MAX_FILES ? `\n…and ${files.length - MAX_FILES} more` : ""); + return { + system: "You analyze a coding task before anyone changes code. Be short and concrete. Don't write code.", + prompt: [ + `Task: ${task}`, + `Repo: ${profile.language}, package manager ${profile.packageManager ?? "none"}, tests: ${profile.testCommand ?? "none"}, Node ${profile.nodeVersion}`, + `Files:\n${list || "(none)"}`, + notes ? `What earlier runs learned about this repo:\n${notes}` : "", + "Say in at most 8 lines: what the task really asks, which files probably matter, and what could go wrong.", + ].filter(Boolean).join("\n\n"), + }; +} + +export function planPrompt(task: string, analysis: string): Prompt { + return { + system: "You plan a small code change for another agent to carry out. Number the steps. Don't write code.", + prompt: `Task: ${task}\n\nAnalysis:\n${analysis}\n\nWrite a plan of at most 6 numbered steps. The last step is always running the tests.`, + }; +} + +// The implement step's prompt goes to Claude Code, which has tools and does the work. +export function implementPrompt(task: string, plan: string): string { + return `${task}\n\nFollow this plan:\n${plan}\n\nKeep the change as small as the task allows.`; +} + +export function reviewPrompt(task: string, plan: string, patch: string, test: SandboxResult): Prompt { + const tests = test.error + ? `The tests did not run: ${test.error}. ${test.message ?? ""}` + : `The tests ${test.passed ? "passed" : "failed"} (exit ${test.exitCode}${test.timedOut ? ", timed out" : ""}${test.oomKilled ? ", out of memory" : ""}). Output:\n${tail(test.output, MAX_TEST_OUTPUT)}`; + return { + system: "You review a code change for a person who will approve or reject it. Be short and honest.", + prompt: [ + `Task: ${task}`, + `Plan:\n${plan}`, + `The change:\n${head(patch, MAX_PATCH) || "(no change)"}`, + tests, + 'Start with exactly "VERDICT: approve" or "VERDICT: concerns", then at most 6 lines on why.', + ].join("\n\n"), + }; +} + +export type Verdict = "approve" | "concerns" | "unknown"; + +// "unknown" when the model didn't follow the format, or the review step couldn't run. +export function parseVerdict(text: string): Verdict { + const m = /^\s*\**\s*verdict\s*:\s*\**\s*(approve|concerns)\b/im.exec(text); + return m ? (m[1]!.toLowerCase() as Verdict) : "unknown"; +} \ No newline at end of file diff --git a/runner/src/index.ts b/runner/src/index.ts index ba52842..3585217 100644 --- a/runner/src/index.ts +++ b/runner/src/index.ts @@ -8,3 +8,6 @@ export { createFileMemory, repoKey } from "./memory/notes.js"; export { detectRepoProfile, type RepoProfile } from "./memory/profile.js"; export { runInSandbox, type SandboxOptions, type SandboxResult, type SandboxError } from "./runtime/sandbox.js"; export { dockerArgs, installArgs, dockerAvailable, hostTimeZone, DEFAULT_LIMITS, type BoxLimits } from "./runtime/docker.js"; +export { createEngine, modelFor, DEFAULT_MODEL, type EngineOptions } from "./engine/engine.js"; +export { createClaudeModel, createCannedModel, claudeModelArgs, parseClaudeJson, type ModelClient, type ModelRequest, type ModelReply, type ClaudeModelOptions } from "./engine/model.js"; +export { analyzePrompt, planPrompt, implementPrompt, reviewPrompt, parseVerdict, type Prompt, type Verdict } from "./engine/prompts.js"; diff --git a/runner/src/runner.ts b/runner/src/runner.ts index f633f3d..226eda4 100644 --- a/runner/src/runner.ts +++ b/runner/src/runner.ts @@ -36,6 +36,7 @@ export interface RecoveryReport { export interface RecoverableRunner extends Runner { recover(): Promise; + worktree(sessionId: string): string | undefined; } const claudeCommand = (o: StartOptions, notes?: string) => ({ file: "claude", args: claudeArgs(o, notes) }); @@ -201,6 +202,14 @@ export function createRunner({ return { start, + // Where the session's change is, for whoever runs the tests. undefined when there is none. + worktree(sessionId) { + try { + return store.get(sessionId).worktreePath; + } catch { + return undefined; + } + }, // Call once when the daemon starts, before it starts or commands anything. // A session still marked queued or running lost its agent, so it becomes failed. // awaiting_approval and stopped sessions are left alone: approve, reject and resume still work. diff --git a/runner/test/engine.test.ts b/runner/test/engine.test.ts new file mode 100644 index 0000000..c34220a --- /dev/null +++ b/runner/test/engine.test.ts @@ -0,0 +1,277 @@ +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { RunnerEvent, StartOptions } from "@codehands/contracts"; +import { afterAll, describe, expect, it } from "vitest"; +import { + createCannedModel, createEngine, createMemoryStub, createRunner, dockerAvailable, runInSandbox, + type ModelClient, type RunnerOptions, type SandboxOptions, type SandboxResult, +} from "../src/index.js"; + +// The whole engine with no cost: canned model answers, the recorded Claude run as the implement +// step, and a stand-in sandbox. One test at the end uses the real Docker box. +const replay = fileURLToPath(new URL("./helpers/replay-agent.mjs", import.meta.url)); +const spike = fileURLToPath(new URL("./fixtures/spike-1.jsonl", import.meta.url)); +const tmp = mkdtempSync(join(tmpdir(), "codehands-engine-")); +afterAll(() => rmSync(tmp, { recursive: true, force: true })); +const hasDocker = await dockerAvailable(); + +const git = (cwd: string, ...args: string[]) => + execFileSync("git", ["-c", "user.name=test", "-c", "user.email=test@example.com", "-c", "core.autocrlf=false", ...args], { cwd, encoding: "utf8" }).trim(); + +function makeRepo(withCommit = true): string { + const repo = mkdtempSync(join(tmp, "repo-")); + git(repo, "init", "-q", "-b", "main"); + if (withCommit) { + writeFileSync(join(repo, "a.txt"), "hello\n"); + // The test passes only once the agent has added "world" to a.txt. + writeFileSync(join(repo, "package.json"), JSON.stringify({ name: "r", private: true, scripts: { test: `node -e "process.exit(require('fs').readFileSync('a.txt','utf8').includes('world') ? 0 : 1)"` } })); + git(repo, "add", "-A"); + git(repo, "commit", "-q", "-m", "first"); + } + return repo; +} + +const options = (repoPath: string, extra: Partial = {}): StartOptions => + ({ sessionId: randomUUID(), repoPath, prompt: "Fix the failing test", agent: "codehands", ...extra }); + +const passing = (o: SandboxOptions): SandboxResult => ({ + command: "npm test", image: "node:22-slim", passed: true, exitCode: 0, timedOut: false, oomKilled: false, + durationMs: 900, output: `# pass 1 in ${o.worktree}`, outputTruncated: false, +}); + +interface Setup { + model?: ModelClient; + sandbox?: (o: SandboxOptions) => Promise; + runner?: RunnerOptions; +} + +// seen keeps what the implement step was given, so a test can check the prompt and model. +function setup({ model = createCannedModel({ analyze: "ANALYSIS", plan: "1. PLAN", review: "VERDICT: approve\nGood." }), sandbox = async (o) => passing(o), runner = {} }: Setup = {}) { + const seen: StartOptions[] = []; + const runsRoot = mkdtempSync(join(tmp, "runs-")); + const inner = createRunner({ + worktreeRoot: join(tmp, "worktrees"), + sessionRoot: join(tmp, "sessions"), + memory: createMemoryStub(), + agentCommand: (o) => { + seen.push(o); + return { file: process.execPath, args: [replay, spike, "0", "edit"] }; + }, + ...runner, + }); + const engine = createEngine({ runner: inner, model, sandbox, memory: createMemoryStub(), runsRoot }); + return { engine, seen, runsRoot }; +} + +async function drain(run: AsyncIterable): Promise { + const events: RunnerEvent[] = []; + for await (const e of run) events.push(e); + return events; +} + +// The engine's own test.result. Claude Code's run sends some too, tagged step "implement". +const testResult = (events: RunnerEvent[]) => events.find((e) => e.type === "test.result" && e.payload.step === "test")!.payload; + +const steps = (events: RunnerEvent[]) => + events.filter((e) => e.type === "step.started" || e.type === "step.finished").map((e) => `${e.type === "step.started" ? ">" : "<"}${e.payload.step}${e.payload.ok === false ? " x" : ""}`); + +describe("engine.start", () => { + it("walks the five steps in order and ends awaiting approval", async () => { + const { engine } = setup(); + const o = options(makeRepo()); + const events = await drain(engine.start(o)); + + expect(steps(events)).toEqual([">analyze", "plan", "implement", "test", "review", " e.type).slice(-2)).toEqual(["task.finished", "approval.requested"]); + expect(events.at(-1)!.payload).toMatchObject({ verdict: "approve", diffStat: expect.stringMatching(/2 files changed/) }); + expect(events.map((e) => e.seq)).toEqual(events.map((_, i) => i + 1)); + expect(events.every((e) => e.sessionId === o.sessionId)).toBe(true); + }); + + it("passes Claude Code's own events through, tagged with the step", async () => { + const { engine } = setup(); + const events = await drain(engine.start(options(makeRepo()))); + const fromClaude = events.filter((e) => e.payload.step === "implement" && !e.type.startsWith("step.")); + expect(fromClaude.map((e) => e.type)).toContain("file.changed"); + expect(fromClaude.every((e) => e.raw !== undefined)).toBe(true); + expect(events.filter((e) => e.type === "task.finished")).toHaveLength(1); // the runner's own stays inside + }); + + it("hands each step the one before it: analysis to plan, plan to implement, the change to review", async () => { + const model = createCannedModel({ analyze: "ANALYSIS", plan: "1. PLAN", review: "VERDICT: approve" }); + const { engine, seen } = setup({ model }); + await drain(engine.start(options(makeRepo()))); + const [analyze, plan, review] = model.calls; + expect(analyze!.prompt).toContain("Fix the failing test"); + expect(analyze!.prompt).toContain("package.json"); + expect(plan!.prompt).toContain("ANALYSIS"); + expect(seen[0]!.prompt).toContain("1. PLAN"); + expect(seen[0]!.agent).toBe("claude-code"); + expect(review!.prompt).toContain("+world"); + expect(review!.prompt).toContain("The tests passed"); + }); + + it("uses each step's own model, then the run's, then the default", async () => { + const model = createCannedModel({}); + const { engine, seen } = setup({ model }); + await drain(engine.start(options(makeRepo(), { model: "run-model", models: { plan: "plan-model", implement: "impl-model" } }))); + expect(model.calls.map((c) => [c.step, c.model])).toEqual([["analyze", "run-model"], ["plan", "plan-model"], ["review", "run-model"]]); + expect(seen[0]!.model).toBe("impl-model"); + + const plain = createCannedModel({}); + await drain(setup({ model: plain }).engine.start(options(makeRepo()))); + expect(plain.calls[0]!.model).toBe("claude-sonnet-5-5"); + }); + + it("saves a file for every step", async () => { + const { engine, runsRoot } = setup(); + const o = options(makeRepo()); + await drain(engine.start(o)); + const dir = join(runsRoot, o.sessionId); + expect(readdirSync(dir).sort()).toEqual(["1-analyze.json", "2-plan.json", "3-implement.json", "4-test.json", "5-review.json"]); + expect(JSON.parse(readFileSync(join(dir, "2-plan.json"), "utf8"))).toMatchObject({ step: "plan", ok: true, output: "1. PLAN" }); + expect(JSON.parse(readFileSync(join(dir, "3-implement.json"), "utf8")).diffStat).toMatch(/2 files changed/); + }); + + it("refuses an agent that isn't codehands", async () => { + const events = await drain(setup().engine.start(options(makeRepo(), { agent: "claude-code" }))); + expect(events.map((e) => [e.type, e.payload.reason])).toEqual([["task.failed", 'the engine runs agent "codehands", not "claude-code"']]); + }); + + it("fails before any model call when the repo has no commits", async () => { + const model = createCannedModel({}); + const events = await drain(setup({ model }).engine.start(options(makeRepo(false)))); + expect(events.at(-1)).toMatchObject({ type: "task.failed", payload: { reason: expect.stringMatching(/no commits/) } }); + expect(model.calls).toHaveLength(0); + }); +}); + +describe("engine when something goes wrong", () => { + it("analyze fails: the run fails, and no worktree is made", async () => { + const { engine } = setup({ model: createCannedModel({ analyze: new Error("model down") }) }); + const o = options(makeRepo()); + const events = await drain(engine.start(o)); + expect(steps(events)).toEqual([">analyze", " { + const empty = join(tmp, "empty.jsonl"); + writeFileSync(empty, ""); + const { engine } = setup({ runner: { agentCommand: () => ({ file: process.execPath, args: [replay, empty, "1"] }) } }); + const events = await drain(engine.start(options(makeRepo()))); + expect(steps(events).at(-1)).toBe(" { + const sandbox = async (o: SandboxOptions) => ({ ...passing(o), passed: false, exitCode: 1, output: "# fail 1" }); + const model = createCannedModel({ review: "VERDICT: concerns\nThe test still fails." }); + const events = await drain(setup({ model, sandbox }).engine.start(options(makeRepo()))); + expect(testResult(events)).toMatchObject({ ran: true, passed: false, exitCode: 1 }); + expect(model.calls.at(-1)!.prompt).toContain("The tests failed (exit 1)"); + expect(events.at(-1)).toMatchObject({ type: "approval.requested", payload: { verdict: "concerns" } }); + }); + + it("tests can't run: the change is kept, and review is told why", async () => { + const sandbox = async (o: SandboxOptions) => ({ ...passing(o), passed: false, exitCode: null, error: "sandbox.unavailable" as const, message: "Docker is not running." }); + const model = createCannedModel({ review: "VERDICT: concerns\nUntested." }); + const events = await drain(setup({ model, sandbox }).engine.start(options(makeRepo()))); + expect(steps(events)).toContain(" { + const sandbox = async () => { throw new Error("boom"); }; + const events = await drain(setup({ sandbox }).engine.start(options(makeRepo()))); + expect(testResult(events)).toMatchObject({ ran: false, error: "sandbox.failed", message: "boom" }); + expect(events.at(-1)!.type).toBe("approval.requested"); + }); + + it("review fails: the change is kept, with verdict unknown", async () => { + const { engine } = setup({ model: createCannedModel({ review: new Error("model down") }) }); + const events = await drain(engine.start(options(makeRepo()))); + expect(steps(events).at(-1)).toBe(" e.type === "task.finished")!.payload).toMatchObject({ verdict: "unknown", review: "review failed: model down" }); + expect(events.at(-1)).toMatchObject({ type: "approval.requested", payload: { verdict: "unknown" } }); + }); +}); + +describe("engine approve, reject, stop and resume", () => { + it("approve waits for the whole run, then commits on the task branch", async () => { + const { engine } = setup(); + const repo = makeRepo(); + const o = options(repo); + const run = engine.start(o)[Symbol.asyncIterator](); + let e = await run.next(); + while (!e.done && e.value.type !== "step.started") e = await run.next(); + await expect(engine.approve(o.sessionId, "too early")).rejects.toThrow(/hasn't finished/); + + for (e = await run.next(); !e.done; e = await run.next()); + const { branch, commit } = await engine.approve(o.sessionId, "Fix the failing test"); + expect(git(repo, "rev-parse", branch)).toBe(commit); + }); + + it("reject removes the worktree and the branch", async () => { + const { engine } = setup(); + const repo = makeRepo(); + const o = options(repo); + await drain(engine.start(o)); + const worktree = engine.worktree(o.sessionId)!; + await engine.reject(o.sessionId); + expect(existsSync(worktree)).toBe(false); + expect(git(repo, "branch", "--list", "codehands/*")).toBe(""); + }); + + it("stops during implement, then resume carries on through test and review", async () => { + const pidFile = join(tmp, `engine-child-${randomUUID()}.pid`); + const { engine } = setup({ + runner: { + agentCommand: () => ({ file: process.execPath, args: [replay, spike, "0", "hang", pidFile] }), + resumeCommand: () => ({ file: process.execPath, args: [replay, spike, "0", "edit"] }), + }, + }); + const o = options(makeRepo()); + const first: RunnerEvent[] = []; + let stopping: Promise | undefined; + for await (const e of engine.start(o)) { + first.push(e); + if (e.type === "message" && e.payload.text === "Claude Code started") stopping = engine.stop(o.sessionId); + } + await stopping; + expect(first.map((e) => e.type).slice(-2)).toEqual(["step.finished", "task.stopped"]); + await expect(engine.stop(o.sessionId)).rejects.toThrow(/only stop during implement/); + + const rest = await drain(engine.resume(o.sessionId)); + expect(steps(rest)).toEqual([">implement", "test", "review", " e.seq)).toEqual(all.map((_, i) => i + 1)); + }); + + it("refuses to stop outside implement, and to resume what isn't stopped", async () => { + const { engine } = setup(); + const o = options(makeRepo()); + await drain(engine.start(o)); + await expect(engine.stop(o.sessionId)).rejects.toThrow(/only stop during implement \(now: not running\)/); + await expect(drain(engine.resume(o.sessionId))).rejects.toThrow(/is not stopped/); + await expect(engine.stop(randomUUID())).rejects.toThrow(/unknown engine session/); + }); +}); + +describe.skipIf(!hasDocker)("engine with the real Docker box", { timeout: 60_000 }, () => { + it("runs the tests in the box, in the worktree with the agent's change", async () => { + const { engine } = setup({ sandbox: runInSandbox }); + const events = await drain(engine.start(options(makeRepo()))); + expect(testResult(events)).toMatchObject({ ran: true, passed: true, exitCode: 0 }); + expect(events.at(-1)!.type).toBe("approval.requested"); + }); +}); \ No newline at end of file diff --git a/runner/test/helpers/fake-claude-json.mjs b/runner/test/helpers/fake-claude-json.mjs new file mode 100644 index 0000000..b74a5ef --- /dev/null +++ b/runner/test/helpers/fake-claude-json.mjs @@ -0,0 +1,16 @@ +// Stand-in for `claude -p --output-format json`: prints one result object, like the real one. +// node fake-claude-json.mjs ...claudeArgs +// With "ok", result is the arguments it got, so a test can check every flag and the prompt. +const [mode, ...args] = process.argv.slice(2); +if (mode === "garbage") { + process.stdout.write("this is not json"); + process.exitCode = 1; +} else { + process.stdout.write(JSON.stringify({ + type: "result", + subtype: mode === "error" ? "error_max_budget_usd" : "success", + is_error: mode === "error", + result: mode === "error" ? "budget exceeded" : JSON.stringify(args), + total_cost_usd: 0.0012, + })); +} \ No newline at end of file diff --git a/runner/test/model.test.ts b/runner/test/model.test.ts new file mode 100644 index 0000000..e694b63 --- /dev/null +++ b/runner/test/model.test.ts @@ -0,0 +1,64 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { claudeModelArgs, createCannedModel, createClaudeModel, parseClaudeJson, type ModelRequest } from "../src/index.js"; + +const fake = fileURLToPath(new URL("./helpers/fake-claude-json.mjs", import.meta.url)); +const spike = fileURLToPath(new URL("./fixtures/spike-1.jsonl", import.meta.url)); +const req: ModelRequest = { step: "plan", model: "claude-haiku-5-5", system: "You plan.", prompt: 'Line one\nLine "two" with $HOME and %PATH%' }; +const fakeModel = (mode: string) => createClaudeModel({ command: (args) => ({ file: process.execPath, args: [fake, mode, ...args] }) }); +const pair = (args: string[], flag: string) => args[args.indexOf(flag) + 1]; + +describe("claudeModelArgs", () => { + it("asks for one plain answer: no tools, our system prompt, nothing saved", () => { + const a = claudeModelArgs(req, 0.2); + expect(pair(a, "-p")).toBe(req.prompt); + expect(pair(a, "--output-format")).toBe("json"); + expect(pair(a, "--tools")).toBe(""); + expect(pair(a, "--system-prompt")).toBe("You plan."); + expect(pair(a, "--model")).toBe("claude-haiku-5-5"); + expect(pair(a, "--max-budget-usd")).toBe("0.2"); + expect(a).toContain("--strict-mcp-config"); + expect(a).toContain("--no-session-persistence"); + }); +}); + +describe("createClaudeModel", () => { + it("passes the prompt through untouched, newlines and quotes included, and reads the cost", async () => { + const reply = await fakeModel("ok").complete(req); + expect(pair(JSON.parse(reply.text), "-p")).toBe(req.prompt); + expect(reply.costUsd).toBe(0.0012); + }); + + it("throws when Claude reports an error", async () => { + await expect(fakeModel("error").complete(req)).rejects.toThrow(/claude failed: budget exceeded/); + }); + + it("throws when Claude prints something that isn't JSON", async () => { + await expect(fakeModel("garbage").complete(req)).rejects.toThrow(/claude gave no answer \(exit 1\)/); + }); + + it("throws when Claude isn't installed", async () => { + const missing = createClaudeModel({ command: (args) => ({ file: "codehands-no-such-claude", args }) }); + await expect(missing.complete(req)).rejects.toThrow(/^claude (gave no answer|could not start)/); + }); +}); + +describe("parseClaudeJson", () => { + it("reads the result line of a real recorded run", () => { + const last = readFileSync(spike, "utf8").trim().split("\n").at(-1)!; + const reply = parseClaudeJson(last, ""); + expect(reply.text).toMatch(/npm test/); + expect(reply.costUsd).toBeGreaterThan(0); + }); +}); + +describe("createCannedModel", () => { + it("answers per step, records every call, and fails a step given an Error", async () => { + const model = createCannedModel({ plan: "1. do it", review: new Error("model down") }); + expect((await model.complete(req)).text).toBe("1. do it"); + expect((await model.complete({ ...req, step: "analyze" })).text).toBe("(canned analyze)"); + await expect(model.complete({ ...req, step: "review" })).rejects.toThrow("model down"); + expect(model.calls.map((c) => c.step)).toEqual(["plan", "analyze", "review"]); + }); +}); \ No newline at end of file diff --git a/runner/test/prompts.test.ts b/runner/test/prompts.test.ts new file mode 100644 index 0000000..8d31546 --- /dev/null +++ b/runner/test/prompts.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import type { RepoProfile, SandboxResult } from "../src/index.js"; +import { analyzePrompt, implementPrompt, parseVerdict, planPrompt, reviewPrompt } from "../src/index.js"; +import { MAX_FILES, MAX_PATCH } from "../src/engine/prompts.js"; + +const profile: RepoProfile = { language: "node", packageManager: "npm", testCommand: "npm test", nodeVersion: "22", needsInstall: false }; +const tests = (extra: Partial = {}): SandboxResult => ({ + command: "npm test", image: "node:22-slim", passed: true, exitCode: 0, timedOut: false, oomKilled: false, + durationMs: 900, output: "# pass 1", outputTruncated: false, ...extra, +}); + +describe("prompts", () => { + it("analyze sees the task, the repo, its files and its notes", () => { + const { prompt } = analyzePrompt("Fix the date bug", profile, ["src/date.js", "test/date.test.js"], "- Last time: use UTC"); + expect(prompt).toContain("Fix the date bug"); + expect(prompt).toContain("tests: npm test"); + expect(prompt).toContain("src/date.js"); + expect(prompt).toContain("use UTC"); + }); + + it("analyze lists at most 200 files and says how many it left out", () => { + const files = Array.from({ length: 250 }, (_, i) => `f${i}.js`); + const { prompt } = analyzePrompt("t", profile, files); + expect(prompt).toContain(`f${MAX_FILES - 1}.js`); + expect(prompt).not.toContain(`f${MAX_FILES}.js`); + expect(prompt).toContain("…and 50 more"); + }); + + it("plan builds on the analysis, and implement on the plan", () => { + expect(planPrompt("t", "ANALYSIS-TEXT").prompt).toContain("ANALYSIS-TEXT"); + expect(implementPrompt("Fix it", "1. PLAN-STEP")).toMatch(/^Fix it[\s\S]*1\. PLAN-STEP/); + }); + + it("review sees the change and the test result, and cuts a huge patch", () => { + const patch = "+line\n".repeat(5_000); + const { prompt } = reviewPrompt("t", "plan", patch, tests({ passed: false, exitCode: 1, output: "# fail 1" })); + expect(prompt).toContain("The tests failed (exit 1)"); + expect(prompt).toContain("# fail 1"); + expect(prompt).toContain(`cut at ${MAX_PATCH} characters`); + expect(prompt.length).toBeLessThan(20_000); + }); + + it("review says plainly when the tests never ran", () => { + const { prompt } = reviewPrompt("t", "plan", "+x", tests({ passed: false, exitCode: null, error: "sandbox.unavailable", message: "Docker is not running." })); + expect(prompt).toContain("The tests did not run: sandbox.unavailable. Docker is not running."); + }); +}); + +describe("parseVerdict", () => { + it("reads the verdict line, however the model dresses it up", () => { + expect(parseVerdict("VERDICT: approve\nLooks fine.")).toBe("approve"); + expect(parseVerdict("**Verdict:** concerns\nNo test for it.")).toBe("concerns"); + expect(parseVerdict("verdict: Approve")).toBe("approve"); + }); + + it("says unknown when there is no verdict line", () => { + expect(parseVerdict("Looks fine to me.")).toBe("unknown"); + expect(parseVerdict("")).toBe("unknown"); + }); +}); \ No newline at end of file