From 77447889e06d17ee47656c0fd63339d0467fe2e9 Mon Sep 17 00:00:00 2001 From: Tania Ashraf Date: Mon, 9 Nov 2026 10:46:38 +0500 Subject: [PATCH 1/2] runner: classify Claude login failures --- runner/src/agents/claude-code.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runner/src/agents/claude-code.ts b/runner/src/agents/claude-code.ts index d0c4c45..52d75ac 100644 --- a/runner/src/agents/claude-code.ts +++ b/runner/src/agents/claude-code.ts @@ -123,7 +123,8 @@ export function parseClaudeLine(line: string, seq: EventSequence, state = newCla } if (o.type === "result") { - const failed = o.is_error === true || o.subtype !== "success"; + const resultText = typeof o.result === "string" ? o.result : ""; + const failed = o.is_error === true || o.subtype !== "success" || /not logged in|please run \/login/i.test(resultText); return [seq.next(failed ? "task.failed" : "task.finished", { subtype: o.subtype, total_cost_usd: o.total_cost_usd, @@ -131,6 +132,7 @@ export function parseClaudeLine(line: string, seq: EventSequence, state = newCla num_turns: o.num_turns, usage: o.usage, result: o.result, + reason: failed ? resultText : undefined, agentSessionId: o.session_id, }, o)]; } From 1b0d0684bf8c4c58740351444a8a12de0b3ae00c Mon Sep 17 00:00:00 2001 From: imtahirabatool Date: Thu, 12 Nov 2026 11:29:00 +0500 Subject: [PATCH 2/2] claude parser fix --- runner/src/agents/claude-code.ts | 12 ++++++++---- runner/test/claude-parser.test.ts | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/runner/src/agents/claude-code.ts b/runner/src/agents/claude-code.ts index 52d75ac..b8665b3 100644 --- a/runner/src/agents/claude-code.ts +++ b/runner/src/agents/claude-code.ts @@ -42,7 +42,8 @@ export function claudeResumeArgs(sessionId: string, prompt: string, budgetUsd = // assistant tool_use Edit/Write -> file.changed // assistant tool_use Bash/PowerShell -> command.started (Claude uses PowerShell on Windows) // user tool_result for a command -> command.finished (+ test.result for a test command) -// result -> task.finished, or task.failed when it is an error. +// result -> task.finished, or task.failed when it is an error +// or a login failure reported as success. // total_cost_usd and usage exactly as given // Skipped: rate_limit_event (can come before init), thinking, other system lines. // A line that is not JSON becomes a `message` with the raw text. Always keep `raw`. @@ -54,6 +55,9 @@ const COMMAND_TOOLS = new Set(["Bash", "PowerShell"]); const TEST_COMMAND = /\b(npm|pnpm|yarn)( run)? test\b|\bnode --test\b|\b(vitest|jest|pytest)\b/; // Exit codes can hide behind a pipe (`npm test | tail`), so the output is checked too. const TEST_FAILED = /^# fail [1-9]|\b[1-9]\d* (failed|failing)\b|^FAIL\b/m; +// Claude can report a login problem as a "success" result. Match only that whole message, +// so a summary that merely mentions login ("users who are not logged in …") still finishes. +const AUTH_FAILURE = /^\s*(not logged in|invalid api key)\b.*please run \/login\s*$/i; // Results can arrive out of order, so a tool_result is matched to its tool_use by id. export interface ClaudeParseState { @@ -123,8 +127,8 @@ export function parseClaudeLine(line: string, seq: EventSequence, state = newCla } if (o.type === "result") { - const resultText = typeof o.result === "string" ? o.result : ""; - const failed = o.is_error === true || o.subtype !== "success" || /not logged in|please run \/login/i.test(resultText); + const summary = typeof o.result === "string" ? o.result : ""; + const failed = o.is_error === true || o.subtype !== "success" || AUTH_FAILURE.test(summary); return [seq.next(failed ? "task.failed" : "task.finished", { subtype: o.subtype, total_cost_usd: o.total_cost_usd, @@ -132,7 +136,7 @@ export function parseClaudeLine(line: string, seq: EventSequence, state = newCla num_turns: o.num_turns, usage: o.usage, result: o.result, - reason: failed ? resultText : undefined, + reason: failed ? summary || o.subtype : undefined, agentSessionId: o.session_id, }, o)]; } diff --git a/runner/test/claude-parser.test.ts b/runner/test/claude-parser.test.ts index 69433f5..3591f1e 100644 --- a/runner/test/claude-parser.test.ts +++ b/runner/test/claude-parser.test.ts @@ -71,6 +71,25 @@ describe("parseClaudeLine", () => { expect(e!.payload.total_cost_usd).toBe(0.5); }); + it("falls back to the subtype as reason when an error result has no text", () => { + const [e] = replay([JSON.stringify({ type: "result", subtype: "error_max_budget_usd", is_error: true })]); + expect(e!.payload.reason).toBe("error_max_budget_usd"); + }); + + it("maps a login failure reported as success to task.failed", () => { + const result = "Not logged in · Please run /login"; + const [e] = replay([JSON.stringify({ type: "result", subtype: "success", is_error: false, result })]); + expect(e!.type).toBe("task.failed"); + expect(e!.payload.reason).toBe(result); + }); + + it("keeps a successful summary that only mentions login as task.finished", () => { + const result = "Fixed the redirect so users who are not logged in go to /signin. Please run /login to test."; + const [e] = replay([JSON.stringify({ type: "result", subtype: "success", is_error: false, result })]); + expect(e!.type).toBe("task.finished"); + expect(e!.payload.reason).toBeUndefined(); + }); + it("turns a malformed or half-finished line into a message with the raw text, and skips blank lines", () => { const half = lines[1]!.slice(0, 40); const out = replay(["not json", half, "", " "]);