From 9f5aa32e679d44d4a0fb9fc8303ff2d6b00bc63f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 14 Mar 2026 12:37:44 -0500 Subject: [PATCH] =?UTF-8?q?fix:=20fail=20fast=20on=20API=20connectivity=20?= =?UTF-8?q?=E2=80=94=20pre-check=20before=20E2E=20suite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spawn a quick claude -p ping before running 13 tests. If the Anthropic API is unreachable (ConnectionRefused), throw immediately instead of burning through the entire suite with silent false passes. Co-Authored-By: Claude Opus 4.6 --- test/skill-e2e.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/skill-e2e.test.ts b/test/skill-e2e.test.ts index ec4330544ae514790e08507a99a738d40b5f3564..be3c6ad232b992802ec68c4f1b51e74cf8c498fd 100644 --- a/test/skill-e2e.test.ts +++ b/test/skill-e2e.test.ts @@ -5,6 +5,7 @@ import { outcomeJudge } from './helpers/llm-judge'; import { EvalCollector } from './helpers/eval-store'; import type { EvalTestEntry } from './helpers/eval-store'; import { startTestServer } from '../browse/test/test-server'; +import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -116,6 +117,17 @@ function dumpOutcomeDiagnostic(dir: string, label: string, report: string, judge } catch { /* non-fatal */ } } +// Fail fast if Anthropic API is unreachable — don't burn through 13 tests getting ConnectionRefused +if (evalsEnabled) { + const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], { + stdio: 'pipe', timeout: 30_000, + }); + const output = check.stdout?.toString() || ''; + if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) { + throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.'); + } +} + describeE2E('Skill E2E tests', () => { beforeAll(() => { testServer = startTestServer();