import { describe, test, expect, afterAll } from 'bun:test'; import { runSkillTest } from './helpers/session-runner'; import type { SkillTestResult } from './helpers/session-runner'; import { EvalCollector } from './helpers/eval-store'; import type { EvalTestEntry } from './helpers/eval-store'; import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; const ROOT = path.resolve(import.meta.dir, '..'); // Skip unless EVALS=1. const evalsEnabled = !!process.env.EVALS; const describeE2E = evalsEnabled ? describe : describe.skip; // Eval result collector const evalCollector = evalsEnabled ? new EvalCollector('e2e-routing') : null; // Unique run ID for this session const runId = new Date().toISOString().replace(/[:.]/g, '').replace('T', '-').slice(0, 15); // --- Diff-based test selection --- // Journey routing tests use E2E_TOUCHFILES (entries prefixed 'journey-' in touchfiles.ts). let selectedTests: string[] | null = null; if (evalsEnabled && !process.env.EVALS_ALL) { const baseBranch = process.env.EVALS_BASE || detectBaseBranch(ROOT) || 'main'; const changedFiles = getChangedFiles(baseBranch, ROOT); if (changedFiles.length > 0) { const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES); selectedTests = selection.selected; process.stderr.write(`\nRouting E2E selection (${selection.reason}): ${selection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests\n`); if (selection.skipped.length > 0) { process.stderr.write(` Skipped: ${selection.skipped.join(', ')}\n`); } process.stderr.write('\n'); } } // --- Helper functions --- /** Copy all SKILL.md files into tmpDir/.claude/skills/gstack/ for auto-discovery */ function installSkills(tmpDir: string) { const skillDirs = [ '', // root gstack SKILL.md 'qa', 'qa-only', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review', 'plan-design-review', 'design-review', 'design-consultation', 'retro', 'document-release', 'debug', 'office-hours', 'browse', 'setup-browser-cookies', 'gstack-upgrade', 'humanizer', ]; for (const skill of skillDirs) { const srcPath = path.join(ROOT, skill, 'SKILL.md'); if (!fs.existsSync(srcPath)) continue; const destDir = skill ? path.join(tmpDir, '.claude', 'skills', 'gstack', skill) : path.join(tmpDir, '.claude', 'skills', 'gstack'); fs.mkdirSync(destDir, { recursive: true }); fs.copyFileSync(srcPath, path.join(destDir, 'SKILL.md')); } } /** Init a git repo with config */ function initGitRepo(dir: string) { const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 }); run('git', ['init']); run('git', ['config', 'user.email', 'test@test.com']); run('git', ['config', 'user.name', 'Test']); } function logCost(label: string, result: { costEstimate: { turnsUsed: number; estimatedTokens: number; estimatedCost: number }; duration: number }) { const { turnsUsed, estimatedTokens, estimatedCost } = result.costEstimate; const durationSec = Math.round(result.duration / 1000); console.log(`${label}: $${estimatedCost.toFixed(2)} (${turnsUsed} turns, ${(estimatedTokens / 1000).toFixed(1)}k tokens, ${durationSec}s)`); } function recordRouting(name: string, result: SkillTestResult, expectedSkill: string, actualSkill: string | undefined) { evalCollector?.addTest({ name, suite: 'Skill Routing E2E', tier: 'e2e', passed: actualSkill === expectedSkill, duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, transcript: result.transcript, output: result.output?.slice(0, 2000), turns_used: result.costEstimate.turnsUsed, exit_reason: result.exitReason, }); } // --- Tests --- describeE2E('Skill Routing E2E — Developer Journey', () => { afterAll(() => { evalCollector?.finalize(); }); test('journey-ideation', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ideation-')); try { initGitRepo(tmpDir); installSkills(tmpDir); fs.writeFileSync(path.join(tmpDir, 'README.md'), '# New Project\n'); spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); const testName = 'journey-ideation'; const expectedSkill = 'office-hours'; const result = await runSkillTest({ prompt: "I've been thinking about building a waitlist management tool for restaurants. The existing solutions are expensive and overcomplicated. I want something simple — a tablet app where hosts can add parties, see wait times, and text customers when their table is ready. Help me think through whether this is worth building and what the key design decisions are.", workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], timeout: 60_000, testName, runId, }); const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill'); const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined; logCost(`journey: ${testName}`, result); recordRouting(testName, result, expectedSkill, actualSkill); expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0); expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } }, 90_000); test('journey-plan-eng', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-plan-eng-')); try { initGitRepo(tmpDir); installSkills(tmpDir); fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture ## Components - REST API (Express.js) - PostgreSQL database - React frontend - SMS integration (Twilio) ## Data Model - restaurants (id, name, settings) - parties (id, restaurant_id, name, size, phone, status, created_at) - wait_estimates (id, restaurant_id, avg_wait_minutes) ## API Endpoints - POST /api/parties - add party to waitlist - GET /api/parties - list current waitlist - PATCH /api/parties/:id/status - update party status - GET /api/estimate - get current wait estimate `); spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); const testName = 'journey-plan-eng'; const expectedSkill = 'plan-eng-review'; const result = await runSkillTest({ prompt: "I wrote up a plan for the waitlist app in plan.md. Can you take a look at the architecture and make sure I'm not missing any edge cases or failure modes before I start coding?", workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], timeout: 60_000, testName, runId, }); const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill'); const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined; logCost(`journey: ${testName}`, result); recordRouting(testName, result, expectedSkill, actualSkill); expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0); expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } }, 90_000); test('journey-think-bigger', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-think-bigger-')); try { initGitRepo(tmpDir); installSkills(tmpDir); fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture ## Components - REST API (Express.js) - PostgreSQL database - React frontend - SMS integration (Twilio) ## Data Model - restaurants (id, name, settings) - parties (id, restaurant_id, name, size, phone, status, created_at) - wait_estimates (id, restaurant_id, avg_wait_minutes) ## API Endpoints - POST /api/parties - add party to waitlist - GET /api/parties - list current waitlist - PATCH /api/parties/:id/status - update party status - GET /api/estimate - get current wait estimate `); spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); const testName = 'journey-think-bigger'; const expectedSkill = 'plan-ceo-review'; const result = await runSkillTest({ prompt: "Actually, looking at this plan again, I feel like we're thinking too small. We're just doing waitlists but what about the whole restaurant guest experience? Is there a bigger opportunity here we should go after?", workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], timeout: 120_000, testName, runId, }); const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill'); const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined; logCost(`journey: ${testName}`, result); recordRouting(testName, result, expectedSkill, actualSkill); expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0); expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } }, 180_000); test('journey-debug', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-debug-')); try { initGitRepo(tmpDir); installSkills(tmpDir); const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); fs.writeFileSync(path.join(tmpDir, 'src/api.ts'), ` import express from 'express'; const app = express(); app.get('/api/waitlist', async (req, res) => { const db = req.app.locals.db; const parties = await db.query('SELECT * FROM parties WHERE status = $1', ['waiting']); res.json(parties.rows); }); export default app; `); fs.writeFileSync(path.join(tmpDir, 'error.log'), ` [2026-03-18T10:23:45Z] ERROR: GET /api/waitlist - 500 Internal Server Error TypeError: Cannot read properties of undefined (reading 'query') at /src/api.ts:5:32 at Layer.handle [as handle_request] (/node_modules/express/lib/router/layer.js:95:5) [2026-03-18T10:23:46Z] ERROR: GET /api/waitlist - 500 Internal Server Error TypeError: Cannot read properties of undefined (reading 'query') `); run('git', ['add', '.']); run('git', ['commit', '-m', 'initial']); run('git', ['checkout', '-b', 'feature/waitlist-api']); const testName = 'journey-debug'; const expectedSkill = 'debug'; const result = await runSkillTest({ prompt: "The GET /api/waitlist endpoint was working fine yesterday but now it's returning 500 errors. The tests are passing locally but the endpoint fails when I hit it with curl. Can you figure out what's going on?", workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], timeout: 60_000, testName, runId, }); const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill'); const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined; logCost(`journey: ${testName}`, result); recordRouting(testName, result, expectedSkill, actualSkill); expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0); expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } }, 90_000); test('journey-qa', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-qa-')); try { initGitRepo(tmpDir); installSkills(tmpDir); fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app', scripts: { dev: 'next dev' } }, null, 2)); fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); fs.writeFileSync(path.join(tmpDir, 'src/index.html'), '