/** * Live E2E test watcher dashboard. * * Reads heartbeat (e2e-live.json) for current test status and * partial eval results (_partial-e2e.json) for completed tests. * Renders a terminal dashboard every 1s. * * Usage: bun run eval:watch [--tail] */ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev'); const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); const PARTIAL_PATH = path.join(GSTACK_DEV_DIR, 'evals', '_partial-e2e.json'); const STALE_THRESHOLD_SEC = 600; // 10 minutes export interface HeartbeatData { runId: string; pid?: number; startedAt: string; currentTest: string; status: string; turn: number; toolCount: number; lastTool: string; lastToolAt: string; elapsedSec: number; } export interface PartialData { tests: Array<{ name: string; passed: boolean; cost_usd: number; duration_ms: number; turns_used?: number; exit_reason?: string; }>; total_cost_usd: number; _partial?: boolean; } /** Read and parse a JSON file, returning null on any error. */ function readJSON(filePath: string): T | null { try { return JSON.parse(fs.readFileSync(filePath, 'utf-8')); } catch { return null; } } /** Check if a process is alive (signal 0 = existence check, doesn't kill). */ function isProcessAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch { return false; } } /** Format seconds as Xm Ys */ function formatDuration(sec: number): string { if (sec < 60) return `${sec}s`; const m = Math.floor(sec / 60); const s = sec % 60; return `${m}m ${s}s`; } /** Render dashboard from heartbeat + partial data. Pure function for testability. */ export function renderDashboard(heartbeat: HeartbeatData | null, partial: PartialData | null): string { const lines: string[] = []; if (!heartbeat && !partial) { lines.push('E2E Watch — No active run detected'); lines.push(''); lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`); lines.push(`Partial: ${PARTIAL_PATH} (not found)`); lines.push(''); lines.push('Start a run with: EVALS=1 bun test test/skill-e2e.test.ts'); return lines.join('\n'); } const runId = heartbeat?.runId || 'unknown'; const elapsed = heartbeat?.elapsedSec || 0; lines.push(`E2E Watch \u2014 Run ${runId} \u2014 ${formatDuration(elapsed)}`); lines.push('\u2550'.repeat(55)); // Completed tests from partial if (partial?.tests) { for (const t of partial.tests) { const icon = t.passed ? '\u2713' : '\u2717'; const cost = `$${t.cost_usd.toFixed(2)}`; const dur = `${Math.round(t.duration_ms / 1000)}s`; const turns = t.turns_used !== undefined ? `${t.turns_used} turns` : ''; const name = t.name.length > 30 ? t.name.slice(0, 27) + '...' : t.name.padEnd(30); lines.push(` ${icon} ${name} ${cost.padStart(6)} ${dur.padStart(5)} ${turns}`); } } // Current test from heartbeat if (heartbeat && heartbeat.status === 'running') { const name = heartbeat.currentTest.length > 30 ? heartbeat.currentTest.slice(0, 27) + '...' : heartbeat.currentTest.padEnd(30); lines.push(` \u29D6 ${name} ${formatDuration(heartbeat.elapsedSec).padStart(6)} turn ${heartbeat.turn} last: ${heartbeat.lastTool}`); // Stale detection const lastToolTime = new Date(heartbeat.lastToolAt).getTime(); const staleSec = Math.round((Date.now() - lastToolTime) / 1000); if (staleSec > STALE_THRESHOLD_SEC) { lines.push(` \u26A0 STALE: last tool call was ${formatDuration(staleSec)} ago \u2014 run may have crashed`); } } lines.push('\u2500'.repeat(55)); // Summary const completedCount = partial?.tests?.length || 0; const totalCost = partial?.total_cost_usd || 0; const running = heartbeat?.status === 'running' ? 1 : 0; lines.push(` Completed: ${completedCount} Running: ${running} Cost: $${totalCost.toFixed(2)} Elapsed: ${formatDuration(elapsed)}`); if (heartbeat?.runId) { const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log'); lines.push(` Logs: ${logPath}`); } return lines.join('\n'); } // --- Main --- if (import.meta.main) { const showTail = process.argv.includes('--tail'); const render = () => { let heartbeat = readJSON(HEARTBEAT_PATH); const partial = readJSON(PARTIAL_PATH); // Auto-clear heartbeat if the process is dead if (heartbeat?.pid && !isProcessAlive(heartbeat.pid)) { try { fs.unlinkSync(HEARTBEAT_PATH); } catch { /* already gone */ } process.stdout.write('\x1B[2J\x1B[H'); process.stdout.write(`Cleared stale heartbeat — PID ${heartbeat.pid} is no longer running.\n\n`); heartbeat = null; } // Clear screen process.stdout.write('\x1B[2J\x1B[H'); process.stdout.write(renderDashboard(heartbeat, partial) + '\n'); // --tail: show last 10 lines of progress.log if (showTail && heartbeat?.runId) { const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log'); try { const content = fs.readFileSync(logPath, 'utf-8'); const tail = content.split('\n').filter(l => l.trim()).slice(-10); process.stdout.write('\nRecent progress:\n'); for (const line of tail) { process.stdout.write(line + '\n'); } } catch { /* log file may not exist yet */ } } }; render(); setInterval(render, 1000); }