import { describe, test, expect } from 'bun:test';
import { COMMAND_DESCRIPTIONS } from '../browse/src/commands';
import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
import * as fs from 'fs';
import * as path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
describe('gen-skill-docs', () => {
test('generated SKILL.md contains all command categories', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category));
for (const cat of categories) {
expect(content).toContain(`### ${cat}`);
}
});
test('generated SKILL.md contains all commands', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
const display = meta.usage || cmd;
expect(content).toContain(display);
}
});
test('command table is sorted alphabetically within categories', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
// Extract command names from the Navigation section as a test
const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/);
expect(navSection).not.toBeNull();
const rows = navSection![1].trim().split('\n');
const commands = rows.map(r => {
const match = r.match(/\| `(\w+)/);
return match ? match[1] : '';
}).filter(Boolean);
const sorted = [...commands].sort();
expect(commands).toEqual(sorted);
});
test('generated header is present in SKILL.md', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
expect(content).toContain('Regenerate: bun run gen:skill-docs');
});
test('generated header is present in browse/SKILL.md', () => {
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
});
test('snapshot flags section contains all flags', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
for (const flag of SNAPSHOT_FLAGS) {
expect(content).toContain(flag.short);
expect(content).toContain(flag.description);
}
});
// All skills that must have templates — single source of truth
const ALL_SKILLS = [
{ dir: '.', name: 'root gstack' },
{ dir: 'browse', name: 'browse' },
{ dir: 'qa', name: 'qa' },
{ dir: 'qa-only', name: 'qa-only' },
{ dir: 'review', name: 'review' },
{ dir: 'ship', name: 'ship' },
{ dir: 'plan-ceo-review', name: 'plan-ceo-review' },
{ dir: 'plan-eng-review', name: 'plan-eng-review' },
{ dir: 'retro', name: 'retro' },
{ dir: 'setup-browser-cookies', name: 'setup-browser-cookies' },
{ dir: 'gstack-upgrade', name: 'gstack-upgrade' },
];
test('every skill has a SKILL.md.tmpl template', () => {
for (const skill of ALL_SKILLS) {
const tmplPath = path.join(ROOT, skill.dir, 'SKILL.md.tmpl');
expect(fs.existsSync(tmplPath)).toBe(true);
}
});
test('every skill has a generated SKILL.md with auto-generated header', () => {
for (const skill of ALL_SKILLS) {
const mdPath = path.join(ROOT, skill.dir, 'SKILL.md');
expect(fs.existsSync(mdPath)).toBe(true);
const content = fs.readFileSync(mdPath, 'utf-8');
expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
expect(content).toContain('Regenerate: bun run gen:skill-docs');
}
});
test('every generated SKILL.md has valid YAML frontmatter', () => {
for (const skill of ALL_SKILLS) {
const content = fs.readFileSync(path.join(ROOT, skill.dir, 'SKILL.md'), 'utf-8');
expect(content.startsWith('---\n')).toBe(true);
expect(content).toContain('name:');
expect(content).toContain('description:');
}
});
test('generated files are fresh (match --dry-run)', () => {
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--dry-run'], {
cwd: ROOT,
stdout: 'pipe',
stderr: 'pipe',
});
expect(result.exitCode).toBe(0);
const output = result.stdout.toString();
// Every skill should be FRESH
for (const skill of ALL_SKILLS) {
const file = skill.dir === '.' ? 'SKILL.md' : `${skill.dir}/SKILL.md`;
expect(output).toContain(`FRESH: ${file}`);
}
expect(output).not.toContain('STALE');
});
test('no generated SKILL.md contains unresolved placeholders', () => {
for (const skill of ALL_SKILLS) {
const content = fs.readFileSync(path.join(ROOT, skill.dir, 'SKILL.md'), 'utf-8');
const unresolved = content.match(/\{\{[A-Z_]+\}\}/g);
expect(unresolved).toBeNull();
}
});
test('templates contain placeholders', () => {
const rootTmpl = fs.readFileSync(path.join(ROOT, 'SKILL.md.tmpl'), 'utf-8');
expect(rootTmpl).toContain('{{COMMAND_REFERENCE}}');
expect(rootTmpl).toContain('{{SNAPSHOT_FLAGS}}');
const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8');
expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}');
expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}');
});
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');
const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8');
expect(qaOnlyTmpl).toContain('{{QA_METHODOLOGY}}');
});
test('QA_METHODOLOGY appears expanded in both qa and qa-only generated files', () => {
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
const qaOnlyContent = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8');
// Both should contain the health score rubric
expect(qaContent).toContain('Health Score Rubric');
expect(qaOnlyContent).toContain('Health Score Rubric');
// Both should contain framework guidance
expect(qaContent).toContain('Framework-Specific Guidance');
expect(qaOnlyContent).toContain('Framework-Specific Guidance');
// Both should contain the important rules
expect(qaContent).toContain('Important Rules');
expect(qaOnlyContent).toContain('Important Rules');
// Both should contain the 6 phases
expect(qaContent).toContain('Phase 1');
expect(qaOnlyContent).toContain('Phase 1');
expect(qaContent).toContain('Phase 6');
expect(qaOnlyContent).toContain('Phase 6');
});
test('qa-only has no-fix guardrails', () => {
const qaOnlyContent = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8');
expect(qaOnlyContent).toContain('Never fix bugs');
expect(qaOnlyContent).toContain('NEVER fix anything');
// Should not have Edit, Glob, or Grep in allowed-tools
expect(qaOnlyContent).not.toMatch(/allowed-tools:[\s\S]*?Edit/);
expect(qaOnlyContent).not.toMatch(/allowed-tools:[\s\S]*?Glob/);
expect(qaOnlyContent).not.toMatch(/allowed-tools:[\s\S]*?Grep/);
});
test('qa has fix-loop tools and phases', () => {
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
// Should have Edit, Glob, Grep in allowed-tools
expect(qaContent).toContain('Edit');
expect(qaContent).toContain('Glob');
expect(qaContent).toContain('Grep');
// Should have fix-loop phases
expect(qaContent).toContain('Phase 7');
expect(qaContent).toContain('Phase 8');
expect(qaContent).toContain('Fix Loop');
expect(qaContent).toContain('Triage');
expect(qaContent).toContain('WTF');
});
});
/**
* Quality evals — catch description regressions.
*
* These test that generated output is *useful for an AI agent*,
* not just structurally valid. Each test targets a specific
* regression we actually shipped and caught in review.
*/
describe('description quality evals', () => {
// Regression: snapshot flags lost value hints (-d <N>, -s <sel>, -o <path>)
test('snapshot flags with values include value hints in output', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
for (const flag of SNAPSHOT_FLAGS) {
if (flag.takesValue) {
expect(flag.valueHint).toBeDefined();
expect(content).toContain(`${flag.short} ${flag.valueHint}`);
}
}
});
// Regression: "is" lost the valid states enum
test('is command lists valid state values', () => {
const desc = COMMAND_DESCRIPTIONS['is'].description;
for (const state of ['visible', 'hidden', 'enabled', 'disabled', 'checked', 'editable', 'focused']) {
expect(desc).toContain(state);
}
});
// Regression: "press" lost common key examples
test('press command lists example keys', () => {
const desc = COMMAND_DESCRIPTIONS['press'].description;
expect(desc).toContain('Enter');
expect(desc).toContain('Tab');
expect(desc).toContain('Escape');
});
// Regression: "console" lost --errors filter note
test('console command describes --errors behavior', () => {
const desc = COMMAND_DESCRIPTIONS['console'].description;
expect(desc).toContain('--errors');
});
// Regression: snapshot -i lost "@e refs" context
test('snapshot -i mentions @e refs', () => {
const flag = SNAPSHOT_FLAGS.find(f => f.short === '-i')!;
expect(flag.description).toContain('@e');
});
// Regression: snapshot -C lost "@c refs" context
test('snapshot -C mentions @c refs', () => {
const flag = SNAPSHOT_FLAGS.find(f => f.short === '-C')!;
expect(flag.description).toContain('@c');
});
// Guard: every description must be at least 8 chars (catches empty or stub descriptions)
test('all command descriptions have meaningful length', () => {
for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
expect(meta.description.length).toBeGreaterThanOrEqual(8);
}
});
// Guard: snapshot flag descriptions must be at least 10 chars
test('all snapshot flag descriptions have meaningful length', () => {
for (const flag of SNAPSHOT_FLAGS) {
expect(flag.description.length).toBeGreaterThanOrEqual(10);
}
});
// Guard: descriptions must not contain pipe (breaks markdown table cells)
// Usage strings are backtick-wrapped in the table so pipes there are safe.
test('no command description contains pipe character', () => {
for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
expect(meta.description).not.toContain('|');
}
});
// Guard: generated output uses → not ->
test('generated SKILL.md uses unicode arrows', () => {
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
// Check the Tips section specifically (where we regressed -> from →)
const tipsSection = content.slice(content.indexOf('## Tips'));
expect(tipsSection).toContain('→');
expect(tipsSection).not.toContain('->');
});
});