/** * AgentEngine Tests — Chunked Writer Architecture * * 예전 buildup(planner → researcher → reflector → writer → synthesizer)을 단일 * ChunkedWriter 의 outline → section[N] → polish 로 교체한 뒤의 회귀 테스트. * * 다루는 범위: * 1. ErrorClassifier — TRANSIENT / PERMANENT / ABORT 분류 패턴 * 2. ErrorRecoveryMatrix — 각 유형이 의도된 action·재시도 카운트로 매핑 * 3. MissionState — audit trail / 상태 전환 / 구조화 로그 * 4. AgentEngine.runMission — chunked 흐름(outline 1회 → section N회 → polish 1회) * 5. WikiFormatter gate — formatAsKnowledgeArtifact 옵션에 한해서만 wrap */ import { AgentEngine, IAgent, AgentExecuteOptions, ErrorClassifier, ErrorType, ERROR_RECOVERY_MATRIX, MissionState, PipelineStage, } from '../src/lib/engine'; import * as fs from 'fs'; import * as path from 'path'; // ─── Setup ─────────────────────────────────────────────────────────────────── const getBaseDir = () => process.env.ASTRA_TEST_ROOT || process.cwd(); const clearCache = () => { const baseDir = getBaseDir(); for (const sub of ['cache', 'missions']) { const dir = path.join(baseDir, '.astra', sub); if (fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); } }; beforeAll(() => { process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'engine'); if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) { fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true }); } clearCache(); }); beforeEach(() => { clearCache(); }); // ─── Mock agents ───────────────────────────────────────────────────────────── /** * Role-aware mock — ChunkedWriter 처럼 options.config.role 에 따라 다른 응답을 * 돌려준다. 실제 작은 모델의 행동을 단순 시뮬레이션. */ class MockChunkedWriter implements IAgent { public calls: Array<{ role: string; input: string }> = []; constructor( private readonly outlineJson: string = '[{"heading":"본문","scope":"전체 답변을 다루는 단일 섹션"}]', private readonly sectionText: string = '본문 내용은 충분히 길게 작성된 mock 응답입니다.', private readonly polished: string = '최종 답변 본문 — 사용자에게 보일 polish 결과입니다. 충분히 긴 문자열.', private readonly direct: string = '직답 결과 — single-pass mock 응답입니다.', ) {} async execute(input: string, _ctx?: string, _signal?: AbortSignal, options?: AgentExecuteOptions): Promise { const role = (options?.config?.role as string | undefined) ?? 'section'; this.calls.push({ role, input }); if (role === 'outline') return this.outlineJson; if (role === 'polish') return this.polished; if (role === 'direct') return this.direct; return this.sectionText; } } class MockPermanentAgent implements IAgent { async execute(): Promise { throw new Error('404: model not found'); } } class MockAbortAgent implements IAgent { async execute(): Promise { const err = new Error('AbortError'); err.name = 'AbortError'; throw err; } } const noopProgress = (_stage: PipelineStage, _message: string) => {}; const createAbortSignal = (): AbortSignal => new AbortController().signal; /** * Fast-path 휴리스틱을 통과해 chunked 흐름으로 가도록 충분히 긴 + 분석 키워드를 * 포함한 prompt. 200자 미만 / 키워드 없음 / 본문 첨부 없음이면 fast-path 가 발동해 * outline·section 단계가 모두 우회된다. */ const CHUNKED_PROMPT = `다음 보고서를 종합적으로 분석해서 핵심 요점을 정리하고, ` + `각 섹션의 강점과 약점을 평가하며, 향후 개선 방향을 제안해 주세요. ` + `프로젝트 전반의 기획 의도와 실제 구현 결과 사이의 정합성도 함께 검토해 주세요. ` + `리뷰는 가능한 한 상세하고 꼼꼼하게 작성되어야 합니다.`; // ─── ErrorClassifier ───────────────────────────────────────────────────────── describe('ErrorClassifier', () => { describe('Transient', () => { const messages = [ 'ECONNREFUSED: Connection refused', 'Request timeout exceeded', 'ETIMEDOUT: operation timed out', 'ECONNRESET: connection reset by peer', 'network error occurred', 'Failed to fetch', 'HTTP 503: Service Unavailable', 'HTTP 502: Bad Gateway', 'HTTP 429: Too Many Requests', 'socket hang up', ]; test.each(messages)('"%s" → TRANSIENT', (msg) => { const r = ErrorClassifier.classify(new Error(msg)); expect(r.type).toBe(ErrorType.TRANSIENT); expect(r.rule.action).toBe('retry'); expect(r.rule.maxRetries).toBe(3); }); }); describe('Permanent', () => { const messages = [ 'HTTP 401: Unauthorized', 'HTTP 403: Forbidden', 'HTTP 404: Not Found', 'Planner 에이전트로부터 유효한 응답을 받지 못했습니다', 'Ollama URL이 설정되지 않았습니다', 'invalid model name specified', 'model not found in registry', ]; test.each(messages)('"%s" → PERMANENT', (msg) => { const r = ErrorClassifier.classify(new Error(msg)); expect(r.type).toBe(ErrorType.PERMANENT); expect(r.rule.action).toBe('fail_with_message'); expect(r.rule.maxRetries).toBe(0); }); }); describe('Abort', () => { test('name="AbortError" → ABORT', () => { const err = new Error('cancelled'); err.name = 'AbortError'; expect(ErrorClassifier.classify(err).type).toBe(ErrorType.ABORT); }); test('message="AbortError" → ABORT', () => { expect(ErrorClassifier.classify(new Error('AbortError')).type).toBe(ErrorType.ABORT); }); }); test('unknown → PERMANENT (보수적 처리)', () => { const r = ErrorClassifier.classify(new Error('something completely unexpected')); expect(r.type).toBe(ErrorType.PERMANENT); }); }); // ─── ErrorRecoveryMatrix ───────────────────────────────────────────────────── describe('ErrorRecoveryMatrix', () => { test('세 유형 모두 정의돼 있어야 한다', () => { const types = ERROR_RECOVERY_MATRIX.map(r => r.type); expect(types).toContain(ErrorType.TRANSIENT); expect(types).toContain(ErrorType.PERMANENT); expect(types).toContain(ErrorType.ABORT); }); test('TRANSIENT 은 재시도 가능', () => { const r = ERROR_RECOVERY_MATRIX.find(x => x.type === ErrorType.TRANSIENT)!; expect(r.maxRetries).toBeGreaterThan(0); expect(r.action).toBe('retry'); }); test('PERMANENT 은 재시도 없이 즉시 실패', () => { const r = ERROR_RECOVERY_MATRIX.find(x => x.type === ErrorType.PERMANENT)!; expect(r.maxRetries).toBe(0); expect(r.action).toBe('fail_with_message'); }); test('ABORT 은 조용히 종료', () => { const r = ERROR_RECOVERY_MATRIX.find(x => x.type === ErrorType.ABORT)!; expect(r.action).toBe('abort'); }); }); // ─── MissionState ──────────────────────────────────────────────────────────── describe('MissionState', () => { test('초기 상태는 idle', () => { const s = new MissionState('m1'); expect(s.stage).toBe('idle'); expect(s.auditTrail.length).toBe(0); }); test('chunked 흐름 stage 전환이 audit trail 에 기록된다', () => { const s = new MissionState('m2'); s.transition('outline', '구조 잡는 중'); s.transition('section', '섹션 1'); s.transition('section', '섹션 2'); s.transition('polish', '다듬는 중'); s.transition('completed', '완료'); expect(s.stage).toBe('completed'); expect(s.auditTrail.length).toBe(5); expect(s.auditTrail[0].from).toBe('idle'); expect(s.auditTrail[0].to).toBe('outline'); expect(s.auditTrail[3].to).toBe('polish'); }); test('toStructuredLog() 가 올바른 JSON 구조를 반환한다', () => { const s = new MissionState('m3'); s.transition('outline', 'a'); s.transition('completed', 'b'); const log = s.toStructuredLog() as any; expect(log.missionId).toBe('m3'); expect(log.status).toBe('completed'); expect(log.transitions).toHaveLength(2); expect(log.transitions[0].to).toBe('outline'); }); test('getElapsedMs() 는 음수가 아니다', () => { const s = new MissionState('m4'); expect(s.getElapsedMs()).toBeGreaterThanOrEqual(0); }); }); // ─── AgentEngine — chunked flow ────────────────────────────────────────────── describe('AgentEngine — chunked flow', () => { test('outline 1개 섹션 → outline + 1 section + polish (총 3회 호출)', async () => { const writer = new MockChunkedWriter(); const engine = new AgentEngine(writer); const result = await engine.runMission( 'chunked_single', CHUNKED_PROMPT, 'ctx', createAbortSignal(), noopProgress, ); expect(result).toContain('최종 답변 본문'); const roles = writer.calls.map(c => c.role); expect(roles).toEqual(['outline', 'section', 'polish']); }); test('outline N개 → outline + N section + polish (N=3 일 때 5회 호출)', async () => { const writer = new MockChunkedWriter( '[{"heading":"A","scope":"a"},{"heading":"B","scope":"b"},{"heading":"C","scope":"c"}]' ); const engine = new AgentEngine(writer); await engine.runMission( 'chunked_multi', CHUNKED_PROMPT, 'ctx', createAbortSignal(), noopProgress, ); const roles = writer.calls.map(c => c.role); expect(roles[0]).toBe('outline'); expect(roles[roles.length - 1]).toBe('polish'); expect(roles.filter(r => r === 'section')).toHaveLength(3); }); test('outline MAX_SECTIONS 초과 응답은 5개로 cap 된다', async () => { // 7개를 줘도 5개로 잘려야 함 const writer = new MockChunkedWriter( JSON.stringify( Array.from({ length: 7 }, (_, i) => ({ heading: `H${i}`, scope: `s${i}` })) ) ); const engine = new AgentEngine(writer); await engine.runMission( 'chunked_cap', CHUNKED_PROMPT, 'ctx', createAbortSignal(), noopProgress, ); const sectionCount = writer.calls.filter(c => c.role === 'section').length; expect(sectionCount).toBe(AgentEngine.MAX_SECTIONS); }); test('outline JSON 파싱 실패 시 단일 "본문" 섹션으로 폴백', async () => { const writer = new MockChunkedWriter('this is not json at all'); const engine = new AgentEngine(writer); await engine.runMission( 'chunked_fallback', CHUNKED_PROMPT, 'ctx', createAbortSignal(), noopProgress, ); const sectionCount = writer.calls.filter(c => c.role === 'section').length; expect(sectionCount).toBe(1); }); test('outline JSON 이 ```json ... ``` 펜스로 감싸져 있어도 파싱', async () => { const writer = new MockChunkedWriter( '```json\n[{"heading":"H1","scope":"s1"},{"heading":"H2","scope":"s2"}]\n```' ); const engine = new AgentEngine(writer); await engine.runMission( 'chunked_fenced', CHUNKED_PROMPT, 'ctx', createAbortSignal(), noopProgress, ); const sectionCount = writer.calls.filter(c => c.role === 'section').length; expect(sectionCount).toBe(2); }); test('Permanent 오류는 즉시 중단', async () => { const engine = new AgentEngine(new MockPermanentAgent()); await expect( engine.runMission('chunked_permanent', 'p', 'c', createAbortSignal(), noopProgress) ).rejects.toThrow(); }); test('Abort 시그널은 graceful exit', async () => { const engine = new AgentEngine(new MockAbortAgent()); await expect( engine.runMission('chunked_abort', 'p', 'c', createAbortSignal(), noopProgress) ).rejects.toThrow('AbortError'); }); test('미션 완료 후 getMissionState() 는 null', async () => { const engine = new AgentEngine(new MockChunkedWriter()); await engine.runMission( 'chunked_state_cleanup', 'p', 'c', createAbortSignal(), noopProgress, ); expect(engine.getMissionState()).toBeNull(); }); }); // ─── AgentEngine — single-pass routing ────────────────────────────────────── describe('AgentEngine — single-pass routing', () => { test('isObviouslySimple: 짧은 일반 질문 → true', () => { expect(AgentEngine.isObviouslySimple('오늘 날씨 어때?')).toBe(true); expect(AgentEngine.isObviouslySimple('이 함수 이름 뭐로 짓는 게 좋을까?')).toBe(true); expect(AgentEngine.isObviouslySimple('hello world')).toBe(true); }); test('isObviouslySimple: 분석/리서치 키워드 포함 → false', () => { expect(AgentEngine.isObviouslySimple('이거 분석해줘')).toBe(false); expect(AgentEngine.isObviouslySimple('보고서 작성 부탁')).toBe(false); expect(AgentEngine.isObviouslySimple('아키텍처 설계 좀')).toBe(false); }); test('isObviouslySimple: 본문 첨부 흔적 (코드 펜스 / 빈줄 다수) → false', () => { expect(AgentEngine.isObviouslySimple('```python\nprint(1)\n```')).toBe(false); expect(AgentEngine.isObviouslySimple('첫 줄\n\n\n둘째 줄')).toBe(false); }); test('isObviouslySimple: 길이 200자 이상 → false', () => { const long = '가나다라마바사아자차카타파하'.repeat(20); expect(AgentEngine.isObviouslySimple(long)).toBe(false); }); test('fast-path: 짧은 단순 질문은 outline·section 없이 direct 한 번만 호출', async () => { const writer = new MockChunkedWriter(); const engine = new AgentEngine(writer); const result = await engine.runMission( 'fastpath_simple', '이 함수 이름 뭐로 할까?', 'ctx', createAbortSignal(), noopProgress, ); const roles = writer.calls.map(c => c.role); expect(roles).toEqual(['direct']); expect(result).toContain('직답 결과'); }); test('outline 빈배열 폴백: outline + direct 두 호출 (section·polish 건너뜀)', async () => { // outline 이 빈 배열로 "쪼갤 필요 없음" 신호 → direct 로 폴백. const writer = new MockChunkedWriter('[]'); const engine = new AgentEngine(writer); await engine.runMission( 'outline_empty', CHUNKED_PROMPT, 'ctx', createAbortSignal(), noopProgress, ); const roles = writer.calls.map(c => c.role); expect(roles).toEqual(['outline', 'direct']); }); test('direct stage 가 audit trail 에 기록된다', async () => { const writer = new MockChunkedWriter(); const engine = new AgentEngine(writer); const stages: PipelineStage[] = []; await engine.runMission( 'fastpath_audit', '뭐가 좋아?', 'ctx', createAbortSignal(), (stage) => { stages.push(stage); }, ); expect(stages).toContain('direct'); expect(stages).not.toContain('outline'); expect(stages).not.toContain('section'); }); }); // ─── WikiFormatter gate ────────────────────────────────────────────────────── describe('WikiFormatter gate', () => { test('기본은 wiki 포맷을 *적용하지 않는다*', async () => { const writer = new MockChunkedWriter(); const engine = new AgentEngine(writer); const result = await engine.runMission( 'wiki_off', 'p', 'c', createAbortSignal(), noopProgress, ); expect(result).not.toContain('P-Reinforce v3.0'); expect(result).not.toContain('Reliability & Audit Summary'); }); test('options.config.formatAsKnowledgeArtifact=true 일 때만 wrap', async () => { const writer = new MockChunkedWriter(); const engine = new AgentEngine(writer); const result = await engine.runMission( 'wiki_on', 'p', 'c', createAbortSignal(), noopProgress, { config: { formatAsKnowledgeArtifact: true } }, ); expect(result).toContain('P-Reinforce v3.0'); expect(result).toContain('Reliability & Audit Summary'); }); });