feat(engine): implement self-reflection (reflector) stage in multi-agent pipeline
- Added ReflectorAgent for meta-cognition and critical review between Research and Writing - Updated WriterAgent to explicitly address reflection critiques - Introduced 'g1nation.enableReflection' configuration setting - Added comprehensive integration tests for the self-reflection stage - Documented design decisions in ADR-0010 and related discussion records
This commit is contained in:
g1nation
@@ -364,6 +364,142 @@ describe('AgentEngine Integration', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// Test Suite 4b: Self-Reflection Stage
|
||||
// ═══════════════════════════════════════════════
|
||||
|
||||
class SpyAgent implements IAgent {
|
||||
public callCount = 0;
|
||||
public lastInput: string | undefined;
|
||||
public lastContext: string | undefined;
|
||||
public lastOptions: AgentExecuteOptions | undefined;
|
||||
public calls: { input: string; context?: string; options?: AgentExecuteOptions }[] = [];
|
||||
constructor(private readonly response: string) {}
|
||||
async execute(input: string, context?: string, _signal?: AbortSignal, options?: AgentExecuteOptions): Promise<string> {
|
||||
this.callCount++;
|
||||
this.lastInput = input;
|
||||
this.lastContext = context;
|
||||
this.lastOptions = options;
|
||||
this.calls.push({ input, context, options });
|
||||
return this.response;
|
||||
}
|
||||
}
|
||||
|
||||
class ThrowingAgent implements IAgent {
|
||||
public callCount = 0;
|
||||
constructor(private readonly message: string = '404: model not found') {}
|
||||
async execute(): Promise<string> {
|
||||
this.callCount++;
|
||||
throw new Error(this.message);
|
||||
}
|
||||
}
|
||||
|
||||
describe('AgentEngine — Self-Reflection Stage', () => {
|
||||
test('Reflector 주입 시 Researcher와 Writer 사이에 1회 실행되며 결과가 Writer.priorResults.reflection 으로 전달되어야 한다', async () => {
|
||||
const planner = new SpyAgent('Plan: rigorous blueprint covering all objectives.');
|
||||
const researcher = new SpyAgent('Research: dense factual synthesis with supporting evidence.');
|
||||
const reflector = new SpyAgent('## ✅ Guidance for Writer\n- Add missing risk section.');
|
||||
const writer = new SpyAgent('Final report incorporating critique.');
|
||||
|
||||
const engine = new AgentEngine(planner, researcher, writer, reflector);
|
||||
const stages: PipelineStage[] = [];
|
||||
await engine.runMission(
|
||||
'reflect_001', 'Test prompt', 'brain ctx',
|
||||
createAbortSignal(),
|
||||
(stage) => { stages.push(stage); }
|
||||
);
|
||||
|
||||
// 정확히 1회 호출
|
||||
expect(reflector.callCount).toBe(1);
|
||||
// 'reflector' 단계가 onProgress에 등장
|
||||
expect(stages).toContain('reflector');
|
||||
// Stage 순서: planner → researcher → reflector → writer → completed
|
||||
const idxResearcher = stages.indexOf('researcher');
|
||||
const idxReflector = stages.indexOf('reflector');
|
||||
const idxWriter = stages.indexOf('writer');
|
||||
expect(idxResearcher).toBeGreaterThanOrEqual(0);
|
||||
expect(idxReflector).toBeGreaterThan(idxResearcher);
|
||||
expect(idxWriter).toBeGreaterThan(idxReflector);
|
||||
|
||||
// Reflector 입력: research 결과를 input으로 받고 plan/originalPrompt를 priorResults로 받는다
|
||||
expect(reflector.lastInput).toContain('Research: dense factual synthesis');
|
||||
expect(reflector.lastOptions?.priorResults?.plan).toContain('Plan: rigorous blueprint');
|
||||
expect(reflector.lastOptions?.priorResults?.originalPrompt).toBe('Test prompt');
|
||||
|
||||
// Writer 는 (1) writer 단계와 (2) Phase5 generateProactiveAdvice(advisor 모드)에서 각각 1회씩 호출된다.
|
||||
// 첫 번째 호출(메인 writer 단계)이 reflection 을 받았어야 한다.
|
||||
expect(writer.calls.length).toBeGreaterThanOrEqual(1);
|
||||
const writerMainCall = writer.calls.find(c => c.options?.config?.role === 'writer');
|
||||
expect(writerMainCall).toBeDefined();
|
||||
expect(writerMainCall?.options?.priorResults?.reflection).toContain('Guidance for Writer');
|
||||
});
|
||||
|
||||
test('Reflector 미주입 시 기존 3단계 파이프라인이 그대로 동작해야 한다 (역호환성)', async () => {
|
||||
const writer = new SpyAgent('Final report without reflection.');
|
||||
const engine = new AgentEngine(
|
||||
new SpyAgent('Plan output of sufficient length to pass validation.'),
|
||||
new SpyAgent('Research output of sufficient length to pass validation.'),
|
||||
writer
|
||||
// reflector 미전달
|
||||
);
|
||||
|
||||
const stages: PipelineStage[] = [];
|
||||
const result = await engine.runMission(
|
||||
'reflect_002', 'Test prompt', 'ctx',
|
||||
createAbortSignal(),
|
||||
(stage) => { stages.push(stage); }
|
||||
);
|
||||
|
||||
expect(result).toContain('Final report without reflection');
|
||||
expect(stages).not.toContain('reflector');
|
||||
// Writer는 reflection 없이도 동작 (priorResults.reflection은 빈 문자열)
|
||||
expect(writer.lastOptions?.priorResults?.reflection ?? '').toBe('');
|
||||
});
|
||||
|
||||
test('config.enableReflection=false 옵션으로 Reflector 가 주입돼있어도 스킵되어야 한다', async () => {
|
||||
const reflector = new SpyAgent('should not be called');
|
||||
const writer = new SpyAgent('Final report bypassing reflection.');
|
||||
const engine = new AgentEngine(
|
||||
new SpyAgent('Plan output of sufficient length to pass validation.'),
|
||||
new SpyAgent('Research output of sufficient length to pass validation.'),
|
||||
writer,
|
||||
reflector
|
||||
);
|
||||
|
||||
const stages: PipelineStage[] = [];
|
||||
await engine.runMission(
|
||||
'reflect_003', 'Test prompt', 'ctx',
|
||||
createAbortSignal(),
|
||||
(stage) => { stages.push(stage); },
|
||||
{ config: { enableReflection: false } }
|
||||
);
|
||||
|
||||
expect(reflector.callCount).toBe(0);
|
||||
expect(stages).not.toContain('reflector');
|
||||
// Writer는 정상 실행되고 reflection은 빈 문자열
|
||||
expect(writer.lastOptions?.priorResults?.reflection ?? '').toBe('');
|
||||
});
|
||||
|
||||
test('Reflector 실패는 soft-fail — Writer 가 빈 critique 으로 진행되어 미션이 완료되어야 한다', async () => {
|
||||
const reflector = new ThrowingAgent('Failed to fetch'); // transient → 재시도 소진 후 throw
|
||||
const writer = new SpyAgent('Final report despite reflector failure.');
|
||||
const engine = new AgentEngine(
|
||||
new SpyAgent('Plan output of sufficient length to pass validation.'),
|
||||
new SpyAgent('Research output of sufficient length to pass validation.'),
|
||||
writer,
|
||||
reflector
|
||||
);
|
||||
|
||||
const result = await engine.runMission(
|
||||
'reflect_004', 'Test prompt', 'ctx', createAbortSignal(), noopProgress
|
||||
);
|
||||
|
||||
expect(result).toContain('Final report despite reflector failure');
|
||||
// Writer는 빈 reflection으로 진행되었어야 함
|
||||
expect(writer.lastOptions?.priorResults?.reflection ?? '').toBe('');
|
||||
}, 60000);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// Test Suite 5: Performance Benchmark
|
||||
// ═══════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user