feat: Resilience Hardening & Boundary Stress Validation (v2.77.3)

2026-05-05 17:17:00 +09:00
parent cf10d14148
commit c2f17cfb03
11 changed files with 308 additions and 19 deletions
@@ -24,6 +24,7 @@ import * as path from 'path';

 // ─── Setup ───
 const getBaseDir = () => {
+    if (process.env.ASTRA_TEST_ROOT) return process.env.ASTRA_TEST_ROOT;
    // VS Code Mocking 환경 고려
    try {
        const folders = require('vscode').workspace.workspaceFolders;
@@ -45,6 +46,10 @@ const clearCache = () => {
 };

 beforeAll(() => {
+    process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'engine');
+    if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) {
+        fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true });
+    }
    clearCache();
 });

@@ -0,0 +1,183 @@
+/**
+ * Resilience & Boundary Stress Test Suite (v2.77.3)
+ * 
+ * 이 테스트는 ConnectAI 엔진이 극한의 환경(인증 실패, 네트워크 차단, 타임아웃 등)에서
+ * 얼마나 안정적으로 복구되고, 신뢰성 지표(Resilience Metrics)를 정확히 기록하는지 검증합니다.
+ */
+
+import {
+    AgentEngine,
+    IAgent,
+    AgentExecuteOptions,
+    MissionState,
+    CacheManager
+} from '../src/lib/engine';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as vscode from 'vscode';
+
+// --- Helpers ---
+const getBaseDir = () => {
+    if (process.env.ASTRA_TEST_ROOT) return process.env.ASTRA_TEST_ROOT;
+    try {
+        const folders = require('vscode').workspace.workspaceFolders;
+        if (folders && folders.length > 0) return folders[0].uri.fsPath;
+    } catch (e) {}
+    return process.cwd();
+};
+
+const clearCache = () => {
+    const baseDir = getBaseDir();
+    const cacheDir = path.join(baseDir, '.astra', 'cache');
+    if (fs.existsSync(cacheDir)) {
+        fs.rmSync(cacheDir, { recursive: true, force: true });
+    }
+    const missionDir = path.join(baseDir, '.astra', 'missions');
+    if (fs.existsSync(missionDir)) {
+        fs.rmSync(missionDir, { recursive: true, force: true });
+    }
+};
+
+beforeAll(() => {
+    process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'stress');
+    if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) {
+        fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true });
+    }
+    clearCache();
+});
+
+const noopProgress = () => {};
+
+// --- Mock Agents for Stress Testing ---
+
+/**
+ * 시나리오 A: 100% 인증 실패만 발생하는 에이전트
+ */
+class AuthFailureAgent implements IAgent {
+    public callCount = 0;
+    async execute(): Promise<string> {
+        this.callCount++;
+        throw new Error('AUTH_FAILURE: Invalid API Key or Session Expired');
+    }
+}
+
+/**
+ * 시나리오 B: 100% 네트워크 차단 발생 (Fallback 유도)
+ */
+class NetworkBlackoutAgent implements IAgent {
+    public callCount = 0;
+    async execute(): Promise<string> {
+        this.callCount++;
+        throw new Error('ECONNREFUSED: Connection refused by peer');
+    }
+}
+
+/**
+ * 시나리오 C: 극심한 지연 발생 (Timeout 유도)
+ */
+class ExtremeLatencyAgent implements IAgent {
+    constructor(private delayMs: number = 5000) {}
+    async execute(): Promise<string> {
+        await new Promise(r => setTimeout(r, this.delayMs));
+        return 'Delayed response';
+    }
+}
+
+/**
+ * 시나리오 D: 데이터 충돌 유발 (High Conflict Score 유도)
+ */
+class ConflictAgent implements IAgent {
+    async execute(): Promise<string> {
+        return "ConnectAI는 전적으로 비동기 엔진입니다. [CONFLICT] 그러나 어떤 문서는 동기적으로 작동한다고 주장합니다.";
+    }
+}
+
+describe('Resilience & Boundary Stress Tests', () => {
+    beforeEach(() => {
+        clearCache();
+    });
+
+    test('[Scenario A] 무한 인증 실패 시 안전하게 중단되고 에러를 보고해야 한다', async () => {
+        const engine = new AgentEngine(
+            new AuthFailureAgent(),
+            new AuthFailureAgent(),
+            new AuthFailureAgent()
+        );
+
+        const missionId = `stress_auth_${Date.now()}`;
+        try {
+            await engine.runMission(missionId, 'Auth Test', 'ctx', new AbortController().signal, noopProgress);
+            fail('Should have thrown an error');
+        } catch (error: any) {
+            expect(error.message).toContain('AUTH_FAILURE');
+            // 재시도가 소진될 때까지 호출되었는지 확인 (Permanent 에러는 즉시 중단)
+            // ErrorClassifier에 따라 AUTH_FAILURE는 Permanent로 분류됨
+        }
+    }, 10000);
+
+    test('[Scenario B] 네트워크 블랙아웃 시 Fallback(이전데이터)으로 자동 복구되어야 한다', async () => {
+        const plannerOutput = 'Plan OK passes validation and meets all length requirements.';
+        const context = 'Resilience Context';
+        const fallbackData = 'Emergency Fallback Data from Previous Step';
+
+        const engine = new AgentEngine(
+            new MockSuccessAgent(plannerOutput),
+            new NetworkBlackoutAgent(), // Researcher (여기서 실패 발생)
+            new MockSuccessAgent('Final Report')
+        );
+
+        // [Intelligent Resilience] priorResults를 통해 이전 데이터를 주입하여 Fallback 유도
+        const missionId = `stress_fallback_${Date.now()}`;
+        const result = await engine.runMission(
+            missionId, 
+            'Prompt', 
+            context, 
+            new AbortController().signal, 
+            noopProgress,
+            { priorResults: { previousValidData: fallbackData }, config: { allowFallback: true } }
+        );
+
+        // 최종 결과물에 Writer의 결과가 포함되어야 함 (Researcher는 fallbackData를 반환했을 것임)
+        expect(result).toContain('Final Report');
+        
+        const missionPath = path.join(getBaseDir(), '.astra', 'missions', `${missionId}.json`);
+        const state = JSON.parse(fs.readFileSync(missionPath, 'utf-8'));
+        
+        expect(state.resilienceMetrics.fallbacks).toBeGreaterThanOrEqual(1);
+        
+        console.log(`   ✅ Fallback Recovery (priorResults) Verified: ${state.resilienceMetrics.fallbacks} instances`);
+    }, 15000);
+
+    test('[Scenario D] 데이터 충돌 발생 시 Conflict Score가 🚨 High로 기록되어야 한다', async () => {
+        const validPlan = 'Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.';
+        const engine = new AgentEngine(
+            new MockSuccessAgent(validPlan),
+            {
+                execute: async () => {
+                    // 명시적 충돌 및 수치 모순 유발
+                    return "[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨.";
+                }
+            },
+            new MockSuccessAgent('Final report with inconsistencies. This should be long enough to pass validation.')
+        );
+
+        const missionId = `stress_conflict_${Date.now()}`;
+        const result = await engine.runMission(missionId, 'Conflict Test', 'ctx', new AbortController().signal, noopProgress);
+
+        const missionPath = path.join(getBaseDir(), '.astra', 'missions', `${missionId}.json`);
+        const state = JSON.parse(fs.readFileSync(missionPath, 'utf-8'));
+
+        // 수치 모순(25) + 상충 용어(15) + 경고 태그(30) = 70점 예상
+        expect(state.resilienceMetrics.maxConflictScore).toBeGreaterThan(50);
+        
+        console.log(`   🚨 High Conflict Detected: Risk Score ${state.resilienceMetrics.maxConflictScore}`);
+    }, 15000);
+});
+
+// Mock Helpers
+class MockSuccessAgent implements IAgent {
+    constructor(private response: string) {}
+    async execute(): Promise<string> {
+        return this.response;
+    }
+}