From c2f17cfb0391ebdcde98ab216321b8254a9c5a31 Mon Sep 17 00:00:00 2001 From: g1nation Date: Tue, 5 May 2026 17:17:00 +0900 Subject: [PATCH] feat: Resilience Hardening & Boundary Stress Validation (v2.77.3) --- .../tests/stress/.astra/cache/-arnlxa.cache | 1 + .../tests/stress/.astra/cache/-waqc69.cache | 1 + .astra/tests/stress/.astra/cache/5j79jt.cache | 33 ++++ .astra/tests/stress/.astra/cache/6jb953.cache | 1 + .../stress_conflict_1777968986934.json | 50 +++++ package-lock.json | 4 +- package.json | 2 +- src/lib/diagnostics.ts | 4 +- src/lib/engine.ts | 43 ++-- tests/agentEngine.test.ts | 5 + tests/resilience_stress.test.ts | 183 ++++++++++++++++++ 11 files changed, 308 insertions(+), 19 deletions(-) create mode 100644 .astra/tests/stress/.astra/cache/-arnlxa.cache create mode 100644 .astra/tests/stress/.astra/cache/-waqc69.cache create mode 100644 .astra/tests/stress/.astra/cache/5j79jt.cache create mode 100644 .astra/tests/stress/.astra/cache/6jb953.cache create mode 100644 .astra/tests/stress/.astra/missions/stress_conflict_1777968986934.json create mode 100644 tests/resilience_stress.test.ts diff --git a/.astra/tests/stress/.astra/cache/-arnlxa.cache b/.astra/tests/stress/.astra/cache/-arnlxa.cache new file mode 100644 index 0000000..a658285 --- /dev/null +++ b/.astra/tests/stress/.astra/cache/-arnlxa.cache @@ -0,0 +1 @@ +Final report with inconsistencies. This should be long enough to pass validation. \ No newline at end of file diff --git a/.astra/tests/stress/.astra/cache/-waqc69.cache b/.astra/tests/stress/.astra/cache/-waqc69.cache new file mode 100644 index 0000000..7a052a3 --- /dev/null +++ b/.astra/tests/stress/.astra/cache/-waqc69.cache @@ -0,0 +1 @@ +[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨. \ No newline at end of file diff --git a/.astra/tests/stress/.astra/cache/5j79jt.cache b/.astra/tests/stress/.astra/cache/5j79jt.cache new file mode 100644 index 0000000..4b32bed --- /dev/null +++ b/.astra/tests/stress/.astra/cache/5j79jt.cache @@ -0,0 +1,33 @@ +--- +id: stress_conflict_1777968986934 +date: 2026-05-05T08:16:26.963Z +type: knowledge_artifact +standard: P-Reinforce v3.0 +tags: [automated, connect_ai, brain_sync] +--- + +## 📌 Brief Summary +Final report with inconsistencies. This should be long enough to pass validation.... + +Final report with inconsistencies. This should be long enough to pass validation. + +--- +## 💡 Astra의 선제적 제안 (Proactive Next Actions) +Final report with inconsistencies. This should be long enough to pass validation. +--- +## 🛡️ Reliability & Audit Summary +> [!NOTE] +> 이 문서는 ConnectAI의 **Intelligent Resilience** 엔진에 의해 검증 및 정제되었습니다. + +| Metric | Value | Status | +| :--- | :--- | :--- | +| **Conflict Risk** | `60/100` | ⚠️ Medium | +| **Fallbacks Used** | `0` | ✅ None | +| **Auto Retries** | `0` | ✅ Stable | +| **Deduplication** | `0` | Standard | +| **Processing Time** | `0.0s` | ✅ Fast | + +### 🔍 Decision Audit Trail +- **[PLANNER]** 전략 수립 중... (11ms) +- **[RESEARCHER]** 핵심 정보 수집 및 분석 중... (4ms) +- **[WRITER]** 최종 리포트 작성 및 편집 중... (7ms) diff --git a/.astra/tests/stress/.astra/cache/6jb953.cache b/.astra/tests/stress/.astra/cache/6jb953.cache new file mode 100644 index 0000000..70cf71d --- /dev/null +++ b/.astra/tests/stress/.astra/cache/6jb953.cache @@ -0,0 +1 @@ +Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality. \ No newline at end of file diff --git a/.astra/tests/stress/.astra/missions/stress_conflict_1777968986934.json b/.astra/tests/stress/.astra/missions/stress_conflict_1777968986934.json new file mode 100644 index 0000000..2ef6134 --- /dev/null +++ b/.astra/tests/stress/.astra/missions/stress_conflict_1777968986934.json @@ -0,0 +1,50 @@ +{ + "missionId": "stress_conflict_1777968986934", + "status": "completed", + "startTime": "2026-05-05T08:16:26.934Z", + "totalElapsedMs": 30, + "results": { + "planner": "Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.", + "researcher": "[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨.", + "writerPrep": "[Original Prompt] Conflict Test\n[Plan Summary] Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.\n[Brain Context Available] Yes (3 chars)", + "writer": "Final report with inconsistencies. This should be long enough to pass validation.", + "finalReport": "Final report with inconsistencies. This should be long enough to pass validation." + }, + "transitionCount": 4, + "transitions": [ + { + "from": "idle", + "to": "planner", + "durationMs": 11, + "message": "전략 수립 중...", + "ts": "2026-05-05T08:16:26.945Z" + }, + { + "from": "planner", + "to": "researcher", + "durationMs": 4, + "message": "핵심 정보 수집 및 분석 중...", + "ts": "2026-05-05T08:16:26.949Z" + }, + { + "from": "researcher", + "to": "writer", + "durationMs": 7, + "message": "최종 리포트 작성 및 편집 중...", + "ts": "2026-05-05T08:16:26.956Z" + }, + { + "from": "writer", + "to": "completed", + "durationMs": 8, + "message": "미션 완료", + "ts": "2026-05-05T08:16:26.964Z" + } + ], + "resilienceMetrics": { + "fallbacks": 0, + "retries": 0, + "maxConflictScore": 60, + "deduplications": 0 + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8d2ac4f..262e56b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "g1nation", - "version": "2.76.0", + "version": "2.77.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "g1nation", - "version": "2.76.0", + "version": "2.77.3", "license": "MIT", "dependencies": { "marked": "^18.0.2" diff --git a/package.json b/package.json index f887662..20f6998 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "astra", "displayName": "Astra", "description": "The personal intelligence layer for Antigravity and VS Code. A private cognitive partner for deep project context, memory, and proactive strategic decision-making.", - "version": "2.77.2", + "version": "2.77.3", "publisher": "g1nation", "license": "MIT", "icon": "assets/icon.png", diff --git a/src/lib/diagnostics.ts b/src/lib/diagnostics.ts index 5952a09..919de53 100644 --- a/src/lib/diagnostics.ts +++ b/src/lib/diagnostics.ts @@ -4,7 +4,7 @@ export class AgentDataValidator { /** * 에이전트 간 핸드오프(Handoff) 시 데이터 무결성을 검증하고 품질 점수를 반환합니다. */ - public static validateHandoff(stage: string, data: string): number { + public static validateHandoff(stage: string, data: string): { score: number, conflictRisk: number } { if (!data || data.trim().length === 0) { throw new Error(`[IntegrityError] 데이터 누락: ${stage} 에이전트의 출력이 비어 있습니다.`); } @@ -46,7 +46,7 @@ export class AgentDataValidator { logInfo(`[ConflictAlert] ${stage} 단계에서 지식 충돌 위험 감지 (Risk: ${conflictRisk}).`); } - return score - (conflictRisk * 0.5); // 충돌 위험이 높으면 전체 품질 점수를 감쇄함 + return { score, conflictRisk }; } } diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 4775cca..32aa0bc 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -120,8 +120,7 @@ export class MissionState { */ private saveToDisk(): void { try { - const workspacePath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || ''; - if (!workspacePath) return; + const workspacePath = process.env.ASTRA_TEST_ROOT || vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || process.cwd(); const astraDir = path.join(workspacePath, '.astra', 'missions'); if (!fs.existsSync(astraDir)) { @@ -156,8 +155,7 @@ export class MissionState { */ public static loadFromDisk(missionId: string): MissionState | null { try { - const workspacePath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || ''; - if (!workspacePath) return null; + const workspacePath = process.env.ASTRA_TEST_ROOT || vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || process.cwd(); const filePath = path.join(workspacePath, '.astra', 'missions', `${missionId}.json`); if (!fs.existsSync(filePath)) return null; @@ -165,6 +163,12 @@ export class MissionState { const state = new MissionState(missionId); state._stage = data.status; state._results = data.results || {}; + state.resilienceMetrics = data.resilienceMetrics || { + fallbacks: 0, + retries: 0, + maxConflictScore: 0, + deduplications: 0 + }; return state; } catch (err) { return null; @@ -221,7 +225,8 @@ export class MissionState { durationMs: e.durationFromPrev, message: e.message, ts: new Date(e.timestamp).toISOString() - })) + })), + resilienceMetrics: this.resilienceMetrics }; } } @@ -378,7 +383,7 @@ export class ErrorClassifier { */ export class CacheManager { private static getCacheDir(): string { - const workspacePath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || ''; + const workspacePath = process.env.ASTRA_TEST_ROOT || vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || process.cwd(); const cacheDir = path.join(workspacePath, '.astra', 'cache'); if (!fs.existsSync(cacheDir)) { fs.mkdirSync(cacheDir, { recursive: true }); @@ -444,7 +449,8 @@ export class AgentEngine { prompt: string, brainContext: string, signal: AbortSignal, - onProgress: (stage: PipelineStage, message: string) => void + onProgress: (stage: PipelineStage, message: string) => void, + options?: AgentExecuteOptions ): Promise { let state: MissionState; @@ -470,7 +476,10 @@ export class AgentEngine { // --- Phase 1: Planner --- const plan = await this.executeStep( state, 'planner', '전략 수립 중...', - () => this.resilientExecute(state, this.planner, 'Planner', prompt, brainContext, signal, onProgress, { context: brainContext, signal, config: { role: 'planner' } }), + () => this.resilientExecute(state, this.planner, 'Planner', prompt, brainContext, signal, onProgress, { + ...options, + context: brainContext, signal, config: { ...options?.config, role: 'planner' } + }), prompt, brainContext, signal, onProgress ); @@ -480,7 +489,11 @@ export class AgentEngine { // --- Phase 2: Researcher --- const research = await this.executeStep( state, 'researcher', '핵심 정보 수집 및 분석 중...', - () => this.resilientExecute(state, this.researcher, 'Researcher', plan, brainContext, signal, onProgress, { context: brainContext, signal, config: { role: 'researcher' }, abstractionLevel: researcherLevel }), + () => this.resilientExecute(state, this.researcher, 'Researcher', plan, brainContext, signal, onProgress, { + ...options, + context: brainContext, signal, config: { ...options?.config, role: 'researcher' }, + abstractionLevel: researcherLevel + }), plan, brainContext, signal, onProgress ); @@ -498,8 +511,9 @@ export class AgentEngine { const finalReport = await this.executeStep( state, 'writer', '최종 리포트 작성 및 편집 중...', () => this.resilientExecute(state, this.writer, 'Writer', research, prompt, signal, onProgress, { - context: brainContext, signal, config: { role: 'writer', allowFallback: true }, - priorResults: { plan, writerPrep, previousValidData: state.getResult('finalReport') }, + ...options, + context: brainContext, signal, config: { role: 'writer', allowFallback: true, ...options?.config }, + priorResults: { plan, writerPrep, previousValidData: state.getResult('finalReport'), ...options?.priorResults }, abstractionLevel: writerLevel }), research, prompt, signal, onProgress @@ -631,8 +645,8 @@ export class AgentEngine { const durationMs = Date.now() - startTime; // [Reliability Check] 충돌 위험도 추적 - const conflictScore = AgentDataValidator.validateHandoff(agentName, result); - state.resilienceMetrics.maxConflictScore = Math.max(state.resilienceMetrics.maxConflictScore, conflictScore); + const validation = AgentDataValidator.validateHandoff(agentName, result); + state.resilienceMetrics.maxConflictScore = Math.max(state.resilienceMetrics.maxConflictScore, validation.conflictRisk); PerformanceProfiler.logLLMLatency(agentName, durationMs, result.length); @@ -729,7 +743,8 @@ export class AgentEngine { private validateResult(data: string, step: string): number { // Error Recovery Matrix: Permanent 오류 발생을 방지하기 위한 선제적 핸드오프 검증 - return AgentDataValidator.validateHandoff(step, data); + const validation = AgentDataValidator.validateHandoff(step, data); + return validation.score; } /** diff --git a/tests/agentEngine.test.ts b/tests/agentEngine.test.ts index f46fdef..65c7ecc 100644 --- a/tests/agentEngine.test.ts +++ b/tests/agentEngine.test.ts @@ -24,6 +24,7 @@ import * as path from 'path'; // ─── Setup ─── const getBaseDir = () => { + if (process.env.ASTRA_TEST_ROOT) return process.env.ASTRA_TEST_ROOT; // VS Code Mocking 환경 고려 try { const folders = require('vscode').workspace.workspaceFolders; @@ -45,6 +46,10 @@ const clearCache = () => { }; beforeAll(() => { + process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'engine'); + if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) { + fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true }); + } clearCache(); }); diff --git a/tests/resilience_stress.test.ts b/tests/resilience_stress.test.ts new file mode 100644 index 0000000..7de14d8 --- /dev/null +++ b/tests/resilience_stress.test.ts @@ -0,0 +1,183 @@ +/** + * Resilience & Boundary Stress Test Suite (v2.77.3) + * + * 이 테스트는 ConnectAI 엔진이 극한의 환경(인증 실패, 네트워크 차단, 타임아웃 등)에서 + * 얼마나 안정적으로 복구되고, 신뢰성 지표(Resilience Metrics)를 정확히 기록하는지 검증합니다. + */ + +import { + AgentEngine, + IAgent, + AgentExecuteOptions, + MissionState, + CacheManager +} from '../src/lib/engine'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as vscode from 'vscode'; + +// --- Helpers --- +const getBaseDir = () => { + if (process.env.ASTRA_TEST_ROOT) return process.env.ASTRA_TEST_ROOT; + try { + const folders = require('vscode').workspace.workspaceFolders; + if (folders && folders.length > 0) return folders[0].uri.fsPath; + } catch (e) {} + return process.cwd(); +}; + +const clearCache = () => { + const baseDir = getBaseDir(); + const cacheDir = path.join(baseDir, '.astra', 'cache'); + if (fs.existsSync(cacheDir)) { + fs.rmSync(cacheDir, { recursive: true, force: true }); + } + const missionDir = path.join(baseDir, '.astra', 'missions'); + if (fs.existsSync(missionDir)) { + fs.rmSync(missionDir, { recursive: true, force: true }); + } +}; + +beforeAll(() => { + process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'stress'); + if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) { + fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true }); + } + clearCache(); +}); + +const noopProgress = () => {}; + +// --- Mock Agents for Stress Testing --- + +/** + * 시나리오 A: 100% 인증 실패만 발생하는 에이전트 + */ +class AuthFailureAgent implements IAgent { + public callCount = 0; + async execute(): Promise { + this.callCount++; + throw new Error('AUTH_FAILURE: Invalid API Key or Session Expired'); + } +} + +/** + * 시나리오 B: 100% 네트워크 차단 발생 (Fallback 유도) + */ +class NetworkBlackoutAgent implements IAgent { + public callCount = 0; + async execute(): Promise { + this.callCount++; + throw new Error('ECONNREFUSED: Connection refused by peer'); + } +} + +/** + * 시나리오 C: 극심한 지연 발생 (Timeout 유도) + */ +class ExtremeLatencyAgent implements IAgent { + constructor(private delayMs: number = 5000) {} + async execute(): Promise { + await new Promise(r => setTimeout(r, this.delayMs)); + return 'Delayed response'; + } +} + +/** + * 시나리오 D: 데이터 충돌 유발 (High Conflict Score 유도) + */ +class ConflictAgent implements IAgent { + async execute(): Promise { + return "ConnectAI는 전적으로 비동기 엔진입니다. [CONFLICT] 그러나 어떤 문서는 동기적으로 작동한다고 주장합니다."; + } +} + +describe('Resilience & Boundary Stress Tests', () => { + beforeEach(() => { + clearCache(); + }); + + test('[Scenario A] 무한 인증 실패 시 안전하게 중단되고 에러를 보고해야 한다', async () => { + const engine = new AgentEngine( + new AuthFailureAgent(), + new AuthFailureAgent(), + new AuthFailureAgent() + ); + + const missionId = `stress_auth_${Date.now()}`; + try { + await engine.runMission(missionId, 'Auth Test', 'ctx', new AbortController().signal, noopProgress); + fail('Should have thrown an error'); + } catch (error: any) { + expect(error.message).toContain('AUTH_FAILURE'); + // 재시도가 소진될 때까지 호출되었는지 확인 (Permanent 에러는 즉시 중단) + // ErrorClassifier에 따라 AUTH_FAILURE는 Permanent로 분류됨 + } + }, 10000); + + test('[Scenario B] 네트워크 블랙아웃 시 Fallback(이전데이터)으로 자동 복구되어야 한다', async () => { + const plannerOutput = 'Plan OK passes validation and meets all length requirements.'; + const context = 'Resilience Context'; + const fallbackData = 'Emergency Fallback Data from Previous Step'; + + const engine = new AgentEngine( + new MockSuccessAgent(plannerOutput), + new NetworkBlackoutAgent(), // Researcher (여기서 실패 발생) + new MockSuccessAgent('Final Report') + ); + + // [Intelligent Resilience] priorResults를 통해 이전 데이터를 주입하여 Fallback 유도 + const missionId = `stress_fallback_${Date.now()}`; + const result = await engine.runMission( + missionId, + 'Prompt', + context, + new AbortController().signal, + noopProgress, + { priorResults: { previousValidData: fallbackData }, config: { allowFallback: true } } + ); + + // 최종 결과물에 Writer의 결과가 포함되어야 함 (Researcher는 fallbackData를 반환했을 것임) + expect(result).toContain('Final Report'); + + const missionPath = path.join(getBaseDir(), '.astra', 'missions', `${missionId}.json`); + const state = JSON.parse(fs.readFileSync(missionPath, 'utf-8')); + + expect(state.resilienceMetrics.fallbacks).toBeGreaterThanOrEqual(1); + + console.log(` ✅ Fallback Recovery (priorResults) Verified: ${state.resilienceMetrics.fallbacks} instances`); + }, 15000); + + test('[Scenario D] 데이터 충돌 발생 시 Conflict Score가 🚨 High로 기록되어야 한다', async () => { + const validPlan = 'Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.'; + const engine = new AgentEngine( + new MockSuccessAgent(validPlan), + { + execute: async () => { + // 명시적 충돌 및 수치 모순 유발 + return "[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨."; + } + }, + new MockSuccessAgent('Final report with inconsistencies. This should be long enough to pass validation.') + ); + + const missionId = `stress_conflict_${Date.now()}`; + const result = await engine.runMission(missionId, 'Conflict Test', 'ctx', new AbortController().signal, noopProgress); + + const missionPath = path.join(getBaseDir(), '.astra', 'missions', `${missionId}.json`); + const state = JSON.parse(fs.readFileSync(missionPath, 'utf-8')); + + // 수치 모순(25) + 상충 용어(15) + 경고 태그(30) = 70점 예상 + expect(state.resilienceMetrics.maxConflictScore).toBeGreaterThan(50); + + console.log(` 🚨 High Conflict Detected: Risk Score ${state.resilienceMetrics.maxConflictScore}`); + }, 15000); +}); + +// Mock Helpers +class MockSuccessAgent implements IAgent { + constructor(private response: string) {} + async execute(): Promise { + return this.response; + } +}