bluemsi/connectai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Resilience Hardening & Boundary Stress Validation (v2.77.3)

Browse Source
This commit is contained in:
g1nation
2026-05-05 17:17:00 +09:00
parent cf10d14148
commit c2f17cfb03
11 changed files with 308 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.astra/tests/stress/.astra/cache/-arnlxa.cache
Vendored
+1
View File
@@ -0,0 +1 @@
Final report with inconsistencies. This should be long enough to pass validation.
.astra/tests/stress/.astra/cache/-waqc69.cache
Vendored
+1
View File
@@ -0,0 +1 @@
[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨.
.astra/tests/stress/.astra/cache/5j79jt.cache
Vendored
+33
View File
@@ -0,0 +1,33 @@
---
id: stress_conflict_1777968986934
date: 2026-05-05T08:16:26.963Z
type: knowledge_artifact
standard: P-Reinforce v3.0
tags: [automated, connect_ai, brain_sync]
---
## 📌 Brief Summary
Final report with inconsistencies. This should be long enough to pass validation....
Final report with inconsistencies. This should be long enough to pass validation.
---
## 💡 Astra의 선제적 제안 (Proactive Next Actions)
Final report with inconsistencies. This should be long enough to pass validation.
---
## 🛡️ Reliability & Audit Summary
> [!NOTE]
> 이 문서는 ConnectAI의 **Intelligent Resilience** 엔진에 의해 검증 및 정제되었습니다.
| Metric | Value | Status |
| :--- | :--- | :--- |
| **Conflict Risk** | `60/100` | ⚠️ Medium |
| **Fallbacks Used** | `0` | ✅ None |
| **Auto Retries** | `0` | ✅ Stable |
| **Deduplication** | `0` | Standard |
| **Processing Time** | `0.0s` | ✅ Fast |
### 🔍 Decision Audit Trail
- **[PLANNER]** 전략 수립 중... (11ms)
- **[RESEARCHER]** 핵심 정보 수집 및 분석 중... (4ms)
- **[WRITER]** 최종 리포트 작성 및 편집 중... (7ms)
.astra/tests/stress/.astra/cache/6jb953.cache
Vendored
+1
View File
@@ -0,0 +1 @@
Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.
.astra/tests/stress/.astra/missions/stress_conflict_1777968986934.json
+50
View File
@@ -0,0 +1,50 @@
{
"missionId": "stress_conflict_1777968986934",
"status": "completed",
"startTime": "2026-05-05T08:16:26.934Z",
"totalElapsedMs": 30,
"results": {
"planner": "Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.",
"researcher": "[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨.",
"writerPrep": "[Original Prompt] Conflict Test\n[Plan Summary] Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.\n[Brain Context Available] Yes (3 chars)",
"writer": "Final report with inconsistencies. This should be long enough to pass validation.",
"finalReport": "Final report with inconsistencies. This should be long enough to pass validation."
},
"transitionCount": 4,
"transitions": [
{
"from": "idle",
"to": "planner",
"durationMs": 11,
"message": "전략 수립 중...",
"ts": "2026-05-05T08:16:26.945Z"
},
{
"from": "planner",
"to": "researcher",
"durationMs": 4,
"message": "핵심 정보 수집 및 분석 중...",
"ts": "2026-05-05T08:16:26.949Z"
},
{
"from": "researcher",
"to": "writer",
"durationMs": 7,
"message": "최종 리포트 작성 및 편집 중...",
"ts": "2026-05-05T08:16:26.956Z"
},
{
"from": "writer",
"to": "completed",
"durationMs": 8,
"message": "미션 완료",
"ts": "2026-05-05T08:16:26.964Z"
}
],
"resilienceMetrics": {
"fallbacks": 0,
"retries": 0,
"maxConflictScore": 60,
"deduplications": 0
}
}
package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "g1nation",
"version": "2.76.0",
"version": "2.77.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "g1nation",
"version": "2.76.0",
"version": "2.77.3",
"license": "MIT",
"dependencies": {
"marked": "^18.0.2"
package.json
+1 -1
View File
@@ -2,7 +2,7 @@
"name": "astra",
"displayName": "Astra",
"description": "The personal intelligence layer for Antigravity and VS Code. A private cognitive partner for deep project context, memory, and proactive strategic decision-making.",
"version": "2.77.2",
"version": "2.77.3",
"publisher": "g1nation",
"license": "MIT",
"icon": "assets/icon.png",
src/lib/diagnostics.ts
+2 -2
View File
@@ -4,7 +4,7 @@ export class AgentDataValidator {
/**
* 에이전트 간 핸드오프(Handoff) 시 데이터 무결성을 검증하고 품질 점수를 반환합니다.
*/
public static validateHandoff(stage: string, data: string): number {
public static validateHandoff(stage: string, data: string): { score: number, conflictRisk: number } {
if (!data || data.trim().length === 0) {
throw new Error(`[IntegrityError] 데이터 누락: ${stage} 에이전트의 출력이 비어 있습니다.`);
}
@@ -46,7 +46,7 @@ export class AgentDataValidator {
logInfo(`[ConflictAlert] ${stage} 단계에서 지식 충돌 위험 감지 (Risk: ${conflictRisk}).`);
}
return score - (conflictRisk * 0.5); // 충돌 위험이 높으면 전체 품질 점수를 감쇄함
return { score, conflictRisk };
}
}
src/lib/engine.ts
+29 -14
View File
@@ -120,8 +120,7 @@ export class MissionState {
*/
private saveToDisk(): void {
try {
const workspacePath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || '';
if (!workspacePath) return;
const workspacePath = process.env.ASTRA_TEST_ROOT || vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || process.cwd();
const astraDir = path.join(workspacePath, '.astra', 'missions');
if (!fs.existsSync(astraDir)) {
@@ -156,8 +155,7 @@ export class MissionState {
*/
public static loadFromDisk(missionId: string): MissionState | null {
try {
const workspacePath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || '';
if (!workspacePath) return null;
const workspacePath = process.env.ASTRA_TEST_ROOT || vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || process.cwd();
const filePath = path.join(workspacePath, '.astra', 'missions', `${missionId}.json`);
if (!fs.existsSync(filePath)) return null;
@@ -165,6 +163,12 @@ export class MissionState {
const state = new MissionState(missionId);
state._stage = data.status;
state._results = data.results || {};
state.resilienceMetrics = data.resilienceMetrics || {
fallbacks: 0,
retries: 0,
maxConflictScore: 0,
deduplications: 0
};
return state;
} catch (err) {
return null;
@@ -221,7 +225,8 @@ export class MissionState {
durationMs: e.durationFromPrev,
message: e.message,
ts: new Date(e.timestamp).toISOString()
}))
})),
resilienceMetrics: this.resilienceMetrics
};
}
}
@@ -378,7 +383,7 @@ export class ErrorClassifier {
*/
export class CacheManager {
private static getCacheDir(): string {
const workspacePath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || '';
const workspacePath = process.env.ASTRA_TEST_ROOT || vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || process.cwd();
const cacheDir = path.join(workspacePath, '.astra', 'cache');
if (!fs.existsSync(cacheDir)) {
fs.mkdirSync(cacheDir, { recursive: true });
@@ -444,7 +449,8 @@ export class AgentEngine {
prompt: string,
brainContext: string,
signal: AbortSignal,
onProgress: (stage: PipelineStage, message: string) => void
onProgress: (stage: PipelineStage, message: string) => void,
options?: AgentExecuteOptions
): Promise<string> {
let state: MissionState;
@@ -470,7 +476,10 @@ export class AgentEngine {
// --- Phase 1: Planner ---
const plan = await this.executeStep(
state, 'planner', '전략 수립 중...',
() => this.resilientExecute(state, this.planner, 'Planner', prompt, brainContext, signal, onProgress, { context: brainContext, signal, config: { role: 'planner' } }),
() => this.resilientExecute(state, this.planner, 'Planner', prompt, brainContext, signal, onProgress, {
...options,
context: brainContext, signal, config: { ...options?.config, role: 'planner' }
}),
prompt, brainContext, signal, onProgress
);
@@ -480,7 +489,11 @@ export class AgentEngine {
// --- Phase 2: Researcher ---
const research = await this.executeStep(
state, 'researcher', '핵심 정보 수집 및 분석 중...',
() => this.resilientExecute(state, this.researcher, 'Researcher', plan, brainContext, signal, onProgress, { context: brainContext, signal, config: { role: 'researcher' }, abstractionLevel: researcherLevel }),
() => this.resilientExecute(state, this.researcher, 'Researcher', plan, brainContext, signal, onProgress, {
...options,
context: brainContext, signal, config: { ...options?.config, role: 'researcher' },
abstractionLevel: researcherLevel
}),
plan, brainContext, signal, onProgress
);
@@ -498,8 +511,9 @@ export class AgentEngine {
const finalReport = await this.executeStep(
state, 'writer', '최종 리포트 작성 및 편집 중...',
() => this.resilientExecute(state, this.writer, 'Writer', research, prompt, signal, onProgress, {
context: brainContext, signal, config: { role: 'writer', allowFallback: true },
priorResults: { plan, writerPrep, previousValidData: state.getResult('finalReport') },
...options,
context: brainContext, signal, config: { role: 'writer', allowFallback: true, ...options?.config },
priorResults: { plan, writerPrep, previousValidData: state.getResult('finalReport'), ...options?.priorResults },
abstractionLevel: writerLevel
}),
research, prompt, signal, onProgress
@@ -631,8 +645,8 @@ export class AgentEngine {
const durationMs = Date.now() - startTime;
// [Reliability Check] 충돌 위험도 추적
const conflictScore = AgentDataValidator.validateHandoff(agentName, result);
state.resilienceMetrics.maxConflictScore = Math.max(state.resilienceMetrics.maxConflictScore, conflictScore);
const validation = AgentDataValidator.validateHandoff(agentName, result);
state.resilienceMetrics.maxConflictScore = Math.max(state.resilienceMetrics.maxConflictScore, validation.conflictRisk);
PerformanceProfiler.logLLMLatency(agentName, durationMs, result.length);
@@ -729,7 +743,8 @@ export class AgentEngine {
private validateResult(data: string, step: string): number {
// Error Recovery Matrix: Permanent 오류 발생을 방지하기 위한 선제적 핸드오프 검증
return AgentDataValidator.validateHandoff(step, data);
const validation = AgentDataValidator.validateHandoff(step, data);
return validation.score;
}
/**
tests/agentEngine.test.ts
+5
View File
@@ -24,6 +24,7 @@ import * as path from 'path';
// ─── Setup ───
const getBaseDir = () => {
if (process.env.ASTRA_TEST_ROOT) return process.env.ASTRA_TEST_ROOT;
// VS Code Mocking 환경 고려
try {
const folders = require('vscode').workspace.workspaceFolders;
@@ -45,6 +46,10 @@ const clearCache = () => {
};
beforeAll(() => {
process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'engine');
if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) {
fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true });
}
clearCache();
});
tests/resilience_stress.test.ts
+183
View File
@@ -0,0 +1,183 @@
/**
* Resilience & Boundary Stress Test Suite (v2.77.3)
*
* 이 테스트는 ConnectAI 엔진이 극한의 환경(인증 실패, 네트워크 차단, 타임아웃 등)에서
* 얼마나 안정적으로 복구되고, 신뢰성 지표(Resilience Metrics)를 정확히 기록하는지 검증합니다.
*/
import {
AgentEngine,
IAgent,
AgentExecuteOptions,
MissionState,
CacheManager
} from '../src/lib/engine';
import * as fs from 'fs';
import * as path from 'path';
import * as vscode from 'vscode';
// --- Helpers ---
const getBaseDir = () => {
if (process.env.ASTRA_TEST_ROOT) return process.env.ASTRA_TEST_ROOT;
try {
const folders = require('vscode').workspace.workspaceFolders;
if (folders && folders.length > 0) return folders[0].uri.fsPath;
} catch (e) {}
return process.cwd();
};
const clearCache = () => {
const baseDir = getBaseDir();
const cacheDir = path.join(baseDir, '.astra', 'cache');
if (fs.existsSync(cacheDir)) {
fs.rmSync(cacheDir, { recursive: true, force: true });
}
const missionDir = path.join(baseDir, '.astra', 'missions');
if (fs.existsSync(missionDir)) {
fs.rmSync(missionDir, { recursive: true, force: true });
}
};
beforeAll(() => {
process.env.ASTRA_TEST_ROOT = path.join(process.cwd(), '.astra', 'tests', 'stress');
if (!fs.existsSync(process.env.ASTRA_TEST_ROOT)) {
fs.mkdirSync(process.env.ASTRA_TEST_ROOT, { recursive: true });
}
clearCache();
});
const noopProgress = () => {};
// --- Mock Agents for Stress Testing ---
/**
* 시나리오 A: 100% 인증 실패만 발생하는 에이전트
*/
class AuthFailureAgent implements IAgent {
public callCount = 0;
async execute(): Promise<string> {
this.callCount++;
throw new Error('AUTH_FAILURE: Invalid API Key or Session Expired');
}
}
/**
* 시나리오 B: 100% 네트워크 차단 발생 (Fallback 유도)
*/
class NetworkBlackoutAgent implements IAgent {
public callCount = 0;
async execute(): Promise<string> {
this.callCount++;
throw new Error('ECONNREFUSED: Connection refused by peer');
}
}
/**
* 시나리오 C: 극심한 지연 발생 (Timeout 유도)
*/
class ExtremeLatencyAgent implements IAgent {
constructor(private delayMs: number = 5000) {}
async execute(): Promise<string> {
await new Promise(r => setTimeout(r, this.delayMs));
return 'Delayed response';
}
}
/**
* 시나리오 D: 데이터 충돌 유발 (High Conflict Score 유도)
*/
class ConflictAgent implements IAgent {
async execute(): Promise<string> {
return "ConnectAI는 전적으로 비동기 엔진입니다. [CONFLICT] 그러나 어떤 문서는 동기적으로 작동한다고 주장합니다.";
}
}
describe('Resilience & Boundary Stress Tests', () => {
beforeEach(() => {
clearCache();
});
test('[Scenario A] 무한 인증 실패 시 안전하게 중단되고 에러를 보고해야 한다', async () => {
const engine = new AgentEngine(
new AuthFailureAgent(),
new AuthFailureAgent(),
new AuthFailureAgent()
);
const missionId = `stress_auth_${Date.now()}`;
try {
await engine.runMission(missionId, 'Auth Test', 'ctx', new AbortController().signal, noopProgress);
fail('Should have thrown an error');
} catch (error: any) {
expect(error.message).toContain('AUTH_FAILURE');
// 재시도가 소진될 때까지 호출되었는지 확인 (Permanent 에러는 즉시 중단)
// ErrorClassifier에 따라 AUTH_FAILURE는 Permanent로 분류됨
}
}, 10000);
test('[Scenario B] 네트워크 블랙아웃 시 Fallback(이전데이터)으로 자동 복구되어야 한다', async () => {
const plannerOutput = 'Plan OK passes validation and meets all length requirements.';
const context = 'Resilience Context';
const fallbackData = 'Emergency Fallback Data from Previous Step';
const engine = new AgentEngine(
new MockSuccessAgent(plannerOutput),
new NetworkBlackoutAgent(), // Researcher (여기서 실패 발생)
new MockSuccessAgent('Final Report')
);
// [Intelligent Resilience] priorResults를 통해 이전 데이터를 주입하여 Fallback 유도
const missionId = `stress_fallback_${Date.now()}`;
const result = await engine.runMission(
missionId,
'Prompt',
context,
new AbortController().signal,
noopProgress,
{ priorResults: { previousValidData: fallbackData }, config: { allowFallback: true } }
);
// 최종 결과물에 Writer의 결과가 포함되어야 함 (Researcher는 fallbackData를 반환했을 것임)
expect(result).toContain('Final Report');
const missionPath = path.join(getBaseDir(), '.astra', 'missions', `${missionId}.json`);
const state = JSON.parse(fs.readFileSync(missionPath, 'utf-8'));
expect(state.resilienceMetrics.fallbacks).toBeGreaterThanOrEqual(1);
console.log(` ✅ Fallback Recovery (priorResults) Verified: ${state.resilienceMetrics.fallbacks} instances`);
}, 15000);
test('[Scenario D] 데이터 충돌 발생 시 Conflict Score가 🚨 High로 기록되어야 한다', async () => {
const validPlan = 'Detailed Execution Plan: 1. Research 2. Analyze 3. Write report with high quality.';
const engine = new AgentEngine(
new MockSuccessAgent(validPlan),
{
execute: async () => {
// 명시적 충돌 및 수치 모순 유발
return "[CONFLICT WARNING] 성능이 200% 증가했습니다. vs 그러나 동시에 50% 감소했습니다. 최적화와 성능 저하가 동시에 발견됨.";
}
},
new MockSuccessAgent('Final report with inconsistencies. This should be long enough to pass validation.')
);
const missionId = `stress_conflict_${Date.now()}`;
const result = await engine.runMission(missionId, 'Conflict Test', 'ctx', new AbortController().signal, noopProgress);
const missionPath = path.join(getBaseDir(), '.astra', 'missions', `${missionId}.json`);
const state = JSON.parse(fs.readFileSync(missionPath, 'utf-8'));
// 수치 모순(25) + 상충 용어(15) + 경고 태그(30) = 70점 예상
expect(state.resilienceMetrics.maxConflictScore).toBeGreaterThan(50);
console.log(` 🚨 High Conflict Detected: Risk Score ${state.resilienceMetrics.maxConflictScore}`);
}, 15000);
});
// Mock Helpers
class MockSuccessAgent implements IAgent {
constructor(private response: string) {}
async execute(): Promise<string> {
return this.response;
}
}