From fcc07e013e161ce8f05932efd67c5157c366d528 Mon Sep 17 00:00:00 2001 From: g1nation Date: Sat, 2 May 2026 18:06:28 +0900 Subject: [PATCH] Feat: Implement Project Claim Output Brake and refine agent reasoning --- .../2026-05-02_project-claim-output-brake.md | 35 +++++++++++++++++ docs/records/ConnectAI/timeline.md | 1 + src/agent.ts | 6 ++- src/features/secondBrainTrace.ts | 39 +++++++++++++++++++ tests/secondBrainTrace.test.ts | 32 +++++++++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 docs/records/ConnectAI/development/2026-05-02_project-claim-output-brake.md diff --git a/docs/records/ConnectAI/development/2026-05-02_project-claim-output-brake.md b/docs/records/ConnectAI/development/2026-05-02_project-claim-output-brake.md new file mode 100644 index 0000000..e0a2941 --- /dev/null +++ b/docs/records/ConnectAI/development/2026-05-02_project-claim-output-brake.md @@ -0,0 +1,35 @@ +# Development Log: Project Claim Output Brake + +## Purpose +Stop unsupported project technical claims in the final answer body, not only in Trace diagnostics. + +## Problem +Even when Trace reported `projectClaimPolicy=general-only`, the model could still write phrases such as: +- "기술적 기반 면에서는 안정적" +- "아키텍처는 유연하나" +- "모듈화된 구조는 향후 기능 추가 시 유연성을 제공합니다" + +Prompt guidance alone was not enough. + +## Implementation Summary +- Added `enforceProjectClaimPolicyInAnswer`. +- When `projectClaimPolicy=general-only`, final answer text is scanned before output. +- Unsupported technical structure claim sentences are removed. +- A safe notice is inserted: + - "현재 정보만으로는 기술 구조를 판단할 수 없습니다." + - "기술적 안정성, 아키텍처 유연성, 모듈화 여부는 소스 코드나 설계 문서 확인이 필요합니다." +- Connected the enforcement function in `AgentExecutor` before final response streaming and history persistence. +- Added regression tests for the exact problematic phrases. + +## Changed Files +- `src/features/secondBrainTrace.ts` +- `src/agent.ts` +- `tests/secondBrainTrace.test.ts` + +## Verification +- `./node_modules/.bin/tsc --noEmit` +- `npm run compile` +- `./node_modules/.bin/jest --runInBand` + +## Result +The system now has a deterministic output brake for unsupported project implementation claims when Trace says only general/reference evidence is available. diff --git a/docs/records/ConnectAI/timeline.md b/docs/records/ConnectAI/timeline.md index c82cb4c..fe92a7e 100644 --- a/docs/records/ConnectAI/timeline.md +++ b/docs/records/ConnectAI/timeline.md @@ -17,3 +17,4 @@ - Added progressive answer format guidance: short conclusion first, brief summary second, detailed answer third. - Added No Evidence, No Project Claim rules and Second Brain source type classification to prevent general notes from being treated as project implementation evidence. - Added project claim policy enforcement so main answers must treat general-only or mixed evidence as cautious and avoid unsupported technical structure claims. +- Added a deterministic output brake that removes unsupported technical structure claims from final answers when Trace policy is `general-only`. diff --git a/src/agent.ts b/src/agent.ts index 6ef6eaf..fd00ad1 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -34,6 +34,7 @@ import { actionQueue } from './core/queue'; import { ConflictResolver } from './core/conflict'; import { buildSecondBrainTrace, + enforceProjectClaimPolicyInAnswer, renderSecondBrainTraceContext, renderSecondBrainTraceMarkdown, SecondBrainTrace @@ -409,7 +410,10 @@ export class AgentExecutor { // 5. Execute Actions const rationale = this.parseRationale(aiResponseText); - const assistantContent = this.sanitizeAssistantContent(aiResponseText); + const assistantContent = enforceProjectClaimPolicyInAnswer( + this.sanitizeAssistantContent(aiResponseText), + secondBrainTrace + ); const traceMarkdown = secondBrainTrace ? renderSecondBrainTraceMarkdown(secondBrainTrace, !!options.secondBrainTraceDebug) : ''; diff --git a/src/features/secondBrainTrace.ts b/src/features/secondBrainTrace.ts index afd9117..be105aa 100644 --- a/src/features/secondBrainTrace.ts +++ b/src/features/secondBrainTrace.ts @@ -246,6 +246,45 @@ export function renderSecondBrainTraceMarkdown(trace: SecondBrainTrace, debug: b ].join('\n'); } +export function enforceProjectClaimPolicyInAnswer(answer: string, trace: SecondBrainTrace | null): string { + if (!trace || trace.projectClaimPolicy !== 'general-only') return answer; + + const forbiddenPatterns = [ + /[^.!?。！？\n]*(?:현재\s*)?(?:개발\s*방향은\s*)?기술적\s*기반(?:\s*면에서는)?\s*(?:안정적|탄탄|준비)[^.!?。！？\n]*(?:[.!?。！？]|$)/gi, + /[^.!?。！？\n]*아키텍처(?:는|가)?\s*(?:유연|안정|확장성|준비|견고)[^.!?。！？\n]*(?:[.!?。！？]|$)/gi, + /[^.!?。！？\n]*모듈화(?:된)?\s*구조[^.!?。！？\n]*(?:[.!?。！？]|$)/gi, + /[^.!?。！？\n]*확장성\s*(?:측면에서)?\s*(?:준비|확보|충분|높)[^.!?。！？\n]*(?:[.!?。！？]|$)/gi, + /[^.!?。！？\n]*구조적\s*안정성[^.!?。！？\n]*(?:[.!?。！？]|$)/gi, + /[^.!?。！？\n]*API\s*Gateway\s*기반[^.!?。！？\n]*(?:[.!?。！？]|$)/gi, + /[^.!?。！？\n]*비즈니스\s*로직[^.!?。！？\n]*데이터\s*접근\s*계층[^.!?。！？\n]*(?:[.!?。！？]|$)/gi + ]; + + let sanitized = answer; + let removed = false; + for (const pattern of forbiddenPatterns) { + sanitized = sanitized.replace(pattern, (match) => { + removed = true; + return match.includes('\n') ? '\n' : ''; + }); + } + + if (!removed) return answer; + + const warning = [ + '> 현재 정보만으로는 기술 구조를 판단할 수 없습니다.', + '> 기술적 안정성, 아키텍처 유연성, 모듈화 여부는 소스 코드나 설계 문서 확인이 필요합니다.' + ].join('\n'); + + return insertAfterFirstBlock(sanitized.trim(), warning); +} + +function insertAfterFirstBlock(answer: string, insertion: string): string { + if (!answer.trim()) return insertion; + const blocks = answer.split(/\n{2,}/); + if (blocks.length <= 1) return `${insertion}\n\n${answer}`; + return [blocks[0], insertion, ...blocks.slice(1)].join('\n\n'); +} + function deriveProjectClaimPolicy( docs: SecondBrainTraceDocument[], groundingScore: number diff --git a/tests/secondBrainTrace.test.ts b/tests/secondBrainTrace.test.ts index ed0bdb3..b2e3784 100644 --- a/tests/secondBrainTrace.test.ts +++ b/tests/secondBrainTrace.test.ts @@ -3,6 +3,7 @@ import * as os from 'os'; import * as path from 'path'; import { buildSecondBrainTrace, + enforceProjectClaimPolicyInAnswer, renderSecondBrainTraceContext, renderSecondBrainTraceMarkdown } from '../src/features/secondBrainTrace'; @@ -157,4 +158,35 @@ describe('Second Brain Trace', () => { fs.rmSync(generalOnlyRoot, { recursive: true, force: true }); } }); + + it('removes unsupported technical structure claims from final answers under general-only policy', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'second-brain-policy-')); + try { + fs.mkdirSync(path.join(root, '02_Architecture_Principles'), { recursive: true }); + fs.writeFileSync( + path.join(root, '02_Architecture_Principles', 'API Gateway.md'), + '# API Gateway\n\nGeneral Knowledge: API Gateway can route requests.', + 'utf8' + ); + const trace = buildSecondBrainTrace('현재 프로젝트는 API Gateway 라우팅 구조를 갖추고 있어?', root, { force: true }); + const answer = [ + '결론은 간단합니다. 현재 개발 방향은 기술적 기반 면에서는 안정적입니다.', + '', + '아키텍처는 유연하나 구매 전환 시나리오를 보완해야 합니다.', + '모듈화된 구조는 향후 기능 추가 시 유연성을 제공합니다.', + '', + '다만 UX 관점에서는 공간 경험과 상품 탐색 연결을 확인해야 합니다.' + ].join('\n'); + + const sanitized = enforceProjectClaimPolicyInAnswer(answer, trace); + + expect(sanitized).toContain('현재 정보만으로는 기술 구조를 판단할 수 없습니다'); + expect(sanitized).not.toContain('기술적 기반 면에서는 안정적'); + expect(sanitized).not.toContain('아키텍처는 유연'); + expect(sanitized).not.toContain('모듈화된 구조'); + expect(sanitized).toContain('UX 관점'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); });