diff --git a/docs/records/ConnectAI/development/2026-05-02_project-claim-policy-enforcement.md b/docs/records/ConnectAI/development/2026-05-02_project-claim-policy-enforcement.md new file mode 100644 index 0000000..610bf19 --- /dev/null +++ b/docs/records/ConnectAI/development/2026-05-02_project-claim-policy-enforcement.md @@ -0,0 +1,35 @@ +# Development Log: Project Claim Policy Enforcement + +## Purpose +Make the main answer obey Second Brain Trace evidence limits, especially when selected notes are General Knowledge or mixed with weak project evidence. + +## User Feedback +Trace classification improved, but the main answer could still say phrases like "the architecture is flexible" or "the technical foundation is stable" without enough project evidence. + +## Implementation Summary +- Added `projectClaimPolicy` to Second Brain Trace: + - `allow` + - `cautious` + - `general-only` +- Added `projectClaimPolicyReason`. +- Changed policy derivation so broad project claims are allowed only when all selected notes can support project claims and grounding is high. +- Added strict guidance for `general-only`. +- Added caution guidance for mixed evidence. +- Added required wording for unsupported technical structure claims: + - "현재 정보만으로는 기술 구조를 판단할 수 없습니다." +- Added tests for General Knowledge-only and mixed-evidence cases. + +## Changed Files +- `src/features/secondBrainTrace.ts` +- `src/utils.ts` +- `src/features/projectChronicle/guardPrompt.ts` +- `tests/secondBrainTrace.test.ts` +- `tests/projectChronicleGuardPrompt.test.ts` + +## Verification +- `./node_modules/.bin/tsc --noEmit` +- `npm run compile` +- `./node_modules/.bin/jest --runInBand` + +## Result +Trace warnings now feed a clearer policy into the answer context, making it harder for the model to turn general architecture notes into unsupported project facts. diff --git a/docs/records/ConnectAI/timeline.md b/docs/records/ConnectAI/timeline.md index ebc5419..c82cb4c 100644 --- a/docs/records/ConnectAI/timeline.md +++ b/docs/records/ConnectAI/timeline.md @@ -16,3 +16,4 @@ - Removed hard-coded local template replies for Second Brain overview and unproductive-response correction. - Added progressive answer format guidance: short conclusion first, brief summary second, detailed answer third. - Added No Evidence, No Project Claim rules and Second Brain source type classification to prevent general notes from being treated as project implementation evidence. +- Added project claim policy enforcement so main answers must treat general-only or mixed evidence as cautious and avoid unsupported technical structure claims. diff --git a/src/features/projectChronicle/guardPrompt.ts b/src/features/projectChronicle/guardPrompt.ts index 1e73771..3d57e4b 100644 --- a/src/features/projectChronicle/guardPrompt.ts +++ b/src/features/projectChronicle/guardPrompt.ts @@ -44,6 +44,8 @@ export function buildProjectChronicleGuardContext(project: ProjectProfile | null '- 2nd Brain general concept notes can explain concepts, but they cannot prove that the current project implements those concepts.', '- For project-related opinions, organize claims as confirmed facts, inferences, general knowledge, and needs verification.', '- If only general/reference notes are available, avoid phrases like "the current architecture has..." or "the project is prepared for..." and use "if implemented" or "needs verification" wording.', + '- If only general/reference notes are available, explicitly say: "현재 정보만으로는 기술 구조를 판단할 수 없습니다."', + '- Without project evidence, never say the architecture is flexible, technically stable, scalable, gateway-based, microservice-ready, layered, or structurally prepared.', '', 'Tone and scope:', '- Be practical and plain-spoken.', diff --git a/src/features/secondBrainTrace.ts b/src/features/secondBrainTrace.ts index c981a85..afd9117 100644 --- a/src/features/secondBrainTrace.ts +++ b/src/features/secondBrainTrace.ts @@ -28,6 +28,8 @@ export interface SecondBrainTrace { searchedCollections: string[]; retrievedDocuments: SecondBrainTraceDocument[]; groundingScore: number; + projectClaimPolicy: 'allow' | 'cautious' | 'general-only'; + projectClaimPolicyReason: string; } export function buildSecondBrainTrace(userQuery: string, brainRoot: string, options: { @@ -47,7 +49,9 @@ export function buildSecondBrainTrace(userQuery: string, brainRoot: string, opti retrievalQuery, searchedCollections: [], retrievedDocuments: [], - groundingScore: 0 + groundingScore: 0, + projectClaimPolicy: 'general-only', + projectClaimPolicyReason: 'No project evidence was selected.' }; if (!shouldUseSecondBrain) return baseTrace; @@ -81,6 +85,10 @@ export function buildSecondBrainTrace(userQuery: string, brainRoot: string, opti })); const retrievedDocuments = [...usedDocs, ...unusedDocs]; const usedCount = retrievedDocuments.filter((doc) => doc.usedInAnswer).length; + const groundingScore = retrievedDocuments.length === 0 + ? 0 + : Number((usedCount / retrievedDocuments.length).toFixed(2)); + const { projectClaimPolicy, projectClaimPolicyReason } = deriveProjectClaimPolicy(retrievedDocuments, groundingScore); return { ...baseTrace, @@ -90,9 +98,9 @@ export function buildSecondBrainTrace(userQuery: string, brainRoot: string, opti : 'Second Brain search ran, but no sufficiently relevant Markdown notes were found.', searchedCollections: inferCollections(retrievedDocuments), retrievedDocuments, - groundingScore: retrievedDocuments.length === 0 - ? 0 - : Number((usedCount / retrievedDocuments.length).toFixed(2)) + groundingScore, + projectClaimPolicy, + projectClaimPolicyReason }; } @@ -138,10 +146,19 @@ export function renderSecondBrainTraceContext(trace: SecondBrainTrace): string { hasProjectEvidence ? 'At least one selected note can support project-specific claims.' : 'No selected note can support project-specific implementation claims.', + `Project claim policy: ${trace.projectClaimPolicy}`, + `Project claim policy reason: ${trace.projectClaimPolicyReason}`, + trace.projectClaimPolicy === 'cautious' + ? 'CAUTION RULE: selected notes include some project evidence but not enough for broad technical structure claims. State only the directly supported facts and mark broader architecture claims as Needs Verification. Use wording such as "현재 정보만으로는 기술 구조를 판단할 수 없습니다" for unsupported technical structure claims.' + : '', selectedAreGeneralOnly - ? 'Selected notes are general/reference material only. Use cautious wording and mark project implementation claims as Needs Verification.' + ? 'STRICT RULE: selected notes are general/reference material only. In the main answer, do not judge the current project architecture as flexible, stable, scalable, separated, gateway-based, microservice-ready, or technically prepared.' + : '', + selectedAreGeneralOnly + ? 'Required wording for technical claims: "현재 정보만으로는 기술 구조를 판단할 수 없습니다", "일반 원칙상으로는...", "실제 확인을 위해서는 소스 코드/설계 문서/라우팅 구조/데이터 흐름 확인이 필요합니다."' : '', 'Grounding rule: score >= 0.8 with project evidence may support project facts; 0.5-0.8 requires cautious wording; <= 0.5 or General Knowledge only means general/inference only.', + 'Forbidden project claims without project evidence: "아키텍처는 유연합니다", "기술적 기반은 안정적입니다", "확장성 측면에서 준비되어 있습니다", "구조적 안정성이 확보되었습니다", "API Gateway 기반으로 라우팅됩니다", "비즈니스 로직과 데이터 접근 계층이 분리되어 있습니다."', 'If these notes influence the answer, mention them in the final reference section.' ].filter(Boolean).join('\n'); } @@ -184,7 +201,9 @@ export function renderSecondBrainTraceMarkdown(trace: SecondBrainTrace, debug: b '## 참고 품질', `- 검색된 노트: ${trace.retrievedDocuments.length}개`, `- 답변 컨텍스트로 선택된 노트: ${usedDocs.length}개`, - `- 답변 근거도: ${trace.groundingScore}` + `- 답변 근거도: ${trace.groundingScore}`, + `- 프로젝트 주장 정책: ${trace.projectClaimPolicy}`, + `- 정책 이유: ${trace.projectClaimPolicyReason}` ]; if (debug) { @@ -208,7 +227,9 @@ export function renderSecondBrainTraceMarkdown(trace: SecondBrainTrace, debug: b usedFor: doc.usedFor, excludedReason: doc.excludedReason })), - groundingScore: trace.groundingScore + groundingScore: trace.groundingScore, + projectClaimPolicy: trace.projectClaimPolicy, + projectClaimPolicyReason: trace.projectClaimPolicyReason }, null, 2), '```' ); @@ -225,6 +246,30 @@ export function renderSecondBrainTraceMarkdown(trace: SecondBrainTrace, debug: b ].join('\n'); } +function deriveProjectClaimPolicy( + docs: SecondBrainTraceDocument[], + groundingScore: number +): Pick { + const selected = docs.filter((doc) => doc.selectedForAnswerContext); + const projectEvidenceCount = selected.filter((doc) => doc.canSupportProjectClaim).length; + if (projectEvidenceCount === 0) { + return { + projectClaimPolicy: 'general-only', + projectClaimPolicyReason: 'Selected notes are General Knowledge or Reference Only, so they cannot support claims about the current project implementation.' + }; + } + if (projectEvidenceCount === selected.length && groundingScore >= 0.8) { + return { + projectClaimPolicy: 'allow', + projectClaimPolicyReason: 'All selected context can support project claims and grounding is high.' + }; + } + return { + projectClaimPolicy: 'cautious', + projectClaimPolicyReason: 'Selected context includes some project evidence, but it is mixed with general/reference material or grounding is not high enough for broad technical claims.' + }; +} + function shouldUseBrain(query: string): boolean { const normalized = query.toLowerCase(); return /(second brain|2nd brain|제2뇌|브레인|brain|기억|기록|문서|노트|내가|우리|프로젝트|결정|adr|chronicle|가드|설계 원칙|mvp|제외|왜|dependency|schema|documentation|drift|integration|overhead|의존성|스키마|문서화)/i.test(normalized); diff --git a/src/utils.ts b/src/utils.ts index 5e80f66..f37d611 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -160,6 +160,7 @@ Core behavior: - No Evidence, No Project Claim: do not state that the current project has a technical structure unless it is supported by user-provided facts, source code, design docs, project docs, or project records. - Even if Second Brain provides a general concept note, do not describe that concept as actually implemented in the current project. General concept notes are not project evidence. - For project opinions, separate claims into confirmed facts, inferences, general knowledge, and items that need verification. +- If available evidence is only general knowledge, never say the project architecture is flexible, technically stable, scalable, gateway-based, microservice-ready, separated into layers, or structurally prepared. Say the technical structure cannot be judged from the current information. Available action tags: diff --git a/tests/projectChronicleGuardPrompt.test.ts b/tests/projectChronicleGuardPrompt.test.ts index e86c9fc..c12718d 100644 --- a/tests/projectChronicleGuardPrompt.test.ts +++ b/tests/projectChronicleGuardPrompt.test.ts @@ -22,6 +22,8 @@ describe('buildProjectChronicleGuardContext', () => { expect(context).toContain('Detailed answer'); expect(context).toContain('No Evidence, No Project Claim'); expect(context).toContain('confirmed facts, inferences, general knowledge, and needs verification'); + expect(context).toContain('현재 정보만으로는 기술 구조를 판단할 수 없습니다'); + expect(context).toContain('technically stable'); expect(context).toContain('Project record target check'); expect(context).toContain('Record path check'); expect(context).toContain('Question reason'); diff --git a/tests/secondBrainTrace.test.ts b/tests/secondBrainTrace.test.ts index d027221..ed0bdb3 100644 --- a/tests/secondBrainTrace.test.ts +++ b/tests/secondBrainTrace.test.ts @@ -121,6 +121,40 @@ describe('Second Brain Trace', () => { expect(apiGateway?.sourceType).toBe('General Knowledge'); expect(apiGateway?.canSupportProjectClaim).toBe(false); expect(apiGateway?.warning).toContain('실제 구현 근거가 아닙니다'); - expect(renderSecondBrainTraceMarkdown(trace, true)).toContain('"canSupportProjectClaim": false'); + const markdown = renderSecondBrainTraceMarkdown(trace, true); + const context = renderSecondBrainTraceContext(trace); + expect(trace.projectClaimPolicy).not.toBe('allow'); + expect(markdown).toContain('"canSupportProjectClaim": false'); + expect(context).toContain('No Evidence, No Project Claim'); + expect(context).toContain('현재 정보만으로는 기술 구조를 판단할 수 없습니다'); + expect(context).toContain('아키텍처는 유연합니다'); + }); + + it('uses general-only policy when selected notes cannot support project claims', () => { + const generalOnlyRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'second-brain-general-only-')); + try { + fs.mkdirSync(path.join(generalOnlyRoot, '02_Architecture_Principles'), { recursive: true }); + fs.writeFileSync( + path.join(generalOnlyRoot, '02_Architecture_Principles', 'API Gateway.md'), + [ + '# API Gateway', + '', + 'General Knowledge: API Gateway can route requests in a microservice architecture.', + 'This is a concept note, not current project evidence.' + ].join('\n'), + 'utf8' + ); + + const trace = buildSecondBrainTrace( + '현재 프로젝트는 API Gateway 라우팅 구조를 갖추고 있어?', + generalOnlyRoot, + { force: true } + ); + + expect(trace.projectClaimPolicy).toBe('general-only'); + expect(renderSecondBrainTraceContext(trace)).toContain('STRICT RULE'); + } finally { + fs.rmSync(generalOnlyRoot, { recursive: true, force: true }); + } }); });