/**
 * ============================================================
 * Brain Index — persistent, mtime-keyed tokenized cache of the Second Brain
 *
 * RAG 검색은 매 질의마다 브레인의 모든 .md 파일을 읽고 토크나이즈해서 TF-IDF 점수를
 * 계산했습니다 — 파일 수가 많아지면 그게 병목입니다.
 *
 * 이 모듈은 `<brainPath>/.astra/brain-index.json` 에 파일별 토큰 배열을 (mtime+size 키로)
 * 저장해 두고, 다음 질의에서는 *변경된 파일만* 다시 읽어 토크나이즈합니다. 나머지는 디스크/메모리
 * 캐시에서 그대로 가져옵니다. 디스크 쓰기는 디바운스되고 실패해도 in-memory 로만 동작합니다.
 * ============================================================
 */

import * as fs from 'fs';
import * as path from 'path';
import { tokenize, countConflictIndicators } from './scoring';
import { detectLessonKind } from './lessonHelpers';
import { splitIntoSections } from './chunker';
import { logInfo } from '../utils';

// v5 adds optional per-file `chunks` (section-level index, Phase 1-가) alongside the
// v4 per-file `embedding`. Older indexes are auto-rebuilt on first load — no migration
// needed because the cache is fully derivable from the brain itself.
const INDEX_VERSION = 5;
const INDEX_DIR = '.astra';
const INDEX_FILE = 'brain-index.json';
/** 인덱스가 이 개수를 넘으면 이번 스캔에서 못 본 항목을 정리합니다 (삭제된 파일 누적 방지). */
const MAX_INDEX_ENTRIES = 12000;
/** 디스크 쓰기 디바운스. */
const WRITE_DEBOUNCE_MS = 1500;

interface IndexEntry {
    mtimeMs: number;
    size: number;
    title: string;          // basename without .md
    relativePath: string;   // relative to brainPath
    tokens: string[];       // tokenize(`${title} ${content}`)
    titleTokens: string[];  // tokenize(title)
    conflictCount: number;  // countConflictIndicators(`${title} ${content}`)
    kind: string;           // '' for an ordinary note, else 'lesson' | 'playbook' | 'qa-finding'
    /**
     * Dense embedding for hybrid retrieval. Populated lazily by a background
     * pass after the file is tokenized — TF-IDF queries don't wait on it.
     * Cleared when mtimeMs/size change because the content moved on.
     */
    embedding?: number[];
    /** Embedding model the vector was produced with — invalidates the vector when the user switches models. */
    embeddingModel?: string;
    /**
     * Section-level chunks (Phase 1-가). 지연 계산 — chunk 모드 검색이 처음 요청할 때
     * `getBrainChunkIndex` 가 채운다. 파일이 바뀌면 (재색인 시 entry 가 새로 만들어져)
     * 자동으로 사라지므로 stale chunk 가 남지 않는다.
     */
    chunks?: ChunkEntry[];
}

interface ChunkEntry {
    heading: string;
    headingPath: string[];
    tokens: string[];        // tokenize(`${title} ${headingPath} ${sectionText}`) — 문서 제목이 모든 청크에 기여
    headingTokens: string[]; // tokenize(`${title} ${headingPath}`)
    charStart: number;
    charEnd: number;
    /** 청크 단위 dense 벡터 (Phase 1-가 후속). 파일 단위보다 정밀. 지연 backfill. */
    embedding?: number[];
    /** 이 벡터를 만든 임베딩 모델 — 모델 변경 시 무효화. */
    embeddingModel?: string;
}

interface PersistedIndex {
    version: number;
    entries: Record<string, IndexEntry>; // keyed by absolute file path
    /** chunks 를 어떤 targetChars 로 만들었는지 — 설정이 바뀌면 chunk 층을 재생성. */
    chunkTargetChars?: number;
}

export interface IndexedBrainDoc {
    filePath: string;
    relativePath: string;
    title: string;
    tokens: string[];
    titleTokens: string[];
    conflictCount: number;
    mtimeMs: number;
    /** '' for an ordinary note; 'lesson' | 'playbook' | 'qa-finding' for an Experience-Memory card. */
    kind: string;
}

/** Flat chunk view returned by `getBrainChunkIndex` — 한 파일이 여러 청크로 펼쳐진다. */
export interface IndexedChunk {
    filePath: string;
    relativePath: string;
    title: string;
    /** 파일 내 청크 순번 (0-based). */
    chunkIndex: number;
    heading: string;
    headingPath: string[];
    tokens: string[];
    headingTokens: string[];
    charStart: number;
    charEnd: number;
    mtimeMs: number;
    kind: string;
}

interface BrainState {
    index: PersistedIndex;
    dirty: boolean;
    diskPath: string | null;   // null if we can't determine a writable path
    writeTimer?: ReturnType<typeof setTimeout>;
}

const _states = new Map<string, BrainState>();

function indexFileFor(brainPath: string): string {
    return path.join(brainPath, INDEX_DIR, INDEX_FILE);
}

function loadState(brainPath: string): BrainState {
    const existing = _states.get(brainPath);
    if (existing) return existing;

    let index: PersistedIndex = { version: INDEX_VERSION, entries: {} };
    let diskPath: string | null = null;
    try {
        diskPath = indexFileFor(brainPath);
        if (fs.existsSync(diskPath)) {
            const raw = JSON.parse(fs.readFileSync(diskPath, 'utf8'));
            if (raw && raw.version === INDEX_VERSION && raw.entries && typeof raw.entries === 'object') {
                index = raw as PersistedIndex;
            } else {
                logInfo('Brain index is stale/unrecognized — rebuilding.', { brainPath });
            }
        }
    } catch (e: any) {
        logInfo('Brain index load failed — starting fresh.', { brainPath, error: e?.message || String(e) });
        index = { version: INDEX_VERSION, entries: {} };
    }
    const st: BrainState = { index, dirty: false, diskPath };
    _states.set(brainPath, st);
    return st;
}

function scheduleWrite(st: BrainState, brainPath: string): void {
    if (!st.dirty || !st.diskPath || st.writeTimer) return;
    const timer = setTimeout(() => {
        st.writeTimer = undefined;
        if (!st.dirty || !st.diskPath) return;
        try {
            const dir = path.dirname(st.diskPath);
            fs.mkdirSync(dir, { recursive: true });
            // One-time .gitignore so the cache dir never gets committed into a Second Brain git repo.
            const gi = path.join(dir, '.gitignore');
            if (!fs.existsSync(gi)) {
                try { fs.writeFileSync(gi, '*\n', 'utf8'); } catch { /* non-fatal */ }
            }
            const tmp = `${st.diskPath}.tmp`;
            fs.writeFileSync(tmp, JSON.stringify(st.index), 'utf8');
            fs.renameSync(tmp, st.diskPath);
            st.dirty = false;
        } catch (e: any) {
            logInfo('Brain index write failed (continuing in-memory only).', { brainPath, error: e?.message || String(e) });
        }
    }, WRITE_DEBOUNCE_MS);
    if (typeof (timer as any).unref === 'function') (timer as any).unref();
    st.writeTimer = timer;
}

/**
 * Returns tokenized representations for `files` (absolute brain-file paths, already
 * scoped/filtered by the caller). Unchanged files are served from the index; changed/new
 * files are read & tokenized and the index is updated (debounced disk write).
 *
 * Safe to call with an empty/invalid `brainPath` or empty list — returns [].
 */
export function getBrainTokenIndex(brainPath: string, files: string[]): IndexedBrainDoc[] {
    if (!brainPath || !Array.isArray(files) || files.length === 0) return [];
    const st = loadState(brainPath);
    const out: IndexedBrainDoc[] = [];
    const seen = new Set<string>();
    let reindexed = 0;

    for (const file of files) {
        seen.add(file);
        let stat: fs.Stats;
        try {
            stat = fs.statSync(file);
        } catch {
            continue; // listed but gone now — skip silently
        }
        const cached = st.index.entries[file];
        if (cached
            && cached.mtimeMs === stat.mtimeMs
            && cached.size === stat.size
            && Array.isArray(cached.tokens)
            && Array.isArray(cached.titleTokens)) {
            out.push({
                filePath: file,
                relativePath: cached.relativePath,
                title: cached.title,
                tokens: cached.tokens,
                titleTokens: cached.titleTokens,
                conflictCount: cached.conflictCount || 0,
                mtimeMs: cached.mtimeMs,
                kind: cached.kind || '',
            });
            continue;
        }
        // (Re)index this file.
        let content = '';
        try {
            content = fs.readFileSync(file, 'utf8');
        } catch {
            continue;
        }
        const relativePath = path.relative(brainPath, file);
        const title = path.basename(file, '.md');
        const combined = `${title} ${content}`;
        const entry: IndexEntry = {
            mtimeMs: stat.mtimeMs,
            size: stat.size,
            title,
            relativePath,
            tokens: tokenize(combined),
            titleTokens: tokenize(title),
            conflictCount: countConflictIndicators(combined),
            kind: detectLessonKind(relativePath, content),
        };
        st.index.entries[file] = entry;
        st.dirty = true;
        reindexed++;
        out.push({
            filePath: file,
            relativePath,
            title,
            tokens: entry.tokens,
            titleTokens: entry.titleTokens,
            conflictCount: entry.conflictCount,
            mtimeMs: entry.mtimeMs,
            kind: entry.kind,
        });
    }

    // Prune stale entries. We only prune when this looked like a (near-)full scan — i.e. we saw
    // most of the index — so an agent-scoped query doesn't evict cache for out-of-scope files.
    // (Falls back to a hard prune if the index has grown beyond MAX_INDEX_ENTRIES.)
    const entryKeys = Object.keys(st.index.entries);
    const looksFullScan = seen.size >= entryKeys.length * 0.8;
    if (looksFullScan || entryKeys.length > MAX_INDEX_ENTRIES) {
        for (const key of entryKeys) {
            if (!seen.has(key)) {
                delete st.index.entries[key];
                st.dirty = true;
            }
        }
    }

    if (reindexed > 0) {
        logInfo('Brain index updated.', { brainPath, files: files.length, reindexed, totalEntries: Object.keys(st.index.entries).length });
    }
    if (st.dirty) scheduleWrite(st, brainPath);
    return out;
}

/**
 * lesson/playbook/qa-finding 카드는 통째로 한 청크 — 섹션 분할이 essence 추출을
 * 깨뜨리지 않도록. 일반 노트는 `splitIntoSections` 로 섹션 청크화. 문서 제목을 모든
 * 청크 토큰에 prepend 해 제목 매치 강도(파일 모드의 titleTokens)를 보존한다.
 */
function buildChunkEntries(entry: IndexEntry, content: string, targetChars: number): ChunkEntry[] {
    const whole = (): ChunkEntry[] => [{
        heading: entry.title,
        headingPath: [entry.title],
        tokens: entry.tokens,
        headingTokens: entry.titleTokens,
        charStart: 0,
        charEnd: content.length,
    }];
    if (entry.kind && entry.kind !== '') return whole();

    const sections = splitIntoSections(content, {
        targetChars,
        minChars: Math.min(200, Math.floor(targetChars / 4)),
        maxChars: targetChars * 2,
    });
    if (sections.length === 0) return whole();

    return sections.map((s) => {
        const headingText = [entry.title, ...s.headingPath].join(' ');
        return {
            heading: s.heading || entry.title,
            headingPath: s.headingPath.length ? s.headingPath : [entry.title],
            tokens: tokenize(`${headingText} ${s.text}`),
            headingTokens: tokenize(headingText),
            charStart: s.charStart,
            charEnd: s.charEnd,
        };
    });
}

/**
 * Section-level chunk view (Phase 1-가). 먼저 `getBrainTokenIndex` 로 파일 entry 를
 * 최신화한 뒤, 각 파일의 chunk 층을 (없으면) 계산·캐시해서 flat 하게 펼쳐 반환한다.
 * `targetChars` 가 직전 빌드값과 다르면 전체 chunk 캐시를 버리고 재생성한다.
 * Steady-state(변경 없음 + 같은 target)에서는 디스크/CPU 작업 0.
 */
export function getBrainChunkIndex(brainPath: string, files: string[], targetChars: number): IndexedChunk[] {
    if (!brainPath || !Array.isArray(files) || files.length === 0) return [];
    // 1) 파일 entry 최신화 (토큰/메타/prune 까지 여기서 처리).
    getBrainTokenIndex(brainPath, files);
    const st = loadState(brainPath);

    // 2) targetChars 변경 시 chunk 층 전체 무효화.
    if (st.index.chunkTargetChars !== targetChars) {
        for (const key of Object.keys(st.index.entries)) {
            const e = st.index.entries[key];
            if (e) e.chunks = undefined;
        }
        st.index.chunkTargetChars = targetChars;
        st.dirty = true;
    }

    const out: IndexedChunk[] = [];
    let built = 0;
    for (const file of files) {
        const entry = st.index.entries[file];
        if (!entry) continue;
        if (!entry.chunks) {
            let content = '';
            try { content = fs.readFileSync(file, 'utf8'); } catch { continue; }
            entry.chunks = buildChunkEntries(entry, content, targetChars);
            st.dirty = true;
            built++;
        }
        for (let ci = 0; ci < entry.chunks.length; ci++) {
            const ch = entry.chunks[ci];
            out.push({
                filePath: file,
                relativePath: entry.relativePath,
                title: entry.title,
                chunkIndex: ci,
                heading: ch.heading,
                headingPath: ch.headingPath,
                tokens: ch.tokens,
                headingTokens: ch.headingTokens,
                charStart: ch.charStart,
                charEnd: ch.charEnd,
                mtimeMs: entry.mtimeMs,
                kind: entry.kind || '',
            });
        }
    }
    if (built > 0) logInfo('Brain chunk index built.', { brainPath, files: files.length, filesChunked: built, totalChunks: out.length, targetChars });
    if (st.dirty) scheduleWrite(st, brainPath);
    return out;
}

/**
 * Pull (filePath, embedding) for every file in `filePaths` that has a current
 * cached vector under `model`. Caller uses this to rank top TF-IDF candidates
 * by cosine similarity. Files missing an embedding are silently omitted.
 */
export function getBrainEmbeddings(brainPath: string, filePaths: string[], model: string): Map<string, number[]> {
    const out = new Map<string, number[]>();
    if (!brainPath || !model.trim() || !Array.isArray(filePaths) || filePaths.length === 0) return out;
    const embKey = embeddingKey(model);
    const st = _states.get(brainPath);
    if (!st) return out;
    for (const fp of filePaths) {
        const entry = st.index.entries[fp];
        if (!entry?.embedding || entry.embeddingModel !== embKey) continue;
        if (!Array.isArray(entry.embedding) || entry.embedding.length === 0) continue;
        out.set(fp, entry.embedding);
    }
    return out;
}

/**
 * Background fill: for each file under `filePaths`, embed its content with
 * `embedFn` if no current vector exists for `model`. Calls `embedFn` in
 * caller-controlled batches (caller can chunk filePaths as wanted), and saves
 * the disk index. Designed to be fire-and-forget — failures are logged and
 * swallowed.
 *
 * Returns the count of newly embedded files (0 when everything was cached
 * already or the model is empty).
 */
/**
 * 벡터를 소수 4자리로 양자화 — JSON 캐시 크기를 절반 이하로 줄인다.
 * full-precision float64 직렬화는 원소당 ~20자, 4자리는 ~7자. 코사인 유사도는
 * 1e-4 노이즈에 둔감하므로 (정규화된 임베딩 공간에서 순위 변동 없음) 손실 무해.
 */
function quantizeVector(v: number[]): number[] {
    const out = new Array<number>(v.length);
    for (let i = 0; i < v.length; i++) out[i] = Math.round(v[i] * 10000) / 10000;
    return out;
}

/**
 * 임베딩 캐시 키 — 모델명 + 파이프라인 리비전. 입력 구성이 바뀌면 리비전을 올려
 * 기존 벡터를 자동 무효화(재임베딩)한다.
 *   r2: 토큰 죽(tokens.join) → *원문 슬라이스* + task prefix (측정상 r1 은 검색 품질 저하).
 */
const EMBED_REV = 'r2';
function embeddingKey(model: string): string { return `${model}@${EMBED_REV}`; }

export async function backfillBrainEmbeddings(
    brainPath: string,
    filePaths: string[],
    model: string,
    embedFn: (texts: string[]) => Promise<number[][]>,
): Promise<number> {
    if (!brainPath || !model.trim() || !Array.isArray(filePaths) || filePaths.length === 0) return 0;
    const st = _states.get(brainPath);
    if (!st) return 0;
    const embKey = embeddingKey(model);
    const stale: string[] = [];
    for (const fp of filePaths) {
        const entry = st.index.entries[fp];
        if (!entry) continue;
        if (entry.embedding && entry.embeddingModel === embKey) continue;
        stale.push(fp);
    }
    if (stale.length === 0) return 0;
    // 원문을 그대로 임베딩한다 (제목 prepend). 토큰 재조합(tokens.join)은 불용어
    // 제거·소문자화로 문장 구조가 사라져 문장 임베딩 모델 품질을 크게 떨어뜨린다
    // (golden set 측정으로 확인 — EMBED_REV 참고).
    const texts: string[] = [];
    const keys: string[] = [];
    for (const fp of stale) {
        const entry = st.index.entries[fp];
        if (!entry) continue;
        let text = '';
        try { text = `${entry.title}\n${fs.readFileSync(fp, 'utf8')}`; } catch { continue; }
        if (!text.trim()) continue;
        texts.push(text);
        keys.push(fp);
    }
    if (texts.length === 0) return 0;
    try {
        const vectors = await embedFn(texts);
        for (let i = 0; i < vectors.length && i < keys.length; i++) {
            const v = vectors[i];
            if (!Array.isArray(v) || v.length === 0) continue;
            const entry = st.index.entries[keys[i]];
            if (!entry) continue;
            entry.embedding = quantizeVector(v);
            entry.embeddingModel = embKey;
            st.dirty = true;
        }
        if (st.dirty) {
            logInfo('Brain embeddings backfilled.', { brainPath, model, embedded: vectors.length });
            scheduleWrite(st, brainPath);
        }
        return vectors.length;
    } catch (e: any) {
        logInfo('Brain embedding backfill failed (TF-IDF still works).', { brainPath, model, error: e?.message ?? String(e) });
        return 0;
    }
}

/**
 * 청크 단위 임베딩 조회. `${filePath}#${chunkIndex}` → vector. 모델 불일치/미존재 청크는 생략.
 * searchBrainChunks 가 dense blend 에 사용 (파일 단위 공유보다 정밀).
 */
export function getBrainChunkEmbeddings(brainPath: string, model: string): Map<string, number[]> {
    const out = new Map<string, number[]>();
    if (!brainPath || !model.trim()) return out;
    const embKey = embeddingKey(model);
    const st = _states.get(brainPath);
    if (!st) return out;
    for (const [fp, entry] of Object.entries(st.index.entries)) {
        if (!entry?.chunks) continue;
        for (let ci = 0; ci < entry.chunks.length; ci++) {
            const ch = entry.chunks[ci];
            if (!ch.embedding || ch.embeddingModel !== embKey) continue;
            if (!Array.isArray(ch.embedding) || ch.embedding.length === 0) continue;
            out.set(`${fp}#${ci}`, ch.embedding);
        }
    }
    return out;
}

/**
 * Background fill — 주어진 `files` 의 청크 중 현재 모델 벡터가 없는 것만 임베딩한다.
 * 벡터가 필요한 파일만 1회 재read 해 원문 슬라이스를 임베딩한다.
 * Fire-and-forget 용. 새로 임베딩한 청크 수를 반환.
 */
export async function backfillBrainChunkEmbeddings(
    brainPath: string,
    files: string[],
    model: string,
    embedFn: (texts: string[]) => Promise<number[][]>,
    targetChars: number,
): Promise<number> {
    if (!brainPath || !model.trim() || !Array.isArray(files) || files.length === 0) return 0;
    // 청크 층 보장 (없으면 생성).
    getBrainChunkIndex(brainPath, files, targetChars);
    const st = _states.get(brainPath);
    if (!st) return 0;

    const embKey = embeddingKey(model);
    const texts: string[] = [];
    const refs: Array<{ fp: string; ci: number }> = [];
    for (const fp of files) {
        const entry = st.index.entries[fp];
        if (!entry?.chunks) continue;
        const needs = entry.chunks.some(ch => !(ch.embedding && ch.embeddingModel === embKey));
        if (!needs) continue;
        // 원문 슬라이스를 임베딩 (charStart/charEnd) — 토큰 재조합은 문장 구조가
        // 사라져 임베딩 품질 저하 (EMBED_REV 주석 참고). 제목·헤딩 경로를 prepend 해
        // 문서 맥락을 보존한다.
        let content = '';
        try { content = fs.readFileSync(fp, 'utf8'); } catch { continue; }
        for (let ci = 0; ci < entry.chunks.length; ci++) {
            const ch = entry.chunks[ci];
            if (ch.embedding && ch.embeddingModel === embKey) continue;
            const body = content.slice(ch.charStart, ch.charEnd).trim();
            const headingLine = [entry.title, ...ch.headingPath.filter(h => h !== entry.title)].join(' > ');
            // 본문 없는 헤딩-only 섹션도 헤딩 텍스트로 임베딩 — 벡터 없는 청크를
            // 남기면 hybrid 랭킹에서 스코어 스케일이 갈라진다.
            const text = body ? `${headingLine}\n${body}` : headingLine;
            if (!text.trim()) continue;
            texts.push(text);
            refs.push({ fp, ci });
        }
    }
    if (texts.length === 0) return 0;
    try {
        const vectors = await embedFn(texts);
        let n = 0;
        for (let i = 0; i < vectors.length && i < refs.length; i++) {
            const v = vectors[i];
            if (!Array.isArray(v) || v.length === 0) continue;
            const entry = st.index.entries[refs[i].fp];
            const ch = entry?.chunks?.[refs[i].ci];
            if (!ch) continue;
            ch.embedding = quantizeVector(v);
            ch.embeddingModel = embKey;
            st.dirty = true;
            n++;
        }
        if (n > 0) {
            logInfo('Brain chunk embeddings backfilled.', { brainPath, model, embedded: n });
            scheduleWrite(st, brainPath);
        }
        return n;
    } catch (e: any) {
        logInfo('Brain chunk embedding backfill failed (TF-IDF still works).', { brainPath, model, error: e?.message ?? String(e) });
        return 0;
    }
}

/** Drop the in-memory index (and pending write) for one brain, or all brains. The disk file is left as-is. */
export function clearBrainTokenIndex(brainPath?: string): void {
    if (brainPath === undefined) {
        for (const st of _states.values()) {
            if (st.writeTimer) { clearTimeout(st.writeTimer); st.writeTimer = undefined; }
        }
        _states.clear();
        return;
    }
    const st = _states.get(brainPath);
    if (st?.writeTimer) clearTimeout(st.writeTimer);
    _states.delete(brainPath);
}
