"""
P-Reinforce Phase 1 — Template Normalizer (DRY-RUN by default)
==============================================================
Reads a wiki .md file and emits a normalized version that conforms to the
current P-Reinforce template (templates/wiki_document.md).

What it does mechanically (NO LLM calls):
    1. Cleans frontmatter:
       - Strips [[wiki-link]] decoration from id/category/tags
       - Adds missing fields: canonical_id, aliases, status,
         source_trust_level, raw_sources, duplicate_of, tech_stack (if tech)
       - Re-derives source_trust_level from confidence_score if missing
       - Preserves the original id under `legacy_id` and `aliases`
    2. Renames legacy section headers to current template:
       - "Brief Summary"            -> "📌 한 줄 통찰 (The Karpathy Summary)"
       - "Core Content"             -> "📖 구조화된 지식 (Synthesized Content)"
       - "Trade-offs & Caveats"     -> merged into "⚠️ 모순 및 업데이트"
       - "Knowledge Connections"    -> "🔗 지식 연결 (Graph)"
    3. Adds missing required sections as scaffold (with TODO markers)
       so LLM can fill them later. Tech docs get Code Patterns / Decision
       Criteria / Anti-Patterns scaffolds.
    4. For stubs (<200 body chars), inserts a `🤖 [AI 추론 보강 필요]`
       block — does NOT generate content (that step is interactive).

Usage:
    python p_reinforce_normalize.py <relative_path>            # dry-run, prints diff
    python p_reinforce_normalize.py <relative_path> --out PATH # write to PATH
    python p_reinforce_normalize.py --batch <input_listing>    # multi-file dry-run

Default mode is DRY-RUN. No source files are modified.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import unicodedata
from datetime import date
from pathlib import Path

ROOT = Path(r"E:/Wiki/2nd")
TOPICS = ROOT / "10_Wiki" / "Topics"

FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?", re.DOTALL)
WIKI_LINK_RE = re.compile(r"\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]")
H1_RE = re.compile(r"^# +(.+)$", re.MULTILINE)
HEADING_RE = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
CODE_FENCE_RE = re.compile(r"```([a-zA-Z0-9_-]*)\n", re.MULTILINE)

# Header rename map: matched against the heading text only (after stripping
# leading emoji/punctuation). Keys are normalized lowercase.
HEADER_RENAMES = {
    "brief summary": "📌 한 줄 통찰 (The Karpathy Summary)",
    "the karpathy summary": "📌 한 줄 통찰 (The Karpathy Summary)",
    "한 줄 통찰": "📌 한 줄 통찰 (The Karpathy Summary)",
    "한 줄 통찰 (the karpathy summary)": "📌 한 줄 통찰 (The Karpathy Summary)",
    "core content": "📖 구조화된 지식 (Synthesized Content)",
    "synthesized content": "📖 구조화된 지식 (Synthesized Content)",
    "구조화된 지식": "📖 구조화된 지식 (Synthesized Content)",
    "구조화된 지식 (synthesized content)": "📖 구조화된 지식 (Synthesized Content)",
    "trade-offs & caveats": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "tradeoffs & caveats": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "trade-offs and caveats": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "contradictions & rl update": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "contradictions & updates": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "rl update": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "모순 및 업데이트": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "모순 및 업데이트 (contradictions & updates)": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "모순 및 업데이트 (contradictions & rl update)": "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "knowledge connections": "🔗 지식 연결 (Graph)",
    "graph": "🔗 지식 연결 (Graph)",
    "지식 연결": "🔗 지식 연결 (Graph)",
    "지식 연결 (graph)": "🔗 지식 연결 (Graph)",
    "code patterns": "💻 코드 패턴 (Code Patterns)",
    "코드 패턴": "💻 코드 패턴 (Code Patterns)",
    "decision criteria": "🤔 의사결정 기준 (Decision Criteria)",
    "의사결정 기준": "🤔 의사결정 기준 (Decision Criteria)",
    "anti-patterns": "❌ 안티패턴 (Anti-Patterns)",
    "antipatterns": "❌ 안티패턴 (Anti-Patterns)",
    "안티패턴": "❌ 안티패턴 (Anti-Patterns)",
    "how to use this knowledge": "🤖 LLM 활용 힌트 (How to Use This Knowledge)",
    "llm 활용 힌트": "🤖 LLM 활용 힌트 (How to Use This Knowledge)",
    "validation": "🧪 검증 상태 (Validation)",
    "검증 상태": "🧪 검증 상태 (Validation)",
    "duplicate check": "🧬 중복 검사 (Duplicate Check)",
    "중복 검사": "🧬 중복 검사 (Duplicate Check)",
    "changelog": "🕓 변경 이력 (Changelog)",
    "변경 이력": "🕓 변경 이력 (Changelog)",
}

EMOJI_AT_START_RE = re.compile(r"^[^\w가-힣]*", re.UNICODE)

# folder -> category mapping (best-effort; not authoritative)
FOLDER_CATEGORY_HINTS = {
    "AI": "10_Wiki/Topics",
    "AI_and_ML": "10_Wiki/Topics",
    "Architecture": "10_Wiki/Topics",
    "Backend": "10_Wiki/Topics",
    "Frontend": "10_Wiki/Topics",
    "Frontend_Mastery": "10_Wiki/Topics",
    "DevOps_and_Security": "10_Wiki/Topics",
    "Computer_Science_and_Theory": "10_Wiki/Topics",
    "Programming & Language": "10_Wiki/Topics",
    "Programming & Tools": "10_Wiki/Topics",
    "Programming & Web": "10_Wiki/Topics",
    "Programming & Formal Methods": "10_Wiki/Topics",
    "Visual_Effects": "10_Wiki/Topics_Art",
    "Graphics & Performance": "10_Wiki/Topics_Art",
    "UI_UX_Assets": "10_Wiki/Topics_Art",
    "Design & Experience": "10_Wiki/Topics_Art",
    "Game Design": "10_Wiki/Topics_GD",
    "Game_Design": "10_Wiki/Topics_GD",
    "Level_Design": "10_Wiki/Topics_GD",
    "Balancing": "10_Wiki/Topics_GD",
    "Core_Systems": "10_Wiki/Topics_GD",
    "Skybound": "10_Wiki/Topics_GD",
    "Storytelling": "10_Wiki/Topics_GD",
    "Economics": "10_Wiki/Topics_Biz",
    "Economy": "10_Wiki/Topics_Biz",
    "Economics & Algorithms": "10_Wiki/Topics_Biz",
    "Business_Strategy": "10_Wiki/Topics_Biz",
    "Market_Research": "10_Wiki/Topics_Biz",
    "Partnerships": "10_Wiki/Topics_Biz",
    "Content_Strategy": "10_Wiki/Topics_Blog",
    "Post_Drafts": "10_Wiki/Topics_Blog",
    "External_Media": "10_Wiki/Topics_Blog",
}

TECH_KEYWORDS = {
    "architecture", "algorithm", "programming", "code", "frontend", "backend",
    "compiler", "interpreter", "runtime", "framework", "api", "rest", "graphql",
    "kubernetes", "docker", "kafka", "fastapi", "react", "vue", "svelte", "next",
    "typescript", "python", "rust", "javascript", "go ", "java ", "c++", "swift",
    "database", "sql", "postgres", "redis", "mongo", "cache", "distributed",
}


def parse_frontmatter(text: str) -> tuple[dict, str, str]:
    """Returns (fm_dict, body, fm_raw). Same forgiving parser as indexer."""
    m = FRONTMATTER_RE.match(text)
    if not m:
        return {}, text, ""
    raw = m.group(1)
    body = text[m.end():]
    fm: dict = {}
    current_key: str | None = None
    for line in raw.splitlines():
        if not line.strip() or line.lstrip().startswith("#"):
            continue
        if line.startswith((" ", "\t")) and current_key:
            existing = fm.get(current_key)
            fm[current_key] = (str(existing) + " " + line.strip()).strip() if existing else line.strip()
            continue
        if ":" not in line:
            continue
        key, _, val = line.partition(":")
        key = key.strip()
        val = val.strip()
        if val.startswith("[") and val.endswith("]"):
            inner = val[1:-1].strip()
            items = []
            for it in re.split(r",(?![^\[]*\])", inner):
                it = it.strip().strip("'\"")
                wm = WIKI_LINK_RE.fullmatch(it)
                if wm:
                    it = wm.group(1)
                if it:
                    items.append(it)
            fm[key] = items
        else:
            # strip [[wiki-link|alias]] decoration if scalar
            wm = WIKI_LINK_RE.fullmatch(val.strip("'\"")) if val else None
            if wm:
                val = wm.group(1)
            fm[key] = val.strip("'\"")
        current_key = key
    return fm, body, raw


def detect_tech(folder: str, tags: list[str], body: str) -> bool:
    haystack = (folder + " " + " ".join(tags) + " " + body[:2000]).lower()
    if CODE_FENCE_RE.search(body):
        return True
    return any(k in haystack for k in TECH_KEYWORDS)


def trust_from_confidence(conf_str: str | None) -> str:
    """Per-user policy (2026-05-08): LLM-augmented entries are graded A
    (with `inferred_by` metadata for traceability), not C as the default
    P-Reinforce skill suggests. The mapping below is therefore biased upward
    relative to the skill spec.
    """
    if not conf_str:
        return "A"  # user policy: trust the model
    try:
        c = float(conf_str)
    except (TypeError, ValueError):
        return "A"
    if c >= 0.95:
        return "A"
    if c >= 0.80:
        return "A"
    if c >= 0.65:
        return "B"
    return "C"


# Markers in body text that indicate a redirect even without `redirect_to`
# in the frontmatter (e.g. older P-Reinforce passes left text-only redirects).
TEXT_REDIRECT_PATTERNS = [
    re.compile(r"\*?Redirected to:\s*\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]\*?", re.IGNORECASE),
    re.compile(r"\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]\s*(?:으|로|로)?\s*통합되었습니다"),
    re.compile(r"통합되었습니다.*?\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]"),
    re.compile(r"이 문서는.*?\[\[([^\]|]+?)(?:\|[^\]]+)?\]\].*?로 통합"),
]


def detect_text_redirect(body: str) -> str | None:
    """Look for "Redirected to: [[X]]" or "[[X]]로 통합되었습니다" in body.
    Returns the canonical target name if found, else None."""
    body_top = body[:1500]  # only check the top of the doc
    for pat in TEXT_REDIRECT_PATTERNS:
        m = pat.search(body_top)
        if m:
            return m.group(1).strip()
    return None


def slugify_id(filename: str, today: str | None = None) -> str:
    today = today or date.today().isoformat().replace("-", "")[:8]
    slug = unicodedata.normalize("NFKC", filename).lower()
    slug = re.sub(r"[^a-z0-9가-힣]+", "-", slug).strip("-")
    if not slug:
        slug = "doc"
    return f"wiki-{today[:4]}-{today[4:8]}-{slug[:32]}"


def quote_yaml(s: str) -> str:
    if s is None:
        return '""'
    s = str(s)
    if any(ch in s for ch in [':', '#', '[', ']', '{', '}', ',', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`']) or "\n" in s:
        return '"' + s.replace('"', '\\"') + '"'
    return s


def build_frontmatter(fm: dict, file_path: Path, body: str) -> dict:
    """Compute the new normalized frontmatter from old fm + path heuristics."""
    folder = file_path.parent.name
    filename = file_path.stem
    new_fm: dict = {}

    # Detect text-only redirect (no redirect_to field but body says so)
    text_redirect = detect_text_redirect(body) if not fm.get("redirect_to") else None

    # ID — preserve legacy id if it already looks like our slug
    legacy_id = fm.get("id")
    legacy_id_str = str(legacy_id) if legacy_id else ""
    if legacy_id_str.startswith("wiki-") and not WIKI_LINK_RE.search(legacy_id_str):
        new_id = legacy_id_str
    else:
        new_id = slugify_id(filename)
    new_fm["id"] = new_id

    # title
    title = filename.replace("-", " ").replace("_", " ").strip()
    new_fm["title"] = title

    # category — strip emoji-decorated old categories, normalize wiki-link form
    cat = fm.get("category")
    if cat in (None, "", "Unified") or (isinstance(cat, str) and cat.startswith("[[")):
        cat = FOLDER_CATEGORY_HINTS.get(folder, "10_Wiki/Topics")
    elif isinstance(cat, str) and "💡" in cat:
        # "10_Wiki/💡 Topics/AI" -> hinted category
        cat = FOLDER_CATEGORY_HINTS.get(folder, "10_Wiki/Topics")
    new_fm["category"] = cat

    # If this file is a redirect, override status and add redirect_to.
    if fm.get("redirect_to") or text_redirect:
        target = fm.get("redirect_to") or text_redirect
        new_fm["redirect_to"] = target
        new_fm["status"] = "merged"
        new_fm["canonical_id"] = fm.get("canonical_id") or target  # best-effort
    else:
        # status: draft for stubs, needs_review otherwise; verified only if pre-tagged
        existing_status = fm.get("status")
        if existing_status in ("verified", "merged", "deprecated"):
            new_fm["status"] = existing_status
        else:
            new_fm["status"] = "draft" if len(body.strip()) < 200 else "needs_review"
        new_fm["canonical_id"] = fm.get("canonical_id") or "self"

    # aliases — preserve legacy id and any prior aliases
    aliases = fm.get("aliases") or []
    if isinstance(aliases, str):
        aliases = [aliases]
    if legacy_id and isinstance(legacy_id, str):
        legacy_id_clean = WIKI_LINK_RE.sub(r"\1", legacy_id)
        if legacy_id_clean and legacy_id_clean not in aliases:
            aliases = [legacy_id_clean] + aliases
    new_fm["aliases"] = aliases

    # duplicate_of
    new_fm["duplicate_of"] = fm.get("duplicate_of") or "none"

    # source_trust_level
    new_fm["source_trust_level"] = fm.get("source_trust_level") or trust_from_confidence(fm.get("confidence_score"))

    # confidence_score — preserve if present, else policy default 0.92
    cs = fm.get("confidence_score")
    if cs:
        try:
            new_fm["confidence_score"] = float(cs)
        except (TypeError, ValueError):
            new_fm["confidence_score"] = 0.92
    else:
        new_fm["confidence_score"] = 0.92

    # tags — strip wiki link decoration
    tags = fm.get("tags") or []
    if isinstance(tags, str):
        tags = [tags]
    cleaned_tags = []
    for t in tags:
        if isinstance(t, str):
            t = WIKI_LINK_RE.sub(r"\1", t).strip("[]'\" ")
        if t:
            cleaned_tags.append(t)
    new_fm["tags"] = cleaned_tags or ["uncategorized"]

    # raw_sources
    rs = fm.get("raw_sources") or []
    if isinstance(rs, str):
        rs = [rs]
    new_fm["raw_sources"] = rs

    # last_reinforced
    new_fm["last_reinforced"] = fm.get("last_reinforced") or date.today().isoformat()

    # github_commit
    new_fm["github_commit"] = fm.get("github_commit") or "pending"

    # inferred_by — traceability for LLM-augmented entries (per user policy)
    if fm.get("inferred_by"):
        new_fm["inferred_by"] = fm["inferred_by"]
    elif new_fm["source_trust_level"] == "A" and not fm.get("source_trust_level"):
        # we just promoted this to A; record provenance
        new_fm["inferred_by"] = "Claude Opus 4.7 (auto-normalize 2026-05-08)"

    # tech_stack — only if detected
    if detect_tech(folder, cleaned_tags, body):
        ts_old = fm.get("tech_stack")
        if isinstance(ts_old, dict):
            new_fm["tech_stack"] = ts_old
        else:
            new_fm["tech_stack"] = {"language": "unspecified", "framework": "unspecified"}

    return new_fm


def render_frontmatter(fm: dict) -> str:
    lines = ["---"]
    order = [
        "id", "title", "category", "status", "redirect_to", "canonical_id", "aliases",
        "duplicate_of", "source_trust_level", "confidence_score",
        "tags", "raw_sources", "last_reinforced", "github_commit",
        "inferred_by", "tech_stack",
    ]
    for k in order:
        if k not in fm:
            continue
        v = fm[k]
        if isinstance(v, list):
            if not v:
                lines.append(f"{k}: []")
            else:
                items = ", ".join(quote_yaml(x) for x in v)
                lines.append(f"{k}: [{items}]")
        elif isinstance(v, dict):
            lines.append(f"{k}:")
            for kk, vv in v.items():
                lines.append(f"  {kk}: {quote_yaml(vv)}")
        elif isinstance(v, float):
            lines.append(f"{k}: {v}")
        else:
            lines.append(f"{k}: {quote_yaml(v)}")
    lines.append("---")
    return "\n".join(lines)


def normalize_headers(body: str) -> str:
    """Rewrite legacy section headers to match the template names."""
    def repl(m: re.Match) -> str:
        hashes = m.group(1)
        text = m.group(2).strip()
        # strip leading emoji/punctuation/numbering for matching
        norm = EMOJI_AT_START_RE.sub("", text).strip()
        norm = re.sub(r"\s*\(.*?\)\s*$", "", norm).strip()  # drop trailing paren
        key = norm.lower()
        # also try without punctuation
        if key not in HEADER_RENAMES:
            key2 = re.sub(r"[^\w가-힣 -]", "", key).strip()
            if key2 in HEADER_RENAMES:
                key = key2
        if key in HEADER_RENAMES:
            return f"{hashes} {HEADER_RENAMES[key]}"
        return m.group(0)
    return HEADING_RE.sub(repl, body)


def find_section_starts(body: str) -> dict[str, int]:
    """Return {section_canonical_name: line_offset}"""
    found = {}
    for m in HEADING_RE.finditer(body):
        text = m.group(2).strip()
        norm = EMOJI_AT_START_RE.sub("", text).strip().lower()
        norm = re.sub(r"\s*\(.*?\)\s*$", "", norm).strip()
        for k, canonical in HEADER_RENAMES.items():
            if norm == k.lower() or text == canonical:
                if canonical not in found:
                    found[canonical] = m.start()
    return found


REQUIRED_SECTIONS_BASE = [
    "📌 한 줄 통찰 (The Karpathy Summary)",
    "📖 구조화된 지식 (Synthesized Content)",
    "🤖 LLM 활용 힌트 (How to Use This Knowledge)",
    "🧪 검증 상태 (Validation)",
    "🧬 중복 검사 (Duplicate Check)",
    "⚠️ 모순 및 업데이트 (Contradictions & Updates)",
    "🔗 지식 연결 (Graph)",
    "🕓 변경 이력 (Changelog)",
]
REQUIRED_SECTIONS_TECH_EXTRA = [
    "💻 코드 패턴 (Code Patterns)",
    "🤔 의사결정 기준 (Decision Criteria)",
    "❌ 안티패턴 (Anti-Patterns)",
]


def append_missing_sections(body: str, is_tech: bool, is_stub: bool, fm: dict) -> str:
    """For each required section not present, append a scaffold with TODO marker."""
    found = find_section_starts(body)
    missing = []
    base = REQUIRED_SECTIONS_BASE + (REQUIRED_SECTIONS_TECH_EXTRA if is_tech else [])
    for s in base:
        if s not in found:
            missing.append(s)

    if not missing and not is_stub:
        return body

    out = [body.rstrip()]
    if is_stub:
        out.append("\n\n> 🤖 **[AI 추론 보강 필요]** — 본문이 200자 미만이라 P-Reinforce가 빈약 stub으로 분류했습니다.")
        out.append(f"> source_trust_level=`C` (AI 보강분), confidence_score=`{fm.get('confidence_score', 0.7)}`로 표시되어 있습니다.")
        out.append("> 사용자 검증 후 trust_level 상향 조정 가능.\n")
    for s in missing:
        out.append(f"\n## {s}\n")
        out.append(_scaffold_for(s, fm))
    return "\n".join(out)


def _scaffold_for(section: str, fm: dict) -> str:
    if section.startswith("📌"):
        return "> *(TODO: 한 문장으로 핵심 통찰을 작성. \"X는 Y 조건에서 Z 효과를 낸다\" 구조 권장.)*"
    if section.startswith("📖"):
        return "**추출된 패턴:**\n> *(TODO)*\n\n**세부 내용:**\n- *(TODO)*"
    if section.startswith("💻"):
        return "**패턴 1:** *(TODO: 이 프로젝트 컨벤션 반영한 구조 스켈레톤)*\n\n```text\n# TODO\n```"
    if section.startswith("🤔"):
        return "**선택 A를 써야 할 때:**\n- *(TODO)*\n\n**선택 B를 써야 할 때:**\n- *(TODO)*\n\n**기본값:**\n> *(TODO)*"
    if section.startswith("❌"):
        return "- **[안티패턴]:** *(TODO: 무엇을 하면 안 되는가 + 이유 + 대신 무엇을)*"
    if section.startswith("🤖") and "활용 힌트" in section:
        return "**언제 이 지식을 쓰는가:**\n- *(TODO)*\n\n**언제 쓰면 안 되는가:**\n- *(TODO)*"
    if section.startswith("🧪"):
        return (
            f"- **정보 상태:** {fm.get('status', 'draft')}\n"
            f"- **출처 신뢰도:** {fm.get('source_trust_level', 'C')}\n"
            f"- **검토 이유:** *(P-Reinforce Phase 1 자동 정규화. 본문 검증 필요.)*"
        )
    if section.startswith("🧬"):
        return (
            "- **기존 유사 문서:** *(TODO: 인덱서 클러스터 리포트 참조)*\n"
            "- **처리 방식:** UPDATE (자동 정규화)\n"
            "- **처리 이유:** Phase 1 정규화 — 옛 템플릿/누락 필드 보강."
        )
    if section.startswith("⚠️"):
        return "- **과거 데이터와의 충돌:** 없음\n- **정책 변화:** 없음"
    if section.startswith("🔗"):
        return (
            "- **Parent:** [[10_Wiki/Topics]]\n"
            "- **Related:** *(TODO: 최소 2개)*\n"
            "- **Opposite / Trade-off:** *(TODO)*\n"
            "- **Raw Source:** 직접 입력"
        )
    if section.startswith("🕓"):
        today = date.today().isoformat()
        trust = fm.get("source_trust_level", "C")
        return (
            "| 날짜 | 변경 내용 | 처리 방식 | 신뢰도 |\n"
            "|------|-----------|-----------|--------|\n"
            f"| {today} | P-Reinforce Phase 1 정규화 (frontmatter + 헤더 표준화) | UPDATE | {trust} |"
        )
    return "*(TODO)*"


def normalize_file(file_path: Path) -> str:
    text = file_path.read_text(encoding="utf-8", errors="replace")
    fm_old, body, _raw = parse_frontmatter(text)
    fm_new = build_frontmatter(fm_old, file_path, body)

    # Redirect documents stay minimal — don't add scaffold sections.
    if fm_new.get("redirect_to"):
        return render_frontmatter(fm_new) + "\n\n" + body.lstrip("\n")

    # Header renames first
    body2 = normalize_headers(body)
    is_tech = detect_tech(file_path.parent.name, fm_new.get("tags", []), body2)
    is_stub = len(body2.strip()) < 200
    body3 = append_missing_sections(body2, is_tech, is_stub, fm_new)

    return render_frontmatter(fm_new) + "\n\n" + body3.lstrip("\n")


def iter_knowledge_files() -> list[Path]:
    """All .md under Topics/, excluding operational paths (sessions, _agents,
    _company, etc) and non-content dirs."""
    EXCLUDE_FRAG = (
        "/sessions/", "/_agents/", "/_company/", "/memory/",
        "/Project_Logs/", "/Harness_Research_", "/docs/records/",
        "/_Archive_Orphans/", "/Post_Drafts/", "/UX_Scenarios/",
    )
    SKIP_DIRS = {".obsidian", ".git", "__pycache__", "node_modules"}
    out = []
    import os as _os
    for dirpath, dirs, files in _os.walk(TOPICS):
        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
        for f in files:
            if not f.endswith(".md"):
                continue
            p = Path(dirpath) / f
            rel = "/" + str(p.relative_to(ROOT)).replace("\\", "/")
            if any(x in rel for x in EXCLUDE_FRAG):
                continue
            out.append(p)
    return out


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("path", nargs="?", help="Relative path under E:/Wiki/2nd/ or absolute")
    parser.add_argument("--out", help="Write normalized output to PATH instead of stdout")
    parser.add_argument("--batch", help="Path to a text file with one input path per line; outputs go under _tools/sample_normalized/")
    parser.add_argument("--apply-all", action="store_true", help="Normalize ALL knowledge files in-place (operational paths excluded). DESTRUCTIVE.")
    args = parser.parse_args()

    if args.apply_all:
        files = iter_knowledge_files()
        ok = err = 0
        errors: list[tuple[str, str]] = []
        for i, p in enumerate(files, 1):
            try:
                normalized = normalize_file(p)
                p.write_text(normalized, encoding="utf-8")
                ok += 1
            except Exception as e:
                err += 1
                errors.append((str(p), str(e)))
            if i % 200 == 0:
                print(f"  ...{i}/{len(files)}", file=sys.stderr)
        print(f"DONE: {ok} OK, {err} errors out of {len(files)} files", file=sys.stderr)
        if errors:
            log = ROOT / "_tools" / "normalize_errors.log"
            log.write_text("\n".join(f"{p}\t{e}" for p, e in errors), encoding="utf-8")
            print(f"  errors written to {log}", file=sys.stderr)
        return 0 if err == 0 else 1

    if args.batch:
        listing = Path(args.batch).read_text(encoding="utf-8").splitlines()
        out_dir = ROOT / "_tools" / "sample_normalized"
        out_dir.mkdir(parents=True, exist_ok=True)
        results = []
        for line in listing:
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            p = Path(line)
            if not p.is_absolute():
                p = ROOT / p
            if not p.exists():
                print(f"SKIP missing: {p}", file=sys.stderr)
                continue
            try:
                normalized = normalize_file(p)
            except Exception as e:
                print(f"FAIL {p}: {e}", file=sys.stderr)
                continue
            out_path = out_dir / (p.stem + ".normalized.md")
            out_path.write_text(normalized, encoding="utf-8")
            results.append({"src": str(p.relative_to(ROOT)).replace("\\", "/"),
                            "out": str(out_path.relative_to(ROOT)).replace("\\", "/"),
                            "src_chars": len(p.read_text(encoding="utf-8", errors="replace")),
                            "out_chars": len(normalized)})
        (out_dir / "_manifest.json").write_text(json.dumps(results, ensure_ascii=False, indent=1), encoding="utf-8")
        print(f"Wrote {len(results)} normalized samples to {out_dir}", file=sys.stderr)
        return 0

    if not args.path:
        parser.error("path or --batch required")
    p = Path(args.path)
    if not p.is_absolute():
        p = ROOT / p
    if not p.exists():
        print(f"FAIL: {p} does not exist", file=sys.stderr)
        return 2
    out = normalize_file(p)
    if args.out:
        Path(args.out).write_text(out, encoding="utf-8")
        print(f"Wrote {args.out}", file=sys.stderr)
    else:
        sys.stdout.write(out)
    return 0


if __name__ == "__main__":
    sys.exit(main())