#!/usr/bin/env python3
"""ingest.py — normalize heterogeneous sources into staged captures.

Pulls raw content from local files, folders, and URLs into a flat staging
directory of UTF-8 markdown "capture" files, each carrying provenance
frontmatter. A `manifest.json` indexes everything so the host LLM can read the
captures and synthesize OKF concept documents from them.

Pure Python 3 standard library. PDF extraction is attempted via `pypdf` if it
is installed; otherwise the PDF is recorded as needing manual/host extraction
(no hard failure, so the skill stays portable).

Usage:
    python ingest.py --out ./_ingest --source ./docs --source https://example.com/page
    python ingest.py --out ./_ingest --source ./report.pdf --max-bytes 5000000

Sources are auto-detected:
    *  http(s)://...      → fetched and stripped to text
    *  a directory        → walked recursively for known text/doc extensions
    *  a file             → captured directly (PDF handled specially)

Network/auth-bound systems (Notion, Confluence, Google Drive, …) are intentionally
NOT fetched here — prefer the host agent's connectors/MCP tools, then point this
script at the exported files. See references/connectors.md.
"""

from __future__ import annotations

import argparse
import datetime as _dt
import json
import os
import sys
import urllib.request
from html.parser import HTMLParser

from okf_common import eprint, slugify

TEXT_EXTENSIONS = {
    ".md", ".markdown", ".txt", ".rst", ".csv", ".tsv", ".json", ".yaml", ".yml",
    ".html", ".htm", ".py", ".js", ".ts", ".java", ".go", ".rb", ".sql", ".sh",
    ".toml", ".ini", ".cfg", ".xml",
}
SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", ".idea", ".vscode", "dist", "build"}
USER_AGENT = "okf-studio-ingest/1.0"


class _HTMLTextExtractor(HTMLParser):
    """Strip HTML to readable text, dropping script/style and collapsing space."""

    _SKIP = {"script", "style", "noscript"}
    _BLOCK = {"p", "div", "section", "article", "li", "tr", "br", "h1", "h2", "h3", "h4", "h5", "h6"}

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self._chunks: list[str] = []
        self._skip_depth = 0
        self.title: str | None = None
        self._in_title = False

    def handle_starttag(self, tag, attrs):
        if tag in self._SKIP:
            self._skip_depth += 1
        if tag == "title":
            self._in_title = True
        if tag in self._BLOCK:
            self._chunks.append("\n")

    def handle_endtag(self, tag):
        if tag in self._SKIP and self._skip_depth:
            self._skip_depth -= 1
        if tag == "title":
            self._in_title = False
        if tag in self._BLOCK:
            self._chunks.append("\n")

    def handle_data(self, data):
        if self._skip_depth:
            return
        if self._in_title:
            if self.title is None and data.strip():
                self.title = data.strip()
            return  # keep the <title> text out of the body
        self._chunks.append(data)

    def text(self) -> str:
        joined = "".join(self._chunks)
        lines = [ln.strip() for ln in joined.splitlines()]
        out: list[str] = []
        blank = False
        for ln in lines:
            if ln:
                out.append(ln)
                blank = False
            elif not blank:
                out.append("")
                blank = True
        return "\n".join(out).strip()


def _now() -> str:
    return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def _write_capture(out_dir: str, slug: str, source: str, kind: str, title: str, text: str) -> dict:
    fname = f"{slug}.md"
    i = 2
    while os.path.exists(os.path.join(out_dir, fname)):
        fname = f"{slug}-{i}.md"
        i += 1
    path = os.path.join(out_dir, fname)
    frontmatter = (
        "---\n"
        "capture: true\n"
        f"source: {source}\n"
        f"source_kind: {kind}\n"
        f"title: {title}\n"
        f"ingested_at: {_now()}\n"
        f"char_count: {len(text)}\n"
        "---\n\n"
    )
    with open(path, "w", encoding="utf-8") as fh:
        fh.write(frontmatter + text + "\n")
    return {
        "file": fname,
        "source": source,
        "source_kind": kind,
        "title": title,
        "char_count": len(text),
    }


def ingest_url(url: str, out_dir: str, max_bytes: int) -> dict | None:
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:  # noqa: S310 (trusted user input)
            ctype = resp.headers.get("Content-Type", "")
            data = resp.read(max_bytes + 1)
    except Exception as exc:  # noqa: BLE001
        eprint(f"  ! failed to fetch {url}: {exc}")
        return None
    if len(data) > max_bytes:
        eprint(f"  ! {url} exceeds --max-bytes ({max_bytes}); truncating.")
        data = data[:max_bytes]
    charset = "utf-8"
    if "charset=" in ctype:
        charset = ctype.split("charset=", 1)[1].split(";", 1)[0].strip() or "utf-8"
    body = data.decode(charset, errors="replace")
    if "html" in ctype or body.lstrip()[:1] == "<":
        parser = _HTMLTextExtractor()
        parser.feed(body)
        text = parser.text()
        title = parser.title or url
    else:
        text = body
        title = url
    return _write_capture(out_dir, slugify(title or url), url, "url", title, text)


def ingest_pdf(path: str, out_dir: str) -> dict | None:
    try:
        import pypdf  # type: ignore
    except ImportError:
        eprint(
            f"  ! pypdf not installed; cannot extract {path}. "
            "Install with `pip install pypdf`, or have the host agent extract the "
            "text and pass it as a .txt/.md file."
        )
        text = f"[PDF NOT EXTRACTED] {path}\nInstall pypdf or extract via the host agent."
        return _write_capture(out_dir, slugify(os.path.basename(path)), path, "pdf-unextracted",
                              os.path.basename(path), text)
    try:
        reader = pypdf.PdfReader(path)
        pages = [page.extract_text() or "" for page in reader.pages]
    except Exception as exc:  # noqa: BLE001
        eprint(f"  ! failed to read PDF {path}: {exc}")
        return None
    text = "\n\n".join(pages).strip()
    return _write_capture(out_dir, slugify(os.path.basename(path)), path, "pdf",
                          os.path.basename(path), text)


def ingest_file(path: str, out_dir: str, max_bytes: int) -> dict | None:
    ext = os.path.splitext(path)[1].lower()
    if ext == ".pdf":
        return ingest_pdf(path, out_dir)
    if ext not in TEXT_EXTENSIONS:
        eprint(f"  ! skipping unsupported file type: {path}")
        return None
    try:
        with open(path, "rb") as fh:
            data = fh.read(max_bytes + 1)
    except OSError as exc:
        eprint(f"  ! cannot read {path}: {exc}")
        return None
    if len(data) > max_bytes:
        eprint(f"  ! {path} exceeds --max-bytes; truncating.")
        data = data[:max_bytes]
    text = data.decode("utf-8", errors="replace")
    if ext in {".html", ".htm"}:
        parser = _HTMLTextExtractor()
        parser.feed(text)
        title = parser.title or os.path.basename(path)
        text = parser.text()
        slug_base = title
    else:
        title = os.path.basename(path)
        slug_base = os.path.splitext(os.path.basename(path))[0]
    return _write_capture(out_dir, slugify(slug_base), path, "file", title, text)


def ingest_dir(path: str, out_dir: str, max_bytes: int) -> list[dict]:
    captured: list[dict] = []
    for dirpath, dirnames, filenames in os.walk(path):
        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
        for name in sorted(filenames):
            ext = os.path.splitext(name)[1].lower()
            if ext not in TEXT_EXTENSIONS and ext != ".pdf":
                continue
            result = ingest_file(os.path.join(dirpath, name), out_dir, max_bytes)
            if result:
                captured.append(result)
    return captured


def main() -> int:
    parser = argparse.ArgumentParser(description="Ingest sources into staged OKF captures.")
    parser.add_argument("--out", required=True, help="Staging directory for captures.")
    parser.add_argument("--source", action="append", required=True, dest="sources",
                        help="A file, directory, or http(s) URL. Repeatable.")
    parser.add_argument("--max-bytes", type=int, default=10_000_000,
                        help="Per-source byte cap (default 10MB).")
    args = parser.parse_args()

    os.makedirs(args.out, exist_ok=True)
    manifest: list[dict] = []

    for source in args.sources:
        print(f"Ingesting: {source}")
        if source.startswith(("http://", "https://")):
            result = ingest_url(source, args.out, args.max_bytes)
            if result:
                manifest.append(result)
        elif os.path.isdir(source):
            manifest.extend(ingest_dir(source, args.out, args.max_bytes))
        elif os.path.isfile(source):
            result = ingest_file(source, args.out, args.max_bytes)
            if result:
                manifest.append(result)
        else:
            eprint(f"  ! source not found: {source}")

    manifest_path = os.path.join(args.out, "manifest.json")
    with open(manifest_path, "w", encoding="utf-8") as fh:
        json.dump({"ingested_at": _now(), "captures": manifest}, fh, indent=2)

    total_chars = sum(c["char_count"] for c in manifest)
    print(f"\nCaptured {len(manifest)} item(s), {total_chars:,} chars → {args.out}")
    print(f"Manifest: {manifest_path}")
    return 0


if __name__ == "__main__":
    sys.exit(main())
