cayce-ops/cayceops/components/scanner/scanner.py

"""scanner — opportunity discovery. Proposes; never executes.

Sources come from ``CAYCE_OPS_SCANNER_SOURCES`` (JSON array):
    [{"type": "rss",  "url": "https://example.org/feed.xml"},
     {"type": "page", "url": "https://example.org/partners"}]

Every discovered candidate becomes a ``proposals`` row with status ``proposed``
and a stable ``dedupe_key`` (sha1 of url-or-title) so repeated scans stay
idempotent. The scanner performs no side effects beyond writing proposals and
audit entries.
"""

from __future__ import annotations

import hashlib
import json
import re
import urllib.error
import urllib.request
from datetime import UTC, datetime
from html.parser import HTMLParser
from xml.etree import ElementTree

from cayceops.config import Config
from cayceops.db import session_scope
from cayceops.models import DaemonRun, Proposal, ProposalSource, ProposalStatus, audit

# kind classification heuristics (checked in order)
_KIND_PATTERNS = [
    ("partnership", re.compile(r"partner|collaborat|integrat", re.I)),
    ("affiliate", re.compile(r"affiliate|referral|commission|revenue\s*share", re.I)),
]
DEFAULT_KIND = "content_gap"


class _LinkCollector(HTMLParser):
    """Minimal HTML parser: page <title> + anchor (href, text) pairs."""

    def __init__(self) -> None:
        super().__init__()
        self.title: str = ""
        self.links: list[tuple[str, str]] = []
        self._in_title = False
        self._in_anchor = False
        self._href = ""
        self._text: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str]]) -> None:
        if tag == "title":
            self._in_title = True
        elif tag == "a":
            self._in_anchor = True
            self._href = dict(attrs).get("href", "")
            self._text = []

    def handle_endtag(self, tag: str) -> None:
        if tag == "title":
            self._in_title = False
        elif tag == "a" and self._in_anchor:
            self._in_anchor = False
            text = " ".join("".join(self._text).split())
            if self._href and text:
                self.links.append((self._href, text))

    def handle_data(self, data: str) -> None:
        if self._in_title:
            self.title += data
        if self._in_anchor:
            self._text.append(data)


def _fetch(url: str, cfg: Config, timeout: float = 20.0) -> bytes:
    req = urllib.request.Request(
        url, headers={"User-Agent": cfg.scanner_user_agent}, method="GET"
    )
    with urllib.request.urlopen(req, timeout=timeout) as resp:  # noqa: S310
        return resp.read()


def _parse_rss(body: bytes) -> list[tuple[str, str, str]]:
    """Return (title, link, description) tuples from an RSS/Atom feed."""
    out: list[tuple[str, str, str]] = []
    root = ElementTree.fromstring(body)
    ns = {"atom": "http://www.w3.org/2005/Atom"}
    for item in root.iter("item"):  # RSS 2.0
        title = item.findtext("title") or ""
        link = item.findtext("link") or ""
        desc = re.sub(r"<[^>]+>", " ", item.findtext("description") or "")
        out.append((" ".join(title.split()), link, " ".join(desc.split())[:500]))
    for entry in root.findall("atom:entry", ns):  # Atom
        title = entry.findtext("atom:title", namespaces=ns) or ""
        link_el = entry.find("atom:link", ns)
        link = link_el.get("href", "") if link_el is not None else ""
        desc = re.sub(r"<[^>]+>", " ", entry.findtext("atom:summary", namespaces=ns) or "")
        out.append((" ".join(title.split()), link, " ".join(desc.split())[:500]))
    return out


def _parse_page(body: bytes) -> list[tuple[str, str, str]]:
    """Return (title=anchor text, link=href, description=page title) tuples."""
    parser = _LinkCollector()
    parser.feed(body.decode("utf-8", errors="replace"))
    page_title = " ".join(parser.title.split())
    return [(text, href, f"linked from page: {page_title}") for href, text in parser.links]


def _classify(title: str, description: str) -> str:
    blob = f"{title}\n{description}"
    for kind, pattern in _KIND_PATTERNS:
        if pattern.search(blob):
            return kind
    return DEFAULT_KIND


def _score(title: str, description: str, keywords: list[str]) -> float:
    blob = f"{title}\n{description}".lower()
    hits = sum(1 for kw in keywords if kw.lower() in blob)
    return round(min(hits / max(1, len(keywords)), 1.0), 3)


def scan(cfg: Config, sess, fetch=_fetch) -> dict:
    """Run one scan pass against all configured sources. Returns run stats."""
    sources = json.loads(cfg.scanner_sources or "[]")
    stats = {"sources": len(sources), "candidates": 0, "proposed": 0, "skipped_existing": 0}
    existing = {
        key
        for (key,) in sess.query(Proposal.dedupe_key).filter(
            Proposal.dedupe_key.isnot(None)
        )
    }

    for src in sources:
        url = src.get("url", "")
        stype = src.get("type", "page")
        if not url:
            continue
        try:
            body = fetch(url, cfg)
        except (urllib.error.URLError, OSError) as exc:
            stats.setdefault("source_errors", []).append({"url": url, "error": str(exc)[:200]})
            continue
        items = _parse_rss(body) if stype == "rss" else _parse_page(body)
        stats["candidates"] += len(items)

        for title, link, desc in items:
            text = title or link
            if not text:
                continue
            keywords = cfg.scanner_keywords
            score = _score(title or link, desc, keywords)
            if score <= 0 and stype != "page":
                continue  # feeds must match at least one keyword; pages keep links
            dedupe = hashlib.sha1((link or title).encode()).hexdigest()[:32]  # noqa: S324
            if dedupe in existing:
                stats["skipped_existing"] += 1
                continue
            proposal = Proposal(
                source=ProposalSource.SCANNER,
                kind=_classify(text, desc),
                title=text[:512],
                url=(link or None)[:1024] if (link or None) else None,
                detail={
                    "source_type": stype,
                    "source_url": url,
                    "summary": desc,
                    "scanned_at": datetime.now(UTC).isoformat(),
                },
                match_score=score,
                status=ProposalStatus.PROPOSED,
                dedupe_key=dedupe,
            )
            sess.add(proposal)
            sess.flush()
            audit(
                sess,
                actor="daemon:scanner",
                action="propose",
                proposal_id=proposal.id,
                detail={"kind": proposal.kind, "score": score, "source": stype},
            )
            existing.add(dedupe)
            stats["proposed"] += 1
    return stats


def run(cfg: Config | None = None) -> int:
    """CLI entry: one scan pass, wrapped in a DaemonRun record."""
    from cayceops.config import load
    from cayceops.db import engine

    cfg = cfg or load()
    try:
        with session_scope() as sess:
            run_row = DaemonRun(daemon="scanner")
            sess.add(run_row)
            sess.flush()
            try:
                stats = scan(cfg, sess)
            except Exception as exc:  # noqa: BLE001 — daemon boundary: record and fail cleanly
                run_row.status = "error"
                run_row.detail = {"error": str(exc)[:300]}
                run_row.finished_at = datetime.now(UTC)
                return 1
            run_row.status = "ok"
            run_row.detail = stats
            run_row.finished_at = datetime.now(UTC)
        return 0
    finally:
        engine().dispose()