Merge PR #3: mail-negotiator

IMAP classification (rules + optional LLM), conservative needs_review,
reply drafts gated on explicit human approval, SMTP approved-only,
message-id threading, full audit chain. 29 tests green.

cayceops/components/executor/__init__.py

@@ -0,0 +1 @@
+"""executor package."""

cayceops/components/executor/executor.py

@@ -0,0 +1,246 @@
+"""executor — executes APPROVED proposals through registered action plugins.
+
+Guarantees
+----------
+- Only ``approved`` proposals execute (``AUTONOMOUS_MODE=false`` default).
+- With ``AUTONOMOUS_MODE=true`` the executor gains exactly one capability:
+  bounded auto-approval of ``proposed`` items whose ``kind`` is in
+  ``CAYCE_OPS_AUTONOMOUS_KINDS``, capped by ``CAYCE_OPS_AUTONOMOUS_DAILY_CAP``,
+  recorded with actor ``auto:policy`` (DECISIONS D5).
+- Every transition writes an ``audit_log`` row: who approved, what ran,
+  plugin output, success/failure.
+- Unknown proposal kinds fall back to the ``log_action`` plugin (never a
+  network side effect). See DECISIONS D9.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import func, select
+
+from cayceops.components.executor import plugins as plugin_registry
+from cayceops.config import Config
+from cayceops.db import session_scope
+from cayceops.models import AuditEntry, DaemonRun, Proposal, ProposalStatus, audit
+
+Actor = str
+
+
+# --- human transitions -------------------------------------------------------
+
+
+def approve(proposal_id: int, actor: Actor, cfg: Config | None = None, sess=None) -> int:
+    """Human approves a proposal. Audited with the operator's identity."""
+    return _decide(proposal_id, actor, ProposalStatus.APPROVED, cfg, sess)
+
+
+def reject(proposal_id: int, actor: Actor, cfg: Config | None = None, sess=None) -> int:
+    """Human rejects a proposal. Audited with the operator's identity."""
+    return _decide(proposal_id, actor, ProposalStatus.REJECTED, cfg, sess)
+
+
+def _decide(
+    proposal_id: int,
+    actor: Actor,
+    decision: ProposalStatus,
+    cfg: Config | None,
+    sess,
+) -> int:
+    owned = False
+    if sess is None:
+        outer = session_scope()
+        sess = outer.__enter__()
+        owned = True
+    try:
+        proposal = sess.get(Proposal, proposal_id)
+        if proposal is None:
+            return 1
+        if proposal.status != ProposalStatus.PROPOSED:
+            verb = "approve" if decision == ProposalStatus.APPROVED else "reject"
+            audit(
+                sess,
+                actor=actor,
+                action=verb,
+                proposal_id=proposal_id,
+                result="error",
+                detail={"reason": f"invalid state: {proposal.status.value}"},
+            )
+            return 1
+        proposal.status = decision
+        if decision == ProposalStatus.APPROVED:
+            proposal.approved_by = actor
+            proposal.approved_at = datetime.now(UTC)
+        verb = "approve" if decision == ProposalStatus.APPROVED else "reject"
+        audit(sess, actor=actor, action=verb, proposal_id=proposal_id)
+        return 0
+    finally:
+        if owned:
+            outer.__exit__(None, None, None)
+
+
+# --- autonomous policy (the ONLY non-human approval path, D5) -----------------
+
+
+def _auto_approve(cfg: Config, sess) -> int:
+    """Auto-approve proposed items per policy. Returns count approved."""
+    if not cfg.autonomous_mode or not cfg.autonomous_kinds:
+        return 0
+    midnight = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
+    used = (
+        sess.query(func.count(AuditEntry.id))
+        .filter(
+            AuditEntry.actor == "auto:policy",
+            AuditEntry.action == "approve",
+            AuditEntry.ts >= midnight,
+        )
+        .scalar()
+    )
+    budget = max(0, cfg.autonomous_daily_cap - int(used or 0))
+    approved = 0
+    if budget == 0:
+        return 0
+    candidates = (
+        sess.query(Proposal)
+        .filter(
+            Proposal.status == ProposalStatus.PROPOSED,
+            Proposal.kind.in_(cfg.autonomous_kinds),
+        )
+        .order_by(Proposal.risk_score.asc(), Proposal.id.asc())
+        .limit(budget)
+        .all()
+    )
+    for proposal in candidates:
+        proposal.status = ProposalStatus.APPROVED
+        proposal.approved_by = "auto:policy"
+        proposal.approved_at = datetime.now(UTC)
+        audit(
+            sess,
+            actor="auto:policy",
+            action="approve",
+            proposal_id=proposal.id,
+            detail={
+                "reason": "autonomous_mode policy",
+                "kind": proposal.kind,
+                "risk_score": proposal.risk_score,
+                "daily_cap": cfg.autonomous_daily_cap,
+            },
+        )
+        approved += 1
+    return approved
+
+
+# --- execution ----------------------------------------------------------------
+
+
+def execute_one(proposal: Proposal, cfg: Config, sess) -> bool:
+    """Run the plugin for one approved proposal. Returns True on success."""
+    proposal.status = ProposalStatus.RUNNING
+    proposal.attempts += 1
+    audit(
+        sess,
+        actor="daemon:executor",
+        action="execute_start",
+        proposal_id=proposal.id,
+        detail={"plugin": proposal.kind, "attempt": proposal.attempts},
+    )
+    plugin = plugin_registry.REGISTRY.get(proposal.kind)
+    try:
+        if plugin is None:
+            outcome: dict[str, Any] = plugin_registry.log_action(proposal, cfg, sess)
+            outcome["plugin_fallback"] = True
+            outcome["note"] = f"no plugin registered for kind {proposal.kind!r}"
+        else:
+            outcome = plugin(proposal, cfg, sess)
+    except Exception as exc:  # noqa: BLE001 — plugin boundary: capture for audit
+        audit(
+            sess,
+            actor="daemon:executor",
+            action="execute_error",
+            proposal_id=proposal.id,
+            result="error",
+            detail={"error": str(exc)[:300], "attempt": proposal.attempts},
+        )
+        if proposal.attempts >= cfg.max_attempts:
+            proposal.status = ProposalStatus.FAILED
+            audit(
+                sess,
+                actor="daemon:executor",
+                action="fail",
+                proposal_id=proposal.id,
+                result="error",
+                detail={"attempts": proposal.attempts},
+            )
+        else:
+            proposal.status = ProposalStatus.APPROVED  # retry next run
+        return False
+
+    proposal.status = ProposalStatus.DONE
+    audit(
+        sess,
+        actor="daemon:executor",
+        action="execute_done",
+        proposal_id=proposal.id,
+        detail={"outcome": outcome},
+    )
+    return True
+
+
+def run(cfg: Config | None = None, sess=None) -> int:
+    """CLI entry: auto-approve per policy, then execute all approved items."""
+    from cayceops.config import load
+    from cayceops.db import engine as _engine
+
+    cfg = cfg or load()
+    owned = False
+    if sess is None:
+        outer = session_scope()
+        sess = outer.__enter__()
+        owned = True
+    try:
+        run_row = DaemonRun(daemon="executor")
+        sess.add(run_row)
+        sess.flush()
+        try:
+            auto = _auto_approve(cfg, sess)
+            pending = (
+                sess.query(Proposal)
+                .filter(Proposal.status == ProposalStatus.APPROVED)
+                .order_by(Proposal.approved_at.asc())
+                .all()
+            )
+            done = failed = 0
+            for proposal in pending:
+                if execute_one(proposal, cfg, sess):
+                    done += 1
+                else:
+                    failed += 1
+            run_row.status = "ok"
+            run_row.detail = {"auto_approved": auto, "done": done, "failed": failed}
+        except Exception as exc:  # noqa: BLE001 — daemon boundary
+            run_row.status = "error"
+            run_row.detail = {"error": str(exc)[:300]}
+            run_row.finished_at = datetime.now(UTC)
+            return 1
+        run_row.finished_at = datetime.now(UTC)
+        return 0
+    finally:
+        if owned:
+            outer.__exit__(None, None, None)
+            _engine().dispose()
+
+
+def select_approved(sess) -> list[Proposal]:
+    """Helper for tests/tools: currently approved proposals, oldest first."""
+    return list(
+        sess.scalars(
+            select(Proposal)
+            .where(Proposal.status == ProposalStatus.APPROVED)
+            .order_by(Proposal.approved_at.asc())
+        ).all()
+    )
+
+
+ActionPlugin = Callable[[Proposal, Config, Any], dict]

cayceops/components/executor/plugins.py

@@ -0,0 +1,78 @@
+"""executor action plugins.
+
+A plugin maps a proposal ``kind`` to a side effect. Plugins are repo code —
+they are reviewed like everything else and must:
+
+- take ``(proposal, cfg, sess)`` and return a JSON-safe dict (the audit outcome)
+- do their own permission checks (see ``http_post`` + ENABLE_HTTP_ACTIONS)
+- raise on failure (the executor captures the exception into the audit log)
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import Any
+
+from cayceops.config import Config
+from cayceops.models import Proposal, audit
+
+REGISTRY: dict[str, Any] = {}
+
+
+def register(kind: str):
+    """Decorator: register an action plugin for a proposal kind."""
+
+    def deco(fn):
+        REGISTRY[kind] = fn
+        return fn
+
+    return deco
+
+
+@register("log_action")
+def log_action(proposal: Proposal, cfg: Config, sess) -> dict:
+    """No-side-effect plugin: record the execution in the audit log only."""
+    return {"action": "logged", "title": proposal.title[:200]}
+
+
+@register("http_post")
+def http_post(proposal: Proposal, cfg: Config, sess) -> dict:
+    """POST the proposal payload to a webhook.
+
+    Inert unless ``CAYCE_OPS_ENABLE_HTTP_ACTIONS=true``. Target URL comes from
+    the proposal detail (``detail['webhook_url']``) — operator-controlled data.
+    """
+    if not cfg.enable_http_actions:
+        audit(
+            sess,
+            actor="daemon:executor",
+            action="http_post_blocked",
+            proposal_id=proposal.id,
+            result="error",
+            detail={"reason": "CAYCE_OPS_ENABLE_HTTP_ACTIONS is false"},
+        )
+        raise PermissionError("http_post disabled (CAYCE_OPS_ENABLE_HTTP_ACTIONS=false)")
+
+    url = (proposal.detail or {}).get("webhook_url", "")
+    if not url:
+        raise ValueError("proposal detail.webhook_url missing")
+    payload = json.dumps(
+        {"proposal_id": proposal.id, "kind": proposal.kind, "title": proposal.title}
+    ).encode()
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=20.0) as resp:  # noqa: S310
+        return {"action": "http_post", "status": resp.status}
+
+
+# proposals from the scanner default to content_gap/affiliate/partnership kinds;
+# until dedicated plugins exist for those kinds they must not do anything
+# network-visible. register more plugins here as the operator approves them.
+for _safe_kind in ("affiliate", "partnership", "content_gap", "other"):
+    REGISTRY.setdefault(_safe_kind, log_action)

cayceops/components/maild/__init__.py

@@ -0,0 +1 @@
+"""maild package."""

cayceops/components/maild/classify.py

@@ -0,0 +1,68 @@
+"""maild classification: rules first, LLM optional, conservative threshold.
+
+The LLM never decides alone: any result below ``CAYCE_OPS_LLM_REVIEW_THRESHOLD``
+confidence becomes ``needs_review``. With the ``null`` provider (default) only
+deterministic rules run.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+from cayceops.config import Config
+from cayceops.models import MailClassification
+
+_RULES: list[tuple[MailClassification, re.Pattern[str], float]] = [
+    # (class, pattern, confidence when matched)
+    (MailClassification.REJECTION, re.compile(r"\b(declin|reject|not able to|unable to|unfortunately|we regret|won.t be able)\w*", re.I), 0.9),
+    (MailClassification.APPROVAL, re.compile(r"\b(approv|accept|welcome to the|you.re (in|approved)|live|activated|glad to)\w*", re.I), 0.85),
+    (MailClassification.QUESTION, re.compile(r"\b(could you|can you|please (provide|clarify|confirm)|what (is|are)|how (do|does|can)|need more (info|details))\b", re.I), 0.7),
+]
+
+_LLM_PROMPT = """Classify this merchant email into exactly one of:
+approval, rejection, question, other.
+Reply with strict JSON: {"classification": "...", "confidence": 0.0-1.0}
+
+Subject and body:
+{body}"""
+
+
+def _rules(text: str) -> tuple[MailClassification, float]:
+    for kind, pattern, confidence in _RULES:
+        if pattern.search(text):
+            return kind, confidence
+    return MailClassification.OTHER, 0.3
+
+
+def _llm_classify(text: str, cfg: Config) -> tuple[MailClassification, float] | None:
+    """Ask the configured LLM. Returns None on any failure (rules take over)."""
+    from cayceops.llm import get_provider
+
+    try:
+        provider = get_provider(cfg)
+        raw = provider.complete(_LLM_PROMPT.format(body=text[:4000]), system="You classify merchant emails. Reply with JSON only.")
+        data = json.loads(re.search(r"\{.*\}", raw, re.S).group(0))
+        kind = MailClassification(data["classification"].lower())
+        confidence = float(data["confidence"])
+    except Exception:  # noqa: BLE001 — LLM is best-effort by design
+        return None
+    if kind not in MailClassification or kind is MailClassification.NEEDS_REVIEW:
+        return None
+    return kind, min(max(confidence, 0.0), 1.0)
+
+
+def classify(text: str, cfg: Config) -> tuple[MailClassification, float]:
+    """Classify merchant text. Degrades to rules when no LLM is configured."""
+    if cfg.llm_provider != "null":
+        llm_result = _llm_classify(text, cfg)
+        if llm_result is not None:
+            kind, confidence = llm_result
+        else:  # LLM failed — fall through to rules
+            kind, confidence = _rules(text)
+    else:
+        kind, confidence = _rules(text)
+
+    if confidence < cfg.llm_review_threshold:
+        return MailClassification.NEEDS_REVIEW, confidence
+    return kind, confidence

cayceops/components/maild/maild.py

@@ -0,0 +1,364 @@
+"""maild — classify merchant replies, draft responses, gated SMTP.
+
+Flow
+----
+1. ``poll``: fetch unread inbound mail (IMAP), parse, thread to the proposal
+   via ``In-Reply-To`` of a previously-sent draft, classify (rules + optional
+   LLM), store, and create a REPLY DRAFT (never sent automatically).
+2. ``approve_draft``: a human approves a specific draft (audited).
+3. ``send_approved``: SMTP sends *only* drafts in state ``approved``.
+
+Classification is conservative: anything below the confidence threshold becomes
+``needs_review`` and no draft is generated.
+
+Public signatures match the CLI: ``poll()``, ``approve_draft(id, actor)``,
+``send_approved()``. The ``_*`` variants take an explicit session/fetcher for
+tests and composition.
+"""
+
+from __future__ import annotations
+
+import email
+import email.policy
+import imaplib
+import smtplib
+from datetime import UTC, datetime
+from email.message import EmailMessage
+
+from cayceops.components.maild.classify import classify as _classify_text
+from cayceops.config import Config
+from cayceops.db import session_scope
+from cayceops.models import (
+    DaemonRun,
+    DraftStatus,
+    MailClassification,
+    MailDirection,
+    MailMessage,
+    Proposal,
+    audit,
+)
+
+# --- IMAP ---------------------------------------------------------------------
+
+
+def connect_imap(cfg: Config) -> imaplib.IMAP4_SSL:
+    client = imaplib.IMAP4_SSL(cfg.imap_host, cfg.imap_port)
+    client.login(cfg.imap_user, cfg.imap_password)
+    client.select(cfg.imap_mailbox)
+    return client
+
+
+def _fetch_unread(cfg: Config) -> list[bytes]:
+    """Fetch raw unread messages. Returns list of raw RFC822 bytes."""
+    client = connect_imap(cfg)
+    try:
+        _, data = client.search(None, "UNSEEN")
+        raw_messages: list[bytes] = []
+        for num in data[0].split():
+            _, msg_data = client.fetch(num, "(RFC822)")
+            raw_messages.append(msg_data[0][1])
+            client.store(num, "+FLAGS", "\\Seen")
+        return raw_messages
+    finally:
+        client.logout()
+
+
+# --- parsing ------------------------------------------------------------------
+
+
+def parse_raw(raw: bytes) -> dict:
+    """Parse raw RFC822 bytes into our flat dict shape."""
+    msg = email.message_from_bytes(raw, policy=email.policy.default)
+    body = ""
+    if msg.is_multipart():
+        for part in msg.walk():
+            if part.get_content_type() == "text/plain" and not part.get_filename():
+                body = part.get_content()
+                break
+    else:
+        body = msg.get_content()
+    reply_to = msg.get("In-Reply-To") or msg.get("References") or ""
+    message_id = (msg.get("Message-ID") or "").strip()
+    if not message_id:
+        message_id = f"<generated-{datetime.now(UTC).timestamp()}@localhost>"
+    # store the bare address (envelope-safe); display names stay out of the DB
+    from_addr = email.utils.parseaddr(msg.get("From") or "")[1].strip()[:320]
+    return {
+        "message_id": message_id[:998],
+        "in_reply_to": reply_to.split()[-1].strip() if reply_to.strip() else None,
+        "from_addr": from_addr,
+        "subject": (msg.get("Subject") or "").strip()[:998],
+        "body": body[:20000],
+    }
+
+
+# --- proposal threading -------------------------------------------------------
+
+
+def _match_proposal(parsed: dict, sess) -> Proposal | None:
+    """Thread an inbound mail to its proposal via In-Reply-To of sent drafts."""
+    irt = parsed.get("in_reply_to")
+    if not irt:
+        return None
+    sent = (
+        sess.query(MailMessage)
+        .filter(
+            MailMessage.direction == MailDirection.OUTBOUND,
+            MailMessage.message_id == irt,
+        )
+        .one_or_none()
+    )
+    if sent is None or sent.proposal_id is None:
+        return None
+    return sess.get(Proposal, sent.proposal_id)
+
+
+# --- drafts -------------------------------------------------------------------
+
+_DRAFT_TEMPLATES = {
+    MailClassification.APPROVAL: (
+        "Hello,\n\nThank you for the approval regarding \"{title}\". "
+        "We will proceed on our side and follow up with the next steps.\n\nBest regards,\nCayce"
+    ),
+    MailClassification.REJECTION: (
+        "Hello,\n\nThank you for considering \"{title}\". "
+        "We appreciate the reply and would welcome the chance to revisit in the future.\n\n"
+        "Best regards,\nCayce"
+    ),
+    MailClassification.QUESTION: (
+        "Hello,\n\nRegarding \"{title}\": {question}\n\n"
+        "Could you help us with the details so we can move forward?\n\nBest regards,\nCayce"
+    ),
+}
+
+
+def _build_draft(parsed: dict, kind: MailClassification, proposal: Proposal | None) -> str:
+    title = proposal.title if proposal else (parsed.get("subject") or "your message")
+    template = _DRAFT_TEMPLATES.get(kind)
+    if template is None:
+        return ""
+    question = (parsed.get("body") or "")[:200].strip() or "we had a question"
+    return template.format(title=title[:120], question=question)
+
+
+# --- inbound pass ---------------------------------------------------------------
+
+
+def _poll(cfg: Config, sess, fetcher=None) -> dict:
+    """One inbound pass: parse, classify, thread, draft. Returns stats."""
+    stats = {"processed": 0, "drafted": 0, "needs_review": 0, "skipped": 0}
+    for raw in (fetcher or _fetch_unread)(cfg):
+        parsed = parse_raw(raw)
+        if (
+            sess.query(MailMessage)
+            .filter(MailMessage.message_id == parsed["message_id"])
+            .one_or_none()
+        ):
+            stats["skipped"] += 1
+            continue
+
+        proposal = _match_proposal(parsed, sess)
+        kind, confidence = _classify_text(f"{parsed['subject']}\n{parsed['body']}", cfg)
+        if proposal is None and kind == MailClassification.APPROVAL:
+            # an approval we cannot thread is suspicious — human must look
+            kind = MailClassification.NEEDS_REVIEW
+            confidence = min(confidence, 0.4)
+
+        record = MailMessage(
+            direction=MailDirection.INBOUND,
+            message_id=parsed["message_id"],
+            in_reply_to=parsed["in_reply_to"],
+            from_addr=parsed["from_addr"] or None,
+            subject=parsed["subject"] or None,
+            body=parsed["body"],
+            classification=kind,
+            confidence=confidence,
+            proposal_id=proposal.id if proposal else None,
+            processed_at=datetime.now(UTC),
+        )
+        sess.add(record)
+        sess.flush()
+        audit(
+            sess,
+            actor="daemon:maild",
+            action="classify",
+            proposal_id=proposal.id if proposal else None,
+            detail={"classification": kind.value, "confidence": confidence},
+        )
+        stats["processed"] += 1
+
+        if kind == MailClassification.NEEDS_REVIEW:
+            stats["needs_review"] += 1
+            continue
+
+        if proposal is not None:
+            proposal.detail = {
+                **(proposal.detail or {}),
+                "merchant_status": kind.value,
+                "merchant_reply_at": datetime.now(UTC).isoformat(),
+            }
+
+        draft = _build_draft(parsed, kind, proposal)
+        if draft:
+            record.draft_reply = draft
+            record.draft_status = DraftStatus.DRAFTED
+            audit(
+                sess,
+                actor="daemon:maild",
+                action="draft_reply",
+                proposal_id=proposal.id if proposal else None,
+                detail={"mail_id": record.id},
+            )
+            stats["drafted"] += 1
+    return stats
+
+
+# --- human gate: approve draft -------------------------------------------------
+
+
+def _approve_draft(mail_id: int, actor: str, sess) -> int:
+    """Human approves a drafted reply. Only then may SMTP send it."""
+    record = sess.get(MailMessage, mail_id)
+    if record is None or record.direction != MailDirection.INBOUND:
+        return 1
+    if record.draft_status != DraftStatus.DRAFTED:
+        audit(
+            sess,
+            actor=actor,
+            action="approve_draft",
+            proposal_id=record.proposal_id,
+            result="error",
+            detail={"reason": f"invalid draft state: {record.draft_status.value}"},
+        )
+        return 1
+    record.draft_status = DraftStatus.APPROVED
+    audit(
+        sess,
+        actor=actor,
+        action="approve_draft",
+        proposal_id=record.proposal_id,
+        detail={"mail_id": record.id},
+    )
+    return 0
+
+
+# --- SMTP: only approved drafts ------------------------------------------------
+
+
+def _smtp_send(cfg: Config, to_addr: str, subject: str, body: str, message_id: str) -> None:
+    msg = EmailMessage()
+    msg["From"] = cfg.mail_from
+    msg["To"] = to_addr
+    msg["Subject"] = subject
+    msg["Message-ID"] = message_id
+    msg.set_content(body)
+    with smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=30) as server:
+        server.starttls()
+        server.login(cfg.smtp_user, cfg.smtp_password)
+        server.send_message(msg)
+
+
+def _approved_by(sess, mail_id: int) -> str:
+    from cayceops.models import AuditEntry
+
+    for entry in sess.query(AuditEntry).all():
+        if entry.action == "approve_draft" and entry.detail.get("mail_id") == mail_id:
+            return entry.actor
+    return "unknown"
+
+
+def _send_approved(cfg: Config, sess, sender=None) -> dict:
+    """Send every APPROVED draft. Anything else stays untouched."""
+    stats = {"sent": 0, "blocked": 0}
+    pending = (
+        sess.query(MailMessage)
+        .filter(
+            MailMessage.direction == MailDirection.INBOUND,
+            MailMessage.draft_status == DraftStatus.APPROVED,
+        )
+        .all()
+    )
+    for record in pending:
+        to_addr = record.from_addr or ""
+        if not to_addr:
+            record.draft_status = DraftStatus.SKIPPED
+            stats["blocked"] += 1
+            continue
+        message_id = (
+            f"<cayce-draft-{record.id}-{int(datetime.now(UTC).timestamp())}@cayce-co.xyz>"
+        )
+        (sender or _smtp_send)(
+            cfg,
+            to_addr,
+            f"Re: {record.subject or 'your message'}",
+            record.draft_reply or "",
+            message_id,
+        )
+        approved_by = _approved_by(sess, record.id)
+        record.draft_status = DraftStatus.SENT
+        # store the Message-ID so future replies thread to this draft
+        record.message_id = message_id
+        audit(
+            sess,
+            actor="daemon:maild",
+            action="send_draft",
+            proposal_id=record.proposal_id,
+            detail={
+                "mail_id": record.id,
+                "message_id": message_id,
+                "approved_by": approved_by,
+            },
+        )
+        stats["sent"] += 1
+    return stats
+
+
+# --- CLI wrappers (session scope lives here) ------------------------------------
+
+
+def poll(cfg: Config | None = None) -> int:
+    from cayceops.config import load
+    from cayceops.db import engine
+
+    cfg = cfg or load()
+    try:
+        with session_scope() as sess:
+            run_row = DaemonRun(daemon="maild")
+            sess.add(run_row)
+            sess.flush()
+            try:
+                stats = _poll(cfg, sess)
+            except Exception as exc:  # noqa: BLE001 — daemon boundary
+                run_row.status = "error"
+                run_row.detail = {"error": str(exc)[:300]}
+                run_row.finished_at = datetime.now(UTC)
+                return 1
+            run_row.status = "ok"
+            run_row.detail = stats
+            run_row.finished_at = datetime.now(UTC)
+        return 0
+    finally:
+        engine().dispose()
+
+
+def approve_draft(mail_id: int, actor: str) -> int:
+    from cayceops.db import engine
+
+    try:
+        with session_scope() as sess:
+            return _approve_draft(mail_id, actor, sess)
+    finally:
+        engine().dispose()
+
+
+def send_approved(cfg: Config | None = None) -> int:
+    from cayceops.config import load
+    from cayceops.db import engine
+
+    cfg = cfg or load()
+    try:
+        with session_scope() as sess:
+            _send_approved(cfg, sess)
+        return 0
+    finally:
+        engine().dispose()

cayceops/components/scanner/__init__.py

@@ -0,0 +1 @@
+"""scanner package."""

cayceops/components/scanner/scanner.py

@@ -0,0 +1,209 @@
+"""scanner — opportunity discovery. Proposes; never executes.
+
+Sources come from ``CAYCE_OPS_SCANNER_SOURCES`` (JSON array):
+    [{"type": "rss",  "url": "https://example.org/feed.xml"},
+     {"type": "page", "url": "https://example.org/partners"}]
+
+Every discovered candidate becomes a ``proposals`` row with status ``proposed``
+and a stable ``dedupe_key`` (sha1 of url-or-title) so repeated scans stay
+idempotent. The scanner performs no side effects beyond writing proposals and
+audit entries.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import urllib.error
+import urllib.request
+from datetime import UTC, datetime
+from html.parser import HTMLParser
+from xml.etree import ElementTree
+
+from cayceops.config import Config
+from cayceops.db import session_scope
+from cayceops.models import DaemonRun, Proposal, ProposalSource, ProposalStatus, audit
+
+# kind classification heuristics (checked in order)
+_KIND_PATTERNS = [
+    ("partnership", re.compile(r"partner|collaborat|integrat", re.I)),
+    ("affiliate", re.compile(r"affiliate|referral|commission|revenue\s*share", re.I)),
+]
+DEFAULT_KIND = "content_gap"
+
+
+class _LinkCollector(HTMLParser):
+    """Minimal HTML parser: page <title> + anchor (href, text) pairs."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.title: str = ""
+        self.links: list[tuple[str, str]] = []
+        self._in_title = False
+        self._in_anchor = False
+        self._href = ""
+        self._text: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str]]) -> None:
+        if tag == "title":
+            self._in_title = True
+        elif tag == "a":
+            self._in_anchor = True
+            self._href = dict(attrs).get("href", "")
+            self._text = []
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "title":
+            self._in_title = False
+        elif tag == "a" and self._in_anchor:
+            self._in_anchor = False
+            text = " ".join("".join(self._text).split())
+            if self._href and text:
+                self.links.append((self._href, text))
+
+    def handle_data(self, data: str) -> None:
+        if self._in_title:
+            self.title += data
+        if self._in_anchor:
+            self._text.append(data)
+
+
+def _fetch(url: str, cfg: Config, timeout: float = 20.0) -> bytes:
+    req = urllib.request.Request(
+        url, headers={"User-Agent": cfg.scanner_user_agent}, method="GET"
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:  # noqa: S310
+        return resp.read()
+
+
+def _parse_rss(body: bytes) -> list[tuple[str, str, str]]:
+    """Return (title, link, description) tuples from an RSS/Atom feed."""
+    out: list[tuple[str, str, str]] = []
+    root = ElementTree.fromstring(body)
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    for item in root.iter("item"):  # RSS 2.0
+        title = item.findtext("title") or ""
+        link = item.findtext("link") or ""
+        desc = re.sub(r"<[^>]+>", " ", item.findtext("description") or "")
+        out.append((" ".join(title.split()), link, " ".join(desc.split())[:500]))
+    for entry in root.findall("atom:entry", ns):  # Atom
+        title = entry.findtext("atom:title", namespaces=ns) or ""
+        link_el = entry.find("atom:link", ns)
+        link = link_el.get("href", "") if link_el is not None else ""
+        desc = re.sub(r"<[^>]+>", " ", entry.findtext("atom:summary", namespaces=ns) or "")
+        out.append((" ".join(title.split()), link, " ".join(desc.split())[:500]))
+    return out
+
+
+def _parse_page(body: bytes) -> list[tuple[str, str, str]]:
+    """Return (title=anchor text, link=href, description=page title) tuples."""
+    parser = _LinkCollector()
+    parser.feed(body.decode("utf-8", errors="replace"))
+    page_title = " ".join(parser.title.split())
+    return [(text, href, f"linked from page: {page_title}") for href, text in parser.links]
+
+
+def _classify(title: str, description: str) -> str:
+    blob = f"{title}\n{description}"
+    for kind, pattern in _KIND_PATTERNS:
+        if pattern.search(blob):
+            return kind
+    return DEFAULT_KIND
+
+
+def _score(title: str, description: str, keywords: list[str]) -> float:
+    blob = f"{title}\n{description}".lower()
+    hits = sum(1 for kw in keywords if kw.lower() in blob)
+    return round(min(hits / max(1, len(keywords)), 1.0), 3)
+
+
+def scan(cfg: Config, sess, fetch=_fetch) -> dict:
+    """Run one scan pass against all configured sources. Returns run stats."""
+    sources = json.loads(cfg.scanner_sources or "[]")
+    stats = {"sources": len(sources), "candidates": 0, "proposed": 0, "skipped_existing": 0}
+    existing = {
+        key
+        for (key,) in sess.query(Proposal.dedupe_key).filter(
+            Proposal.dedupe_key.isnot(None)
+        )
+    }
+
+    for src in sources:
+        url = src.get("url", "")
+        stype = src.get("type", "page")
+        if not url:
+            continue
+        try:
+            body = fetch(url, cfg)
+        except (urllib.error.URLError, OSError) as exc:
+            stats.setdefault("source_errors", []).append({"url": url, "error": str(exc)[:200]})
+            continue
+        items = _parse_rss(body) if stype == "rss" else _parse_page(body)
+        stats["candidates"] += len(items)
+
+        for title, link, desc in items:
+            text = title or link
+            if not text:
+                continue
+            keywords = cfg.scanner_keywords
+            score = _score(title or link, desc, keywords)
+            if score <= 0 and stype != "page":
+                continue  # feeds must match at least one keyword; pages keep links
+            dedupe = hashlib.sha1((link or title).encode()).hexdigest()[:32]  # noqa: S324
+            if dedupe in existing:
+                stats["skipped_existing"] += 1
+                continue
+            proposal = Proposal(
+                source=ProposalSource.SCANNER,
+                kind=_classify(text, desc),
+                title=text[:512],
+                url=(link or None)[:1024] if (link or None) else None,
+                detail={
+                    "source_type": stype,
+                    "source_url": url,
+                    "summary": desc,
+                    "scanned_at": datetime.now(UTC).isoformat(),
+                },
+                risk_score=score,
+                status=ProposalStatus.PROPOSED,
+                dedupe_key=dedupe,
+            )
+            sess.add(proposal)
+            sess.flush()
+            audit(
+                sess,
+                actor="daemon:scanner",
+                action="propose",
+                proposal_id=proposal.id,
+                detail={"kind": proposal.kind, "score": score, "source": stype},
+            )
+            existing.add(dedupe)
+            stats["proposed"] += 1
+    return stats
+
+
+def run(cfg: Config | None = None) -> int:
+    """CLI entry: one scan pass, wrapped in a DaemonRun record."""
+    from cayceops.config import load
+    from cayceops.db import engine
+
+    cfg = cfg or load()
+    try:
+        with session_scope() as sess:
+            run_row = DaemonRun(daemon="scanner")
+            sess.add(run_row)
+            sess.flush()
+            try:
+                stats = scan(cfg, sess)
+            except Exception as exc:  # noqa: BLE001 — daemon boundary: record and fail cleanly
+                run_row.status = "error"
+                run_row.detail = {"error": str(exc)[:300]}
+                run_row.finished_at = datetime.now(UTC)
+                return 1
+            run_row.status = "ok"
+            run_row.detail = stats
+            run_row.finished_at = datetime.now(UTC)
+        return 0
+    finally:
+        engine().dispose()

tests/test_executor.py

@@ -0,0 +1,133 @@
+"""executor tests — human gate, autonomy bounds, retries, audit chain."""
+
+from __future__ import annotations
+
+import pytest
+
+from cayceops.components.executor import executor, plugins
+from cayceops.models import AuditEntry, Proposal, ProposalStatus
+
+
+def _proposal(kind="content_gap", title="Test item", **kw):
+    return Proposal(kind=kind, title=title, **kw)
+
+
+def test_approve_requires_proposed_state(cfg, sess):
+    p = _proposal()
+    sess.add(p)
+    sess.flush()
+
+    assert executor.approve(p.id, "gustavo", cfg(), sess) == 0
+    # second approve must fail (state is now approved)
+    assert executor.approve(p.id, "gustavo", cfg(), sess) == 1
+    assert p.approved_by == "gustavo"
+    assert p.status == ProposalStatus.APPROVED
+
+
+def test_reject_records_actor(cfg, sess):
+    p = _proposal()
+    sess.add(p)
+    sess.flush()
+    assert executor.reject(p.id, "gustavo", cfg(), sess) == 0
+    assert p.status == ProposalStatus.REJECTED
+
+
+def test_run_executes_only_approved(cfg, sess):
+    approved = _proposal(title="Approved item")
+    proposed = _proposal(title="Proposed item")
+    sess.add_all([approved, proposed])
+    sess.flush()
+    executor.approve(approved.id, "gustavo", cfg(), sess)
+
+    config = cfg()  # AUTONOMOUS_MODE=false
+    assert executor.run(config, sess) == 0
+
+    assert approved.status == ProposalStatus.DONE
+    assert proposed.status == ProposalStatus.PROPOSED  # untouched
+
+    acts = [a.action for a in sess.query(AuditEntry).order_by(AuditEntry.id).all()]
+    assert "execute_start" in acts and "execute_done" in acts
+    assert acts.count("approve") == 1
+
+
+def test_autonomy_off_by_default(cfg, sess):
+    p = _proposal(kind="content_gap", title="Never auto-runs")
+    sess.add(p)
+    sess.flush()
+    config = cfg()  # autonomous_mode False even with kinds configured
+    executor._auto_approve(config, sess)
+    assert p.status == ProposalStatus.PROPOSED
+
+
+def test_autonomy_respects_kinds_and_daily_cap(cfg, sess):
+    p1 = _proposal(kind="content_gap", title="auto ok")
+    p2 = _proposal(kind="content_gap", title="auto over cap")
+    p3 = _proposal(kind="affiliate", title="kind not allowed")
+    sess.add_all([p1, p2, p3])
+    sess.flush()
+
+    config = cfg(
+        CAYCE_OPS_AUTONOMOUS_MODE="true",
+        CAYCE_OPS_AUTONOMOUS_KINDS="content_gap",
+        CAYCE_OPS_AUTONOMOUS_DAILY_CAP="1",
+    )
+    approved = executor._auto_approve(config, sess)
+    assert approved == 1
+    assert p1.status == ProposalStatus.APPROVED
+    assert p2.status == ProposalStatus.PROPOSED  # cap reached
+    assert p3.status == ProposalStatus.PROPOSED  # kind not in policy
+
+    actor = sess.query(AuditEntry).filter_by(action="approve").one().actor
+    assert actor == "auto:policy"
+
+
+def test_failing_plugin_retries_then_fails(cfg, sess, monkeypatch):
+    p = _proposal(kind="affiliate", title="Will fail")
+    sess.add(p)
+    sess.flush()
+    executor.approve(p.id, "gustavo", cfg(), sess)
+
+    calls = {"n": 0}
+
+    def boom(proposal, cfg, sess):
+        calls["n"] += 1
+        raise RuntimeError("upstream exploded")
+
+    monkeypatch.setitem(plugins.REGISTRY, "affiliate", boom)
+    config = cfg(CAYCE_OPS_MAX_ATTEMPTS="2")
+
+    executor.run(config, sess)  # attempt 1 -> back to approved for retry
+    assert p.status == ProposalStatus.APPROVED
+    assert p.attempts == 1
+
+    executor.run(config, sess)  # attempt 2 -> failed for good
+    assert p.status == ProposalStatus.FAILED
+    assert p.attempts == 2
+    assert calls["n"] == 2
+
+    errors = sess.query(AuditEntry).filter_by(result="error").all()
+    assert any(a.action == "fail" for a in errors)
+
+
+def test_http_post_blocked_when_disabled(cfg, sess):
+    p = _proposal(kind="affiliate", title=" webhook target")
+    p.detail = {"webhook_url": "https://hooks.example.org/ingest"}
+    sess.add(p)
+    sess.flush()
+
+    config = cfg()  # CAYCE_OPS_ENABLE_HTTP_ACTIONS=false
+    with pytest.raises(PermissionError):
+        plugins.http_post(p, config, sess)
+
+
+def test_unknown_kind_falls_back_to_log_action(cfg, sess):
+    p = _proposal(kind="brand_new_kind", title="No plugin yet")
+    sess.add(p)
+    sess.flush()
+    executor.approve(p.id, "gustavo", cfg(), sess)
+
+    assert executor.run(cfg(), sess) == 0
+    assert p.status == ProposalStatus.DONE
+
+    done = sess.query(AuditEntry).filter_by(action="execute_done").one()
+    assert done.detail["outcome"]["plugin_fallback"] is True

tests/test_maild.py

@@ -0,0 +1,158 @@
+"""maild tests — parsing, classification, threading, gated send. Invented data only."""
+
+from __future__ import annotations
+
+from cayceops.components.maild import classify as clf
+from cayceops.components.maild import maild
+from cayceops.models import (
+    AuditEntry,
+    DraftStatus,
+    MailClassification,
+    MailDirection,
+    MailMessage,
+    Proposal,
+    ProposalStatus,
+)
+
+OUTBOUND_MSGID = "<cayce-draft-1-1000@cayce-co.xyz>"
+
+
+def _eml(subject: str, body: str, in_reply_to: str | None = None, msgid: str | None = None) -> bytes:
+    lines = [
+        "From: Partner Desk <partners@acme.example.org>",
+        "To: agent@cayce-co.xyz",
+        f"Subject: {subject}",
+        f"Message-ID: {msgid or '<acme-reply-1@acme.example.org>'}",
+    ]
+    if in_reply_to:
+        lines.append(f"In-Reply-To: {in_reply_to}")
+    lines += ["MIME-Version: 1.0", "Content-Type: text/plain; charset=utf-8", "", body]
+    return "\r\n".join(lines).encode()
+
+
+def _proposal_with_sent_draft(sess) -> Proposal:
+    p = Proposal(kind="partnership", title="Acme hotel affiliate program", status=ProposalStatus.APPROVED)
+    sess.add(p)
+    sess.flush()
+    sess.add(
+        MailMessage(
+            direction=MailDirection.OUTBOUND,
+            message_id=OUTBOUND_MSGID,
+            to_addr="partners@acme.example.org",
+            subject="[cayce] partnership inquiry",
+            proposal_id=p.id,
+            draft_status=DraftStatus.SENT,
+        )
+    )
+    sess.flush()
+    return p
+
+
+# --- classifier ---------------------------------------------------------------
+
+
+def test_rules_classification(cfg):
+    config = cfg()
+    assert clf.classify("Your application was approved, welcome to the program", config)[0] is MailClassification.APPROVAL
+    assert clf.classify("Unfortunately we cannot accept new partners", config)[0] is MailClassification.REJECTION
+    assert clf.classify("Could you provide your traffic numbers?", config)[0] is MailClassification.QUESTION
+    kind, confidence = clf.classify("grocery list", config)
+    assert kind is MailClassification.OTHER or kind is MailClassification.NEEDS_REVIEW
+
+
+def test_low_confidence_becomes_needs_review(cfg):
+    config = cfg(CAYCE_OPS_LLM_REVIEW_THRESHOLD="0.95")
+    kind, confidence = clf.classify("Your application was approved, welcome", config)
+    assert kind is MailClassification.NEEDS_REVIEW
+    assert confidence < 0.95
+
+
+# --- inbound pass ---------------------------------------------------------------
+
+
+def test_poll_threads_approval_and_drafts_reply(cfg, sess):
+    p = _proposal_with_sent_draft(sess)
+    raw = _eml("Re: [cayce] partnership inquiry", "Approved! Welcome to the Acme partner program.", in_reply_to=OUTBOUND_MSGID)
+
+    stats = maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
+    assert stats["processed"] == 1 and stats["drafted"] == 1
+
+    inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
+    assert inbound.classification is MailClassification.APPROVAL
+    assert inbound.proposal_id == p.id
+    assert inbound.draft_status is DraftStatus.DRAFTED
+    assert inbound.draft_reply and "approval" in inbound.draft_reply.lower()
+
+    assert p.detail["merchant_status"] == "approval"
+    assert p.status is ProposalStatus.APPROVED  # lifecycle untouched by maild
+
+
+def test_poll_idempotent_on_message_id(cfg, sess):
+    _proposal_with_sent_draft(sess)
+    raw = _eml("Re: hi", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
+    fetcher = lambda cfg: [raw]
+    assert maild._poll(cfg(), sess, fetcher=fetcher)["processed"] == 1
+    assert maild._poll(cfg(), sess, fetcher=fetcher)["skipped"] == 1
+
+
+def test_unthreaded_approval_needs_review(cfg, sess):
+    raw = _eml("Great news", "Approved! Welcome aboard.")  # no In-Reply-To
+    stats = maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
+    inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
+    assert inbound.classification is MailClassification.NEEDS_REVIEW
+    assert stats["needs_review"] == 1
+    assert inbound.draft_status is DraftStatus.NONE  # no draft for review items
+
+
+def test_question_gets_clarification_draft(cfg, sess):
+    _proposal_with_sent_draft(sess)
+    raw = _eml("Re: inquiry", "Could you provide your monthly traffic?", in_reply_to=OUTBOUND_MSGID)
+    maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
+    inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
+    assert inbound.classification is MailClassification.QUESTION
+    assert inbound.draft_status is DraftStatus.DRAFTED
+    assert "traffic" in inbound.draft_reply
+
+
+# --- human gate -------------------------------------------------------------------
+
+
+def test_send_blocked_without_approval(cfg, sess, monkeypatch):
+    _proposal_with_sent_draft(sess)
+    raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
+    maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
+
+    sent = []
+    config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
+    stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
+    assert stats["sent"] == 0 and sent == []
+    assert sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one().draft_status is DraftStatus.DRAFTED
+
+
+def test_send_requires_explicit_human_approval(cfg, sess):
+    _proposal_with_sent_draft(sess)
+    raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
+    maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
+    inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
+
+    assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
+
+    sent = []
+    config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
+    stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
+    assert stats["sent"] == 1
+    assert inbound.draft_status is DraftStatus.SENT
+    assert sent[0][1] == "partners@acme.example.org"
+
+    row = sess.query(AuditEntry).filter_by(action="send_draft").one()
+    assert "gustavo" in str(row.detail)
+
+
+def test_double_approve_is_rejected(cfg, sess):
+    _proposal_with_sent_draft(sess)
+    raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
+    maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
+    inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
+
+    assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
+    assert maild._approve_draft(inbound.id, "gustavo", sess) == 1

tests/test_scanner.py

@@ -0,0 +1,112 @@
+"""scanner tests — discovery, classification, dedupe. No network (injected fetch)."""
+
+from __future__ import annotations
+
+import json
+
+from cayceops.components.scanner import scanner
+from cayceops.models import Proposal, ProposalStatus
+
+RSS = b"""<?xml version="1.0"?>
+<rss><channel>
+  <item><title>Acme launches affiliate program for hotels</title>
+    <link>https://acme.example.org/affiliates</link>
+    <description>Earn commission on every booking.</description></item>
+  <item><title>Weekly crypto digest</title>
+    <link>https://news.example.org/digest</link>
+    <description>Everything unrelated.</description></item>
+</channel></rss>"""
+
+PAGE = b"""<html><head><title>Partners</title></head><body>
+  <a href="https://pay.example.org/partner">Become a payment partner</a>
+  <a href="https://pay.example.org/docs">API docs</a>
+</body></html>"""
+
+
+def _fake_fetch(responses):
+    def fetch(url, cfg, timeout=20.0):
+        return responses[url]
+
+    return fetch
+
+
+SOURCES = [
+    {"type": "rss", "url": "https://feeds.example.org/news.xml"},
+    {"type": "page", "url": "https://pay.example.org/partners"},
+]
+
+
+def _cfg(cfg, **kw):
+    return cfg(
+        CAYCE_OPS_SCANNER_SOURCES=json.dumps(SOURCES),
+        CAYCE_OPS_SCANNER_KEYWORDS="affiliate,partner,bitcoin,commission",
+        **kw,
+    )
+
+
+def test_scan_files_proposals_with_kind_and_score(cfg, sess):
+    config = _cfg(cfg)
+    stats = scanner.scan(config, sess, fetch=_fake_fetch({
+        "https://feeds.example.org/news.xml": RSS,
+        "https://pay.example.org/partners": PAGE,
+    }))
+
+    assert stats["proposed"] == 3  # affiliate item + both page links (pages keep all)
+    proposals = sess.query(Proposal).all()
+    by_title = {p.title: p for p in proposals}
+
+    affiliate = by_title["Acme launches affiliate program for hotels"]
+    assert affiliate.kind == "affiliate"
+    assert affiliate.status == ProposalStatus.PROPOSED
+    assert affiliate.risk_score > 0
+    assert affiliate.dedupe_key
+
+    partner = by_title["Become a payment partner"]
+    assert partner.kind == "partnership"
+
+
+def test_scan_is_idempotent_via_dedupe(cfg, sess):
+    config = _cfg(cfg)
+    scanner.scan(config, sess, fetch=_fake_fetch({
+        "https://feeds.example.org/news.xml": RSS,
+        "https://pay.example.org/partners": PAGE,
+    }))
+    stats2 = scanner.scan(config, sess, fetch=_fake_fetch({
+        "https://feeds.example.org/news.xml": RSS,
+        "https://pay.example.org/partners": PAGE,
+    }))
+    assert stats2["proposed"] == 0
+    assert stats2["skipped_existing"] == 3
+
+
+def test_scan_filters_keywordless_feed_items(cfg, sess):
+    config = _cfg(cfg)
+    scanner.scan(config, sess, fetch=_fake_fetch({
+        "https://feeds.example.org/news.xml": RSS,
+        "https://pay.example.org/partners": PAGE,
+    }))
+    titles = {p.title for p in sess.query(Proposal).all()}
+    assert "Weekly crypto digest" not in titles  # no keyword match in feed -> dropped
+
+
+def test_scan_records_audit_for_each_proposal(cfg, sess):
+    from cayceops.models import AuditEntry
+
+    config = _cfg(cfg)
+    scanner.scan(config, sess, fetch=_fake_fetch({
+        "https://feeds.example.org/news.xml": RSS,
+        "https://pay.example.org/partners": PAGE,
+    }))
+    acts = [a.action for a in sess.query(AuditEntry).all()]
+    assert acts.count("propose") == 3
+    assert all(a.actor == "daemon:scanner" for a in sess.query(AuditEntry).all())
+
+
+def test_scan_survives_source_error(cfg, sess):
+    def broken_fetch(url, cfg, timeout=20.0):
+        raise OSError("connection refused")
+
+    config = _cfg(cfg)
+    stats = scanner.scan(config, sess, fetch=broken_fetch)
+    assert stats["proposed"] == 0
+    assert len(stats["source_errors"]) == 2