From 903c591a066c4870196ba9f6b6f111464dd1813c Mon Sep 17 00:00:00 2001 From: Cayce Date: Tue, 20 Oct 2026 02:20:26 +0000 Subject: [PATCH] =?UTF-8?q?audit-2=20(critical=204):=20approval-veto=20cla?= =?UTF-8?q?ssifier=20=E2=80=94=20any=20negation=20vetoes=20approval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces per-pattern negation lists: if ANY negation token (not, no, never, cannot, cannot, will not, has not, is not, no longer, unable, decline, regret, unfortunately) appears anywhere in the mail, an approval classification downgrades to needs_review. Approval requires a mail with zero negations. The LLM prompt braces stay escaped (round-1 fix retained). tests: the six mandated sentences all classify non-approval. --- cayceops/components/maild/classify.py | 49 ++++++++++++++------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/cayceops/components/maild/classify.py b/cayceops/components/maild/classify.py index e2eda54..3cfa658 100644 --- a/cayceops/components/maild/classify.py +++ b/cayceops/components/maild/classify.py @@ -1,8 +1,12 @@ -"""maild classification: rules first, LLM optional, conservative threshold. +"""maild classification: rules + sentence-level negation, LLM optional. -The LLM never decides alone: any result below ``CAYCE_OPS_LLM_REVIEW_THRESHOLD`` -confidence becomes ``needs_review``. With the ``null`` provider (default) only -deterministic rules run. +Approval rule (audit round 2, point 2): a mail may only classify `approval` +when NO sentence contains a negation. Any negation token anywhere in the mail +(conservative: even outside the approval sentence) downgrades an approval to +`needs_review`. Rejections/questions/other are unaffected. + +The LLM never decides alone: below-threshold confidence → `needs_review`. +With the `null` provider (default) only deterministic rules run. """ from __future__ import annotations @@ -13,25 +17,16 @@ import re from cayceops.config import Config from cayceops.models import MailClassification -# NEGATED approval forms are rejections and must be checked FIRST: -# "We cannot accept your application", "not accepting new partners", -# "Your application was not approved" used to match the bare approval -# keywords (accept/approv) and read as approvals — audit finding 4. -_NEGATED_APPROVAL = ( - MailClassification.REJECTION, - re.compile( - r"\b(cannot|can.?t|will not|won.?t|do not|don.?t|does not|doesn.?t)\s+" - r"(be\s+able\s+to\s+)?(accept|approve|proceed|offer|welcome)" - r"|\bnot\s+(been\s+)?approved\b" - r"|\bnot\s+accepting\b" - r"|\bunfortunately\b", - re.I, - ), - 0.95, +_APPROVAL_WORDS = re.compile(r"\b(approv|accept|welcome)", re.I) + +# any of these tokens anywhere in the mail vetoes an `approval` result +_NEGATIONS = re.compile( + r"\b(not|no|never|cannot|can'?t|won'?t|hasn'?t|isn'?t|" + r"no\s+longer|unable|decline\w*|regret\w*|unfortunately)\b", + re.I, ) _RULES: list[tuple[MailClassification, re.Pattern[str], float]] = [ - _NEGATED_APPROVAL, # (class, pattern, confidence when matched) ( MailClassification.REJECTION, @@ -51,9 +46,6 @@ _RULES: list[tuple[MailClassification, re.Pattern[str], float]] = [ ), ] -# braces are DOUBLED so .format() does not die on the literal JSON sample -# (the old prompt raised KeyError on every call and the generic except -# silently degraded to rules — audit finding 4) _LLM_PROMPT = """Classify this merchant email into exactly one of: approval, rejection, question, other. Reply with strict JSON: {{"classification": "...", "confidence": 0.0-1.0}} @@ -69,6 +61,10 @@ def _rules(text: str) -> tuple[MailClassification, float]: return MailClassification.OTHER, 0.3 +def _has_negation(text: str) -> bool: + return bool(_NEGATIONS.search(text)) + + def _llm_classify(text: str, cfg: Config) -> tuple[MailClassification, float] | None: """Ask the configured LLM. Returns None on any failure (rules take over).""" from cayceops.llm import get_provider @@ -100,6 +96,13 @@ def classify(text: str, cfg: Config) -> tuple[MailClassification, float]: else: kind, confidence = _rules(text) + # APPROVAL VETO (audit round 2): any negation anywhere in the mail + # downgrades an approval to needs_review. Approval requires a mail with + # zero negations — "cannot currently accept", "never accept", + # "hasn't been approved", "no longer accepting" all land here. + if kind is MailClassification.APPROVAL and _has_negation(text): + return MailClassification.NEEDS_REVIEW, min(confidence, 0.6) + if confidence < cfg.llm_review_threshold: return MailClassification.NEEDS_REVIEW, confidence return kind, confidence