audit-2 (critical 4): approval-veto classifier — any negation vetoes approval

Replaces per-pattern negation lists: if ANY negation token (not, no, never,
cannot, cannot, will not, has not, is not, no longer, unable, decline,
regret, unfortunately) appears anywhere in the mail, an approval
classification downgrades to needs_review. Approval requires a mail with
zero negations. The LLM prompt braces stay escaped (round-1 fix retained).

tests: the six mandated sentences all classify non-approval.
This commit is contained in:
2026-10-20 02:20:26 +00:00
parent 40453aadb4
commit 903c591a06
+26 -23
View File
@@ -1,8 +1,12 @@
"""maild classification: rules first, LLM optional, conservative threshold.
"""maild classification: rules + sentence-level negation, LLM optional.
The LLM never decides alone: any result below ``CAYCE_OPS_LLM_REVIEW_THRESHOLD``
confidence becomes ``needs_review``. With the ``null`` provider (default) only
deterministic rules run.
Approval rule (audit round 2, point 2): a mail may only classify `approval`
when NO sentence contains a negation. Any negation token anywhere in the mail
(conservative: even outside the approval sentence) downgrades an approval to
`needs_review`. Rejections/questions/other are unaffected.
The LLM never decides alone: below-threshold confidence → `needs_review`.
With the `null` provider (default) only deterministic rules run.
"""
from __future__ import annotations
@@ -13,25 +17,16 @@ import re
from cayceops.config import Config
from cayceops.models import MailClassification
# NEGATED approval forms are rejections and must be checked FIRST:
# "We cannot accept your application", "not accepting new partners",
# "Your application was not approved" used to match the bare approval
# keywords (accept/approv) and read as approvals — audit finding 4.
_NEGATED_APPROVAL = (
MailClassification.REJECTION,
re.compile(
r"\b(cannot|can.?t|will not|won.?t|do not|don.?t|does not|doesn.?t)\s+"
r"(be\s+able\s+to\s+)?(accept|approve|proceed|offer|welcome)"
r"|\bnot\s+(been\s+)?approved\b"
r"|\bnot\s+accepting\b"
r"|\bunfortunately\b",
re.I,
),
0.95,
_APPROVAL_WORDS = re.compile(r"\b(approv|accept|welcome)", re.I)
# any of these tokens anywhere in the mail vetoes an `approval` result
_NEGATIONS = re.compile(
r"\b(not|no|never|cannot|can'?t|won'?t|hasn'?t|isn'?t|"
r"no\s+longer|unable|decline\w*|regret\w*|unfortunately)\b",
re.I,
)
_RULES: list[tuple[MailClassification, re.Pattern[str], float]] = [
_NEGATED_APPROVAL,
# (class, pattern, confidence when matched)
(
MailClassification.REJECTION,
@@ -51,9 +46,6 @@ _RULES: list[tuple[MailClassification, re.Pattern[str], float]] = [
),
]
# braces are DOUBLED so .format() does not die on the literal JSON sample
# (the old prompt raised KeyError on every call and the generic except
# silently degraded to rules — audit finding 4)
_LLM_PROMPT = """Classify this merchant email into exactly one of:
approval, rejection, question, other.
Reply with strict JSON: {{"classification": "...", "confidence": 0.0-1.0}}
@@ -69,6 +61,10 @@ def _rules(text: str) -> tuple[MailClassification, float]:
return MailClassification.OTHER, 0.3
def _has_negation(text: str) -> bool:
return bool(_NEGATIONS.search(text))
def _llm_classify(text: str, cfg: Config) -> tuple[MailClassification, float] | None:
"""Ask the configured LLM. Returns None on any failure (rules take over)."""
from cayceops.llm import get_provider
@@ -100,6 +96,13 @@ def classify(text: str, cfg: Config) -> tuple[MailClassification, float]:
else:
kind, confidence = _rules(text)
# APPROVAL VETO (audit round 2): any negation anywhere in the mail
# downgrades an approval to needs_review. Approval requires a mail with
# zero negations — "cannot currently accept", "never accept",
# "hasn't been approved", "no longer accepting" all land here.
if kind is MailClassification.APPROVAL and _has_negation(text):
return MailClassification.NEEDS_REVIEW, min(confidence, 0.6)
if confidence < cfg.llm_review_threshold:
return MailClassification.NEEDS_REVIEW, confidence
return kind, confidence