cayce
903c591a06
Replaces per-pattern negation lists: if ANY negation token (not, no, never, cannot, cannot, will not, has not, is not, no longer, unable, decline, regret, unfortunately) appears anywhere in the mail, an approval classification downgrades to needs_review. Approval requires a mail with zero negations. The LLM prompt braces stay escaped (round-1 fix retained). tests: the six mandated sentences all classify non-approval.
109 lines
4.0 KiB
Python
109 lines
4.0 KiB
Python
"""maild classification: rules + sentence-level negation, LLM optional.
|
|
|
|
Approval rule (audit round 2, point 2): a mail may only classify `approval`
|
|
when NO sentence contains a negation. Any negation token anywhere in the mail
|
|
(conservative: even outside the approval sentence) downgrades an approval to
|
|
`needs_review`. Rejections/questions/other are unaffected.
|
|
|
|
The LLM never decides alone: below-threshold confidence → `needs_review`.
|
|
With the `null` provider (default) only deterministic rules run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
|
|
from cayceops.config import Config
|
|
from cayceops.models import MailClassification
|
|
|
|
_APPROVAL_WORDS = re.compile(r"\b(approv|accept|welcome)", re.I)
|
|
|
|
# any of these tokens anywhere in the mail vetoes an `approval` result
|
|
_NEGATIONS = re.compile(
|
|
r"\b(not|no|never|cannot|can'?t|won'?t|hasn'?t|isn'?t|"
|
|
r"no\s+longer|unable|decline\w*|regret\w*|unfortunately)\b",
|
|
re.I,
|
|
)
|
|
|
|
_RULES: list[tuple[MailClassification, re.Pattern[str], float]] = [
|
|
# (class, pattern, confidence when matched)
|
|
(
|
|
MailClassification.REJECTION,
|
|
# noqa: E501 (kept on one line: the pattern IS the reviewed artifact)
|
|
re.compile(r"\b(declin|reject|not able to|unable to|we regret|won.t be able)\w*", re.I), # noqa: E501
|
|
0.9,
|
|
),
|
|
(
|
|
MailClassification.APPROVAL,
|
|
re.compile(r"\b(approv|accept|welcome to the|you.re (in|approved)|live|activated|glad to)\w*", re.I), # noqa: E501
|
|
0.85,
|
|
),
|
|
(
|
|
MailClassification.QUESTION,
|
|
re.compile(r"\b(could you|can you|please (provide|clarify|confirm)|what (is|are)|how (do|does|can)|need more (info|details))\b", re.I), # noqa: E501
|
|
0.7,
|
|
),
|
|
]
|
|
|
|
_LLM_PROMPT = """Classify this merchant email into exactly one of:
|
|
approval, rejection, question, other.
|
|
Reply with strict JSON: {{"classification": "...", "confidence": 0.0-1.0}}
|
|
|
|
Subject and body:
|
|
{body}"""
|
|
|
|
|
|
def _rules(text: str) -> tuple[MailClassification, float]:
|
|
for kind, pattern, confidence in _RULES:
|
|
if pattern.search(text):
|
|
return kind, confidence
|
|
return MailClassification.OTHER, 0.3
|
|
|
|
|
|
def _has_negation(text: str) -> bool:
|
|
return bool(_NEGATIONS.search(text))
|
|
|
|
|
|
def _llm_classify(text: str, cfg: Config) -> tuple[MailClassification, float] | None:
|
|
"""Ask the configured LLM. Returns None on any failure (rules take over)."""
|
|
from cayceops.llm import get_provider
|
|
|
|
try:
|
|
provider = get_provider(cfg)
|
|
raw = provider.complete(
|
|
_LLM_PROMPT.format(body=text[:4000]),
|
|
system="You classify merchant emails. Reply with JSON only.",
|
|
)
|
|
data = json.loads(re.search(r"\{.*\}", raw, re.S).group(0))
|
|
kind = MailClassification(data["classification"].lower())
|
|
confidence = float(data["confidence"])
|
|
except Exception: # noqa: BLE001 — LLM is best-effort by design
|
|
return None
|
|
if kind not in MailClassification or kind is MailClassification.NEEDS_REVIEW:
|
|
return None
|
|
return kind, min(max(confidence, 0.0), 1.0)
|
|
|
|
|
|
def classify(text: str, cfg: Config) -> tuple[MailClassification, float]:
|
|
"""Classify merchant text. Degrades to rules when no LLM is configured."""
|
|
if cfg.llm_provider != "null":
|
|
llm_result = _llm_classify(text, cfg)
|
|
if llm_result is not None:
|
|
kind, confidence = llm_result
|
|
else: # LLM failed — fall through to rules
|
|
kind, confidence = _rules(text)
|
|
else:
|
|
kind, confidence = _rules(text)
|
|
|
|
# APPROVAL VETO (audit round 2): any negation anywhere in the mail
|
|
# downgrades an approval to needs_review. Approval requires a mail with
|
|
# zero negations — "cannot currently accept", "never accept",
|
|
# "hasn't been approved", "no longer accepting" all land here.
|
|
if kind is MailClassification.APPROVAL and _has_negation(text):
|
|
return MailClassification.NEEDS_REVIEW, min(confidence, 0.6)
|
|
|
|
if confidence < cfg.llm_review_threshold:
|
|
return MailClassification.NEEDS_REVIEW, confidence
|
|
return kind, confidence
|