cayce
90e6cfcc19
Critical 1 (approval forgery): audit_log is now hash-chained (prev_hash/entry_hash, verify_audit_chain); executor approve REQUIRES a BIP-340 signature over sha256(cayce-ops:approve:<id>) when CAYCE_OPS_OPERATOR_PUBKEY is set; deploy/roles.sql ships least-privilege per-daemon DB roles. Critical 2 (threading dead): sends create real OUTBOUND rows (own Message-ID, In-Reply-To -> inbound); inbound message_id never overwritten; maild compose adds the first-contact path; _match_proposal falls back to inbound chains. Critical 3 (duplicate sends): per-message transactions in send path — SMTP failure on message N no longer resurrects 1..N-1; failures audited (send_failed) and the row stays queued. Critical 4 (classifier): negated approvals (cannot accept / not approved / not accepting) classify REJECTION at 0.95 before the bare-keyword rule; LLM prompt braces escaped (.format no longer KeyError into silent fallback). Critical 5 (systemd): every daemon has a run subcommand (maild run = poll+send, publisher run = publish-due, monitor run = build); deploy README documents the separate daily publisher plan timer. Mediums: approved_hash binds drafts/captions to the approved text (edited text refuses to send/publish until re-approved); risk_score renamed match_score (migration 0002) and auto-approve orders HIGHEST relevance first; compose binds postgres to 127.0.0.1; IMAP marks \\Seen only after commit (fetch returns (num, raw), mark post-commit); disabled channels are terminal (ContentStatus.SKIPPED) and publish failures burn a bounded attempt budget before abandonment; make hooks installs the pre-push gate. tests: 12 new — 56 total, all green; ruff 0 findings; gitleaks clean.
352 lines
13 KiB
Python
352 lines
13 KiB
Python
"""maild tests — parsing, classification, threading, gated send. Invented data only."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
from cayceops.components.maild import classify as clf
|
|
from cayceops.components.maild import maild
|
|
from cayceops.models import (
|
|
AuditEntry,
|
|
DraftStatus,
|
|
MailClassification,
|
|
MailDirection,
|
|
MailMessage,
|
|
Proposal,
|
|
ProposalStatus,
|
|
)
|
|
|
|
OUTBOUND_MSGID = "<cayce-draft-1-1000@cayce-co.xyz>"
|
|
|
|
STUB_CFG = SimpleNamespace(mail_from="agent@cayce-co.xyz")
|
|
|
|
|
|
def _eml(subject, body, in_reply_to=None, msgid=None) -> bytes:
|
|
lines = [
|
|
"From: Partner Desk <partners@acme.example.org>",
|
|
"To: agent@cayce-co.xyz",
|
|
f"Subject: {subject}",
|
|
f"Message-ID: {msgid or '<acme-reply-1@acme.example.org>'}",
|
|
]
|
|
if in_reply_to:
|
|
lines.append(f"In-Reply-To: {in_reply_to}")
|
|
lines += ["MIME-Version: 1.0", "Content-Type: text/plain; charset=utf-8", "", body]
|
|
return "\r\n".join(lines).encode()
|
|
|
|
|
|
def _proposal_with_sent_draft(sess) -> Proposal:
|
|
"""A proposal whose first-contact mail went through the REAL path:
|
|
compose -> approve -> send (creating a true OUTBOUND row)."""
|
|
p = Proposal(
|
|
kind="partnership",
|
|
title="Acme hotel affiliate program",
|
|
status=ProposalStatus.APPROVED,
|
|
)
|
|
sess.add(p)
|
|
sess.flush()
|
|
mail_id = maild._compose(
|
|
p.id, "partners@acme.example.org", "[cayce] partnership inquiry", "Hello!", sess
|
|
)
|
|
assert maild._approve_draft(mail_id, "gustavo", sess) == 0
|
|
assert maild._send_one(STUB_CFG, sess, mail_id, sender=lambda *a, **k: None) == "sent"
|
|
out = (
|
|
sess.query(MailMessage)
|
|
.filter_by(direction=MailDirection.OUTBOUND, proposal_id=p.id)
|
|
.one()
|
|
)
|
|
out.message_id = OUTBOUND_MSGID # deterministic id for the fixtures
|
|
sess.flush()
|
|
return p
|
|
|
|
|
|
def _fetcher(raws: list[bytes]):
|
|
return lambda cfg: [(bytes(str(i), "ascii"), raw) for i, raw in enumerate(raws)]
|
|
|
|
|
|
# --- classifier ---------------------------------------------------------------
|
|
|
|
|
|
def test_rules_classification(cfg):
|
|
config = cfg()
|
|
assert (
|
|
clf.classify("Your application was approved, welcome to the program", config)[0]
|
|
is MailClassification.APPROVAL
|
|
)
|
|
assert (
|
|
clf.classify("Unfortunately we cannot accept new partners", config)[0]
|
|
is MailClassification.REJECTION
|
|
)
|
|
assert (
|
|
clf.classify("Could you provide your traffic numbers?", config)[0]
|
|
is MailClassification.QUESTION
|
|
)
|
|
kind, _ = clf.classify("grocery list", config)
|
|
assert kind is MailClassification.OTHER or kind is MailClassification.NEEDS_REVIEW
|
|
|
|
|
|
def test_negated_approvals_are_rejections(cfg):
|
|
"""Critical finding 4: 'cannot accept'/'not approved' read as approvals."""
|
|
config = cfg()
|
|
cases = [
|
|
"We cannot accept your application",
|
|
"We are not accepting new partners at this time",
|
|
"Your application was not approved",
|
|
"We can't approve this request",
|
|
"We do not accept external affiliates",
|
|
]
|
|
for text in cases:
|
|
kind, confidence = clf.classify(text, config)
|
|
assert kind is MailClassification.REJECTION, f"{text!r} -> {kind}"
|
|
assert confidence >= 0.9
|
|
|
|
|
|
def test_llm_prompt_format_survives_json_braces():
|
|
"""Critical finding 4: .format() raised KeyError on the JSON sample."""
|
|
formatted = clf._LLM_PROMPT.format(body="x")
|
|
assert '{"classification"' in formatted # literal braces survive
|
|
|
|
|
|
def test_low_confidence_becomes_needs_review(cfg):
|
|
config = cfg(CAYCE_OPS_LLM_REVIEW_THRESHOLD="0.95")
|
|
kind, confidence = clf.classify("Your application was approved, welcome", config)
|
|
assert kind is MailClassification.NEEDS_REVIEW
|
|
assert confidence < 0.95
|
|
|
|
|
|
# --- inbound pass ---------------------------------------------------------------
|
|
|
|
|
|
def test_poll_threads_approval_and_drafts_reply(cfg, sess):
|
|
p = _proposal_with_sent_draft(sess)
|
|
raw = _eml(
|
|
"Re: [cayce] partnership inquiry",
|
|
"Approved! Welcome to the Acme partner program.",
|
|
in_reply_to=OUTBOUND_MSGID,
|
|
)
|
|
|
|
stats, nums = maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
assert stats["processed"] == 1 and stats["drafted"] == 1
|
|
assert len(nums) == 1
|
|
|
|
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
assert inbound.classification is MailClassification.APPROVAL
|
|
assert inbound.proposal_id == p.id
|
|
assert inbound.draft_status is DraftStatus.DRAFTED
|
|
assert inbound.draft_reply and "approval" in inbound.draft_reply.lower()
|
|
|
|
assert p.detail["merchant_status"] == "approval"
|
|
assert p.status is ProposalStatus.APPROVED # lifecycle untouched by maild
|
|
|
|
|
|
def test_poll_idempotent_on_message_id(cfg, sess):
|
|
_proposal_with_sent_draft(sess)
|
|
raw = _eml("Re: hi", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
|
|
|
|
first, second = maild._poll(cfg(), sess, fetcher=_fetcher([raw])), maild._poll(
|
|
cfg(), sess, fetcher=_fetcher([raw])
|
|
)
|
|
assert first[0]["processed"] == 1
|
|
assert second[0]["skipped"] == 1
|
|
|
|
|
|
def test_unthreaded_approval_needs_review(cfg, sess):
|
|
raw = _eml("Great news", "Approved! Welcome aboard.") # no In-Reply-To
|
|
stats, _ = maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
assert inbound.classification is MailClassification.NEEDS_REVIEW
|
|
assert stats["needs_review"] == 1
|
|
assert inbound.draft_status is DraftStatus.NONE # no draft for review items
|
|
|
|
|
|
def test_question_gets_clarification_draft(cfg, sess):
|
|
_proposal_with_sent_draft(sess)
|
|
raw = _eml("Re: inquiry", "Could you provide your monthly traffic?", in_reply_to=OUTBOUND_MSGID)
|
|
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
assert inbound.classification is MailClassification.QUESTION
|
|
assert inbound.draft_status is DraftStatus.DRAFTED
|
|
assert "traffic" in inbound.draft_reply
|
|
|
|
|
|
# --- human gate -------------------------------------------------------------------
|
|
|
|
|
|
def test_send_blocked_without_approval(cfg, sess):
|
|
_proposal_with_sent_draft(sess)
|
|
raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
|
|
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
|
|
sent = []
|
|
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
|
|
stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
|
|
assert stats["sent"] == 0 and sent == []
|
|
assert stats["blocked"] + stats["missing"] >= 0 # no crash paths
|
|
assert config.mail_from == "agent@cayce-co.xyz"
|
|
row = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
assert row.draft_status is DraftStatus.DRAFTED
|
|
|
|
|
|
def test_send_requires_explicit_human_approval(cfg, sess):
|
|
_proposal_with_sent_draft(sess)
|
|
raw = _eml(
|
|
"Re: inquiry",
|
|
"Approved, welcome to the program.",
|
|
in_reply_to=OUTBOUND_MSGID,
|
|
)
|
|
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
original_id = inbound.message_id
|
|
|
|
assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
|
|
|
|
sent = []
|
|
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
|
|
|
|
def sender(cfg, to, subject, body, message_id, in_reply_to=None):
|
|
sent.append((to, subject, body, message_id, in_reply_to))
|
|
|
|
stats = maild._send_approved(config, sess, sender=sender)
|
|
assert stats["sent"] == 1
|
|
assert inbound.draft_status is DraftStatus.SENT
|
|
to, subject, body, message_id, in_reply_to = sent[0]
|
|
assert to == "partners@acme.example.org"
|
|
assert in_reply_to == original_id # In-Reply-To threads to the inbound mail
|
|
|
|
# a real OUTBOUND row now exists (audit fix: they were never created)
|
|
out = (
|
|
sess.query(MailMessage)
|
|
.filter_by(direction=MailDirection.OUTBOUND, message_id=message_id)
|
|
.one()
|
|
)
|
|
assert out.in_reply_to == original_id
|
|
assert out.proposal_id == inbound.proposal_id
|
|
assert inbound.message_id == original_id # inbound id NOT overwritten
|
|
|
|
row = (
|
|
sess.query(AuditEntry)
|
|
.filter_by(action="send_draft")
|
|
.order_by(AuditEntry.id.desc())
|
|
.first()
|
|
)
|
|
assert row is not None and "gustavo" in str(row.detail)
|
|
|
|
|
|
def test_double_approve_is_rejected(cfg, sess):
|
|
_proposal_with_sent_draft(sess)
|
|
raw = _eml("Re: inquiry", "Approved, welcome.", in_reply_to=OUTBOUND_MSGID)
|
|
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
|
|
assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
|
|
assert maild._approve_draft(inbound.id, "gustavo", sess) == 1
|
|
|
|
|
|
# --- audit corrections ----------------------------------------------------------
|
|
|
|
|
|
def test_send_failure_does_not_resurrect_sent_mails(cfg, sess):
|
|
"""Critical finding 3: SMTP failure mid-batch rolled back the whole
|
|
transaction, re-sending already-delivered mails 5 minutes later."""
|
|
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
|
|
for i in range(3):
|
|
# unique first-contact thread per proposal (real compose -> send path)
|
|
p = Proposal(kind="partnership", title=f"Acme partner {i}")
|
|
sess.add(p)
|
|
sess.flush()
|
|
mail_id = maild._compose(
|
|
p.id, f"desk{i}@acme.example.org", f"[cayce] inquiry {i}", "Hello!", sess
|
|
)
|
|
assert maild._approve_draft(mail_id, "gustavo", sess) == 0
|
|
assert maild._send_one(STUB_CFG, sess, mail_id, sender=lambda *a, **k: None) == "sent"
|
|
sess.flush()
|
|
out = sess.get(MailMessage, mail_id)
|
|
raw = _eml(
|
|
f"Re: {i}",
|
|
"Approved, welcome to the program.",
|
|
in_reply_to=out.message_id,
|
|
msgid=f"<acme-{i}@acme.example.org>",
|
|
)
|
|
maild._poll(config, sess, fetcher=_fetcher([raw]))
|
|
inbound = (
|
|
sess.query(MailMessage)
|
|
.filter_by(direction=MailDirection.INBOUND, message_id=f"<acme-{i}@acme.example.org>")
|
|
.one()
|
|
)
|
|
maild._approve_draft(inbound.id, "gustavo", sess)
|
|
sess.commit() # per-message factory sessions only see committed rows
|
|
|
|
calls = {"n": 0}
|
|
|
|
def flaky_smtp(cfg, to, subject, body, message_id, in_reply_to=None):
|
|
calls["n"] += 1
|
|
if calls["n"] == 3:
|
|
raise OSError("smtp connection lost")
|
|
|
|
import cayceops.db as appdb
|
|
|
|
stats = maild._send_approved(
|
|
config, sess, sender=flaky_smtp, factory=appdb.session_factory()
|
|
)
|
|
|
|
assert calls["n"] == 3
|
|
assert stats["sent"] == 2 and stats["error"] == 1 # no rollback of 1..2
|
|
still_approved = (
|
|
sess.query(MailMessage)
|
|
.filter_by(direction=MailDirection.INBOUND, draft_status=DraftStatus.APPROVED)
|
|
.count()
|
|
)
|
|
assert still_approved == 1 # only the failed one stays queued
|
|
|
|
|
|
def test_edited_draft_refuses_to_send(cfg, sess):
|
|
"""Medium finding: approved text was not bound to its content."""
|
|
_proposal_with_sent_draft(sess)
|
|
raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
|
|
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
|
|
assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
|
|
|
|
inbound.draft_reply = "TOTALLY EDITED TEXT after approval" # tamper
|
|
|
|
sent = []
|
|
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
|
|
stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
|
|
assert stats["sent"] == 0 and stats["hash_mismatch"] == 1
|
|
assert sent == []
|
|
assert inbound.draft_status is DraftStatus.DRAFTED # needs fresh approval
|
|
action = sess.query(AuditEntry).filter_by(action="send_refused_hash_mismatch").one()
|
|
assert action.result == "error"
|
|
|
|
|
|
def test_compose_creates_first_contact_draft(cfg, sess):
|
|
"""Critical finding 2: there was no path for the first contact mail."""
|
|
p = Proposal(kind="partnership", title="New hotel partner")
|
|
sess.add(p)
|
|
sess.flush()
|
|
|
|
mail_id = maild._compose(
|
|
p.id, "desk@hotel.example.org", "Partnership", "We would like to...", sess
|
|
)
|
|
assert mail_id > 0
|
|
out = sess.get(MailMessage, mail_id)
|
|
assert out.direction is MailDirection.OUTBOUND
|
|
assert out.draft_status is DraftStatus.DRAFTED
|
|
assert out.proposal_id == p.id
|
|
|
|
# approve + send -> an inbound reply can now thread to this proposal
|
|
assert maild._approve_draft(mail_id, "gustavo", sess) == 0
|
|
stats = maild._send_approved(STUB_CFG, sess, sender=lambda *a, **k: None)
|
|
assert stats["sent"] == 1
|
|
sess.flush()
|
|
|
|
reply = _eml("Re: Partnership", "Approved, welcome!", in_reply_to=out.message_id)
|
|
assert maild._match_proposal(maild.parse_raw(reply), sess) is p
|
|
|
|
|
|
def test_mark_seen_returned_for_post_commit_processing(cfg, sess):
|
|
"""Medium finding: \\\\Seen was set before processing (crash lost mail)."""
|
|
raw = _eml("Re: hi", "Approved, welcome to the program.")
|
|
stats, nums = maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
|
|
assert stats["processed"] == 1
|
|
assert nums == [b"0"] # caller marks seen only after commit
|