cayce/cayce-ops
1
0
Fork 0
Code Issues 1 Pull Requests 27 Actions Packages Projects Releases 2 Wiki Activity

fix: address external audit — criticals 1-5 + mediums

Browse Source 
Critical 1 (approval forgery): audit_log is now hash-chained
(prev_hash/entry_hash, verify_audit_chain); executor approve REQUIRES a
BIP-340 signature over sha256(cayce-ops:approve:<id>) when
CAYCE_OPS_OPERATOR_PUBKEY is set; deploy/roles.sql ships least-privilege
per-daemon DB roles.

Critical 2 (threading dead): sends create real OUTBOUND rows (own
Message-ID, In-Reply-To -> inbound); inbound message_id never overwritten;
maild compose adds the first-contact path; _match_proposal falls back to
inbound chains.

Critical 3 (duplicate sends): per-message transactions in send path —
SMTP failure on message N no longer resurrects 1..N-1; failures audited
(send_failed) and the row stays queued.

Critical 4 (classifier): negated approvals (cannot accept / not approved /
not accepting) classify REJECTION at 0.95 before the bare-keyword rule;
LLM prompt braces escaped (.format no longer KeyError into silent fallback).

Critical 5 (systemd): every daemon has a run subcommand
(maild run = poll+send, publisher run = publish-due, monitor run = build);
deploy README documents the separate daily publisher plan timer.

Mediums: approved_hash binds drafts/captions to the approved text (edited
text refuses to send/publish until re-approved); risk_score renamed
match_score (migration 0002) and auto-approve orders HIGHEST relevance
first; compose binds postgres to 127.0.0.1; IMAP marks \\Seen only after
commit (fetch returns (num, raw), mark post-commit); disabled channels are
terminal (ContentStatus.SKIPPED) and publish failures burn a bounded
attempt budget before abandonment; make hooks installs the pre-push gate.

tests: 12 new — 56 total, all green; ruff 0 findings; gitleaks clean.
This commit is contained in:
cayce
2026-10-19 23:06:29 +00:00
parent 2da5f3ff24
commit 90e6cfcc19
22 changed files with 1012 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_maild.py
+207 -38
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from types import SimpleNamespace
from cayceops.components.maild import classify as clf
from cayceops.components.maild import maild
from cayceops.models import (
@@ -16,13 +18,10 @@ from cayceops.models import (
OUTBOUND_MSGID = "<cayce-draft-1-1000@cayce-co.xyz>"
STUB_CFG = SimpleNamespace(mail_from="agent@cayce-co.xyz")
def _eml(
subject: str,
body: str,
in_reply_to: str | None = None,
msgid: str | None = None,
) -> bytes:
def _eml(subject, body, in_reply_to=None, msgid=None) -> bytes:
lines = [
"From: Partner Desk <partners@acme.example.org>",
"To: agent@cayce-co.xyz",
@@ -36,6 +35,8 @@ def _eml(
def _proposal_with_sent_draft(sess) -> Proposal:
"""A proposal whose first-contact mail went through the REAL path:
compose -> approve -> send (creating a true OUTBOUND row)."""
p = Proposal(
kind="partnership",
title="Acme hotel affiliate program",
@@ -43,35 +44,68 @@ def _proposal_with_sent_draft(sess) -> Proposal:
)
sess.add(p)
sess.flush()
sess.add(
MailMessage(
direction=MailDirection.OUTBOUND,
message_id=OUTBOUND_MSGID,
to_addr="partners@acme.example.org",
subject="[cayce] partnership inquiry",
proposal_id=p.id,
draft_status=DraftStatus.SENT,
)
mail_id = maild._compose(
p.id, "partners@acme.example.org", "[cayce] partnership inquiry", "Hello!", sess
)
assert maild._approve_draft(mail_id, "gustavo", sess) == 0
assert maild._send_one(STUB_CFG, sess, mail_id, sender=lambda *a, **k: None) == "sent"
out = (
sess.query(MailMessage)
.filter_by(direction=MailDirection.OUTBOUND, proposal_id=p.id)
.one()
)
out.message_id = OUTBOUND_MSGID # deterministic id for the fixtures
sess.flush()
return p
def _fetcher(raws: list[bytes]):
return lambda cfg: [(bytes(str(i), "ascii"), raw) for i, raw in enumerate(raws)]
# --- classifier ---------------------------------------------------------------
def test_rules_classification(cfg):
config = cfg()
ok = clf.classify("Your application was approved, welcome to the program", config)
no = clf.classify("Unfortunately we cannot accept new partners", config)
assert ok[0] is MailClassification.APPROVAL
assert no[0] is MailClassification.REJECTION
q = clf.classify("Could you provide your traffic numbers?", config)
assert q[0] is MailClassification.QUESTION
kind, confidence = clf.classify("grocery list", config)
assert (
clf.classify("Your application was approved, welcome to the program", config)[0]
is MailClassification.APPROVAL
)
assert (
clf.classify("Unfortunately we cannot accept new partners", config)[0]
is MailClassification.REJECTION
)
assert (
clf.classify("Could you provide your traffic numbers?", config)[0]
is MailClassification.QUESTION
)
kind, _ = clf.classify("grocery list", config)
assert kind is MailClassification.OTHER or kind is MailClassification.NEEDS_REVIEW
def test_negated_approvals_are_rejections(cfg):
"""Critical finding 4: 'cannot accept'/'not approved' read as approvals."""
config = cfg()
cases = [
"We cannot accept your application",
"We are not accepting new partners at this time",
"Your application was not approved",
"We can't approve this request",
"We do not accept external affiliates",
]
for text in cases:
kind, confidence = clf.classify(text, config)
assert kind is MailClassification.REJECTION, f"{text!r} -> {kind}"
assert confidence >= 0.9
def test_llm_prompt_format_survives_json_braces():
"""Critical finding 4: .format() raised KeyError on the JSON sample."""
formatted = clf._LLM_PROMPT.format(body="x")
assert '{"classification"' in formatted # literal braces survive
def test_low_confidence_becomes_needs_review(cfg):
config = cfg(CAYCE_OPS_LLM_REVIEW_THRESHOLD="0.95")
kind, confidence = clf.classify("Your application was approved, welcome", config)
@@ -90,8 +124,9 @@ def test_poll_threads_approval_and_drafts_reply(cfg, sess):
in_reply_to=OUTBOUND_MSGID,
)
stats = maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
stats, nums = maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
assert stats["processed"] == 1 and stats["drafted"] == 1
assert len(nums) == 1
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
assert inbound.classification is MailClassification.APPROVAL
@@ -107,16 +142,16 @@ def test_poll_idempotent_on_message_id(cfg, sess):
_proposal_with_sent_draft(sess)
raw = _eml("Re: hi", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
def fetcher(cfg):
return [raw]
assert maild._poll(cfg(), sess, fetcher=fetcher)["processed"] == 1
assert maild._poll(cfg(), sess, fetcher=fetcher)["skipped"] == 1 # noqa: E501
first, second = maild._poll(cfg(), sess, fetcher=_fetcher([raw])), maild._poll(
cfg(), sess, fetcher=_fetcher([raw])
)
assert first[0]["processed"] == 1
assert second[0]["skipped"] == 1
def test_unthreaded_approval_needs_review(cfg, sess):
raw = _eml("Great news", "Approved! Welcome aboard.") # no In-Reply-To
stats = maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
stats, _ = maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
assert inbound.classification is MailClassification.NEEDS_REVIEW
assert stats["needs_review"] == 1
@@ -126,7 +161,7 @@ def test_unthreaded_approval_needs_review(cfg, sess):
def test_question_gets_clarification_draft(cfg, sess):
_proposal_with_sent_draft(sess)
raw = _eml("Re: inquiry", "Could you provide your monthly traffic?", in_reply_to=OUTBOUND_MSGID)
maild._poll(cfg(), sess, fetcher=lambda cfg: [raw])
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
assert inbound.classification is MailClassification.QUESTION
assert inbound.draft_status is DraftStatus.DRAFTED
@@ -139,12 +174,14 @@ def test_question_gets_clarification_draft(cfg, sess):
def test_send_blocked_without_approval(cfg, sess):
_proposal_with_sent_draft(sess)
raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
maild._poll(cfg(), sess, fetcher=lambda cfg: [raw]) # noqa: E731 (test stub)
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
sent = []
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
assert stats["sent"] == 0 and sent == []
assert stats["blocked"] + stats["missing"] >= 0 # no crash paths
assert config.mail_from == "agent@cayce-co.xyz"
row = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
assert row.draft_status is DraftStatus.DRAFTED
@@ -156,27 +193,159 @@ def test_send_requires_explicit_human_approval(cfg, sess):
"Approved, welcome to the program.",
in_reply_to=OUTBOUND_MSGID,
)
maild._poll(cfg(), sess, fetcher=lambda cfg: [raw]) # noqa: E731 (test stub)
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
original_id = inbound.message_id
assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
sent = []
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
def sender(cfg, to, subject, body, message_id, in_reply_to=None):
sent.append((to, subject, body, message_id, in_reply_to))
stats = maild._send_approved(config, sess, sender=sender)
assert stats["sent"] == 1
assert inbound.draft_status is DraftStatus.SENT
assert sent[0][1] == "partners@acme.example.org"
to, subject, body, message_id, in_reply_to = sent[0]
assert to == "partners@acme.example.org"
assert in_reply_to == original_id # In-Reply-To threads to the inbound mail
row = sess.query(AuditEntry).filter_by(action="send_draft").one()
assert "gustavo" in str(row.detail)
# a real OUTBOUND row now exists (audit fix: they were never created)
out = (
sess.query(MailMessage)
.filter_by(direction=MailDirection.OUTBOUND, message_id=message_id)
.one()
)
assert out.in_reply_to == original_id
assert out.proposal_id == inbound.proposal_id
assert inbound.message_id == original_id # inbound id NOT overwritten
row = (
sess.query(AuditEntry)
.filter_by(action="send_draft")
.order_by(AuditEntry.id.desc())
.first()
)
assert row is not None and "gustavo" in str(row.detail)
def test_double_approve_is_rejected(cfg, sess):
_proposal_with_sent_draft(sess)
raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
maild._poll(cfg(), sess, fetcher=lambda cfg: [raw]) # noqa: E731 (test stub)
raw = _eml("Re: inquiry", "Approved, welcome.", in_reply_to=OUTBOUND_MSGID)
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
assert maild._approve_draft(inbound.id, "gustavo", sess) == 1
# --- audit corrections ----------------------------------------------------------
def test_send_failure_does_not_resurrect_sent_mails(cfg, sess):
"""Critical finding 3: SMTP failure mid-batch rolled back the whole
transaction, re-sending already-delivered mails 5 minutes later."""
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
for i in range(3):
# unique first-contact thread per proposal (real compose -> send path)
p = Proposal(kind="partnership", title=f"Acme partner {i}")
sess.add(p)
sess.flush()
mail_id = maild._compose(
p.id, f"desk{i}@acme.example.org", f"[cayce] inquiry {i}", "Hello!", sess
)
assert maild._approve_draft(mail_id, "gustavo", sess) == 0
assert maild._send_one(STUB_CFG, sess, mail_id, sender=lambda *a, **k: None) == "sent"
sess.flush()
out = sess.get(MailMessage, mail_id)
raw = _eml(
f"Re: {i}",
"Approved, welcome to the program.",
in_reply_to=out.message_id,
msgid=f"<acme-{i}@acme.example.org>",
)
maild._poll(config, sess, fetcher=_fetcher([raw]))
inbound = (
sess.query(MailMessage)
.filter_by(direction=MailDirection.INBOUND, message_id=f"<acme-{i}@acme.example.org>")
.one()
)
maild._approve_draft(inbound.id, "gustavo", sess)
sess.commit() # per-message factory sessions only see committed rows
calls = {"n": 0}
def flaky_smtp(cfg, to, subject, body, message_id, in_reply_to=None):
calls["n"] += 1
if calls["n"] == 3:
raise OSError("smtp connection lost")
import cayceops.db as appdb
stats = maild._send_approved(
config, sess, sender=flaky_smtp, factory=appdb.session_factory()
)
assert calls["n"] == 3
assert stats["sent"] == 2 and stats["error"] == 1 # no rollback of 1..2
still_approved = (
sess.query(MailMessage)
.filter_by(direction=MailDirection.INBOUND, draft_status=DraftStatus.APPROVED)
.count()
)
assert still_approved == 1 # only the failed one stays queued
def test_edited_draft_refuses_to_send(cfg, sess):
"""Medium finding: approved text was not bound to its content."""
_proposal_with_sent_draft(sess)
raw = _eml("Re: inquiry", "Approved, welcome to the program.", in_reply_to=OUTBOUND_MSGID)
maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
inbound = sess.query(MailMessage).filter_by(direction=MailDirection.INBOUND).one()
assert maild._approve_draft(inbound.id, "gustavo", sess) == 0
inbound.draft_reply = "TOTALLY EDITED TEXT after approval" # tamper
sent = []
config = cfg(CAYCE_OPS_MAIL_FROM="agent@cayce-co.xyz")
stats = maild._send_approved(config, sess, sender=lambda *a, **k: sent.append(a))
assert stats["sent"] == 0 and stats["hash_mismatch"] == 1
assert sent == []
assert inbound.draft_status is DraftStatus.DRAFTED # needs fresh approval
action = sess.query(AuditEntry).filter_by(action="send_refused_hash_mismatch").one()
assert action.result == "error"
def test_compose_creates_first_contact_draft(cfg, sess):
"""Critical finding 2: there was no path for the first contact mail."""
p = Proposal(kind="partnership", title="New hotel partner")
sess.add(p)
sess.flush()
mail_id = maild._compose(
p.id, "desk@hotel.example.org", "Partnership", "We would like to...", sess
)
assert mail_id > 0
out = sess.get(MailMessage, mail_id)
assert out.direction is MailDirection.OUTBOUND
assert out.draft_status is DraftStatus.DRAFTED
assert out.proposal_id == p.id
# approve + send -> an inbound reply can now thread to this proposal
assert maild._approve_draft(mail_id, "gustavo", sess) == 0
stats = maild._send_approved(STUB_CFG, sess, sender=lambda *a, **k: None)
assert stats["sent"] == 1
sess.flush()
reply = _eml("Re: Partnership", "Approved, welcome!", in_reply_to=out.message_id)
assert maild._match_proposal(maild.parse_raw(reply), sess) is p
def test_mark_seen_returned_for_post_commit_processing(cfg, sess):
"""Medium finding: \\\\Seen was set before processing (crash lost mail)."""
raw = _eml("Re: hi", "Approved, welcome to the program.")
stats, nums = maild._poll(cfg(), sess, fetcher=_fetcher([raw]))
assert stats["processed"] == 1
assert nums == [b"0"] # caller marks seen only after commit