"""scanner tests — discovery, classification, dedupe. No network (injected fetch)."""
from __future__ import annotations
import json
from cayceops.components.scanner import scanner
from cayceops.models import Proposal, ProposalStatus
RSS = b"""
- Acme launches affiliate program for hotels
https://acme.example.org/affiliates
Earn commission on every booking.
- Weekly crypto digest
https://news.example.org/digest
Everything unrelated.
"""
PAGE = b"""
Partners
Become a payment partner
API docs
"""
def _fake_fetch(responses):
def fetch(url, cfg, timeout=20.0):
return responses[url]
return fetch
SOURCES = [
{"type": "rss", "url": "https://feeds.example.org/news.xml"},
{"type": "page", "url": "https://pay.example.org/partners"},
]
def _cfg(cfg, **kw):
return cfg(
CAYCE_OPS_SCANNER_SOURCES=json.dumps(SOURCES),
CAYCE_OPS_SCANNER_KEYWORDS="affiliate,partner,bitcoin,commission",
**kw,
)
def test_scan_files_proposals_with_kind_and_score(cfg, sess):
config = _cfg(cfg)
stats = scanner.scan(config, sess, fetch=_fake_fetch({
"https://feeds.example.org/news.xml": RSS,
"https://pay.example.org/partners": PAGE,
}))
assert stats["proposed"] == 3 # affiliate item + both page links (pages keep all)
proposals = sess.query(Proposal).all()
by_title = {p.title: p for p in proposals}
affiliate = by_title["Acme launches affiliate program for hotels"]
assert affiliate.kind == "affiliate"
assert affiliate.status == ProposalStatus.PROPOSED
assert affiliate.risk_score > 0
assert affiliate.dedupe_key
partner = by_title["Become a payment partner"]
assert partner.kind == "partnership"
def test_scan_is_idempotent_via_dedupe(cfg, sess):
config = _cfg(cfg)
scanner.scan(config, sess, fetch=_fake_fetch({
"https://feeds.example.org/news.xml": RSS,
"https://pay.example.org/partners": PAGE,
}))
stats2 = scanner.scan(config, sess, fetch=_fake_fetch({
"https://feeds.example.org/news.xml": RSS,
"https://pay.example.org/partners": PAGE,
}))
assert stats2["proposed"] == 0
assert stats2["skipped_existing"] == 3
def test_scan_filters_keywordless_feed_items(cfg, sess):
config = _cfg(cfg)
scanner.scan(config, sess, fetch=_fake_fetch({
"https://feeds.example.org/news.xml": RSS,
"https://pay.example.org/partners": PAGE,
}))
titles = {p.title for p in sess.query(Proposal).all()}
assert "Weekly crypto digest" not in titles # no keyword match in feed -> dropped
def test_scan_records_audit_for_each_proposal(cfg, sess):
from cayceops.models import AuditEntry
config = _cfg(cfg)
scanner.scan(config, sess, fetch=_fake_fetch({
"https://feeds.example.org/news.xml": RSS,
"https://pay.example.org/partners": PAGE,
}))
acts = [a.action for a in sess.query(AuditEntry).all()]
assert acts.count("propose") == 3
assert all(a.actor == "daemon:scanner" for a in sess.query(AuditEntry).all())
def test_scan_survives_source_error(cfg, sess):
def broken_fetch(url, cfg, timeout=20.0):
raise OSError("connection refused")
config = _cfg(cfg)
stats = scanner.scan(config, sess, fetch=broken_fetch)
assert stats["proposed"] == 0
assert len(stats["source_errors"]) == 2