"""scanner tests — discovery, classification, dedupe. No network (injected fetch).""" from __future__ import annotations import json from cayceops.components.scanner import scanner from cayceops.models import Proposal, ProposalStatus RSS = b""" Acme launches affiliate program for hotels https://acme.example.org/affiliates Earn commission on every booking. Weekly crypto digest https://news.example.org/digest Everything unrelated. """ PAGE = b"""Partners Become a payment partner API docs """ def _fake_fetch(responses): def fetch(url, cfg, timeout=20.0): return responses[url] return fetch SOURCES = [ {"type": "rss", "url": "https://feeds.example.org/news.xml"}, {"type": "page", "url": "https://pay.example.org/partners"}, ] def _cfg(cfg, **kw): return cfg( CAYCE_OPS_SCANNER_SOURCES=json.dumps(SOURCES), CAYCE_OPS_SCANNER_KEYWORDS="affiliate,partner,bitcoin,commission", **kw, ) def test_scan_files_proposals_with_kind_and_score(cfg, sess): config = _cfg(cfg) stats = scanner.scan(config, sess, fetch=_fake_fetch({ "https://feeds.example.org/news.xml": RSS, "https://pay.example.org/partners": PAGE, })) assert stats["proposed"] == 3 # affiliate item + both page links (pages keep all) proposals = sess.query(Proposal).all() by_title = {p.title: p for p in proposals} affiliate = by_title["Acme launches affiliate program for hotels"] assert affiliate.kind == "affiliate" assert affiliate.status == ProposalStatus.PROPOSED assert affiliate.match_score > 0 assert affiliate.dedupe_key partner = by_title["Become a payment partner"] assert partner.kind == "partnership" def test_scan_is_idempotent_via_dedupe(cfg, sess): config = _cfg(cfg) scanner.scan(config, sess, fetch=_fake_fetch({ "https://feeds.example.org/news.xml": RSS, "https://pay.example.org/partners": PAGE, })) stats2 = scanner.scan(config, sess, fetch=_fake_fetch({ "https://feeds.example.org/news.xml": RSS, "https://pay.example.org/partners": PAGE, })) assert stats2["proposed"] == 0 assert stats2["skipped_existing"] == 3 def test_scan_filters_keywordless_feed_items(cfg, sess): config = _cfg(cfg) scanner.scan(config, sess, fetch=_fake_fetch({ "https://feeds.example.org/news.xml": RSS, "https://pay.example.org/partners": PAGE, })) titles = {p.title for p in sess.query(Proposal).all()} assert "Weekly crypto digest" not in titles # no keyword match in feed -> dropped def test_scan_records_audit_for_each_proposal(cfg, sess): from cayceops.models import AuditEntry config = _cfg(cfg) scanner.scan(config, sess, fetch=_fake_fetch({ "https://feeds.example.org/news.xml": RSS, "https://pay.example.org/partners": PAGE, })) acts = [a.action for a in sess.query(AuditEntry).all()] assert acts.count("propose") == 3 assert all(a.actor == "daemon:scanner" for a in sess.query(AuditEntry).all()) def test_scan_survives_source_error(cfg, sess): def broken_fetch(url, cfg, timeout=20.0): raise OSError("connection refused") config = _cfg(cfg) stats = scanner.scan(config, sess, fetch=broken_fetch) assert stats["proposed"] == 0 assert len(stats["source_errors"]) == 2