From 91480ebb1af0dffa517dc66f5751686d1e429a6f Mon Sep 17 00:00:00 2001 From: coder-agent Date: Tue, 3 Nov 2026 16:23:08 +0000 Subject: [PATCH] politica de DMs r33: urgent-watch solo 4 clases + tope 3/dia + dedup tipo:sujeto 24h + backup 2 dias; reporte diario con conteo de DMs y diferidos --- deploy/usr-local-bin/cayce-daily-report | 13 +++ deploy/usr-local-bin/cayce-urgent-watch | 138 ++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 deploy/usr-local-bin/cayce-urgent-watch diff --git a/deploy/usr-local-bin/cayce-daily-report b/deploy/usr-local-bin/cayce-daily-report index b293bc0..4e7d8f8 100755 --- a/deploy/usr-local-bin/cayce-daily-report +++ b/deploy/usr-local-bin/cayce-daily-report @@ -39,6 +39,19 @@ WHERE ts > now() - interval '24 hours' GROUP BY reasons ORDER BY count(*) DESC LIMIT 6; EOF +echo "" +echo "== 2b. DMs urgentes (politica r33) ==" +NDM=$(journalctl -u cayce-nostr --since "-24 hours" --no-pager 2>/dev/null | grep -c "\[dm\] envio a") +UCOUNT=$(python3 -c "import json; print(json.load(open('/srv/cayce-core/.dm-urgent-count.json')).get('$(date -u +%F)', 0))" 2>/dev/null || echo 0) +echo " DMs NIP-04 enviados (24h, todas las causas): $NDM · urgentes de hoy (tope 3): $UCOUNT" +if [ -s /srv/cayce-core/.dm-deferred.log ]; then + echo " diferidos por tope (van aqui en vez de DM):" + tail -5 /srv/cayce-core/.dm-deferred.log | sed "s/^/ /" +fi +if [ -f /srv/cayce-core/PUBLISH_OFF ]; then + echo " kill switch: ACTIVO ($(head -c 20 /srv/cayce-core/PUBLISH_OFF 2>/dev/null))" +fi + echo "" echo "== 3. PRs y deploys ==" BTOK=$(cat /etc/cayce/ci-bot.token 2>/dev/null) diff --git a/deploy/usr-local-bin/cayce-urgent-watch b/deploy/usr-local-bin/cayce-urgent-watch new file mode 100644 index 0000000..2477b04 --- /dev/null +++ b/deploy/usr-local-bin/cayce-urgent-watch @@ -0,0 +1,138 @@ +#!/bin/bash +# cayce-urgent-watch (r21 → r33) — política de DMs del operador: +# DM inmediato SOLO si: +# · kill switch activado o fuga detectada +# · sitio/servicio público caído >15 min (no rollbacks que se recuperan) +# · backup fallido 2 días seguidos +# · gasto de Gemini por encima del umbral +# Todo lo demás (initiative, rollbacks recuperados, ciclos, publicaciones, +# rechazos, merges CODEOWNERS) va SOLO al resumen diario de las 23:50 UTC. +# Ciclos pausados a propósito (p.ej. promo) no generan alertas. +# Dedup por tipo de incidente + sujeto (24 h). Tope: 3 DMs urgentes/día; +# a partir de ahí, una línea en el resumen diario (.dm-deferred.log). +# DRYRUN=1 imprime qué haría sin enviar ni contar. +DEDUP=/srv/cayce-core/.dm-dedup.json +DOWNST=/srv/cayce-core/.critical-down.json +COUNTF=/srv/cayce-core/.dm-urgent-count.json +DEFER=/srv/cayce-core/.dm-deferred.log +USAGE=${LLM_USAGE_LOG:-/srv/cayce-core/.llm-usage.jsonl} +GEMINI_MAX_USD=${GEMINI_MAX_USD:-1.0} +export TTL_MS=$((24 * 3600 * 1000)) +DRY=${DRYRUN:-0} + +dm_urgente() { # $1 clave (tipo:sujeto), $2 mensaje + python3 - "$1" "$2" "$DEDUP" "$TTL_MS" "$COUNTF" "$DEFER" "$DRY" <<'PY' +import json, sys, subprocess, datetime, os +key, msg, dedup, ttl, countf, defer, dry = sys.argv[1:8] +ttl, dry = int(ttl), int(dry) +st = {} +try: st = json.load(open(dedup)) +except Exception: pass +now = datetime.datetime.utcnow().timestamp() * 1000 +if now - st.get(key, 0) < ttl: + print(f"[urgent] DM dedup ({key}) — ya alertado en <24h"); sys.exit(0) +cnt = {} +today = datetime.datetime.utcnow().date().isoformat() +try: cnt = json.load(open(countf)) +except Exception: pass +n = cnt.get(today, 0) +sup = "" +for l in open("/srv/cayce-nostr/.env"): + if l.startswith("SUPERVISOR_NPUB_HEX="): sup = l.strip().split("=",1)[1] +if not sup: + print("[urgent] sin supervisor — solo log"); sys.exit(0) +if n >= 3: + with open(defer, "a") as f: + f.write(f"{datetime.datetime.utcnow().isoformat()} DEFERIDO (tope 3/dia) {key}: {msg}\n") + print(f"[urgent] TOPE 3/dia alcanzado — diferido al resumen diario ({key})") + sys.exit(0) +if dry: + print(f"[urgent] DRYRUN: enviaria DM ({key}): {msg[:100]}") + sys.exit(0) +subprocess.run(["curl","-s","-m","20","-X","POST","http://127.0.0.1:3011/dm", + "-H","Content-Type: application/json","-d",json.dumps({"to_hex":sup,"content":msg})], + capture_output=True) +st[key] = now +cnt[today] = n + 1 +json.dump(st, open(dedup, "w")) +json.dump(cnt, open(countf, "w")) +print(f"[urgent] DM enviado ({key}) — {cnt[today]}/3 hoy") +PY +} + +# 1) kill switch manual o fuga: PUBLISH_OFF sin violación reciente del watcher, +# o violación del relay-watcher (fuga) en la última hora. +if [ -f /srv/cayce-core/PUBLISH_OFF ]; then + SIG=$(head -c 9 /srv/cayce-core/PUBLISH_OFF 2>/dev/null) + AGE=$(( $(date -u +%s) - $(stat -c %Y /srv/cayce-core/PUBLISH_OFF 2>/dev/null || echo 0) )) + if [ "$SIG" != "autodeploy" ] || [ "$AGE" -ge 600 ]; then + V=$(sudo -n -u postgres psql -d cayce -tAc "SELECT count(*) FROM guard_rejections WHERE type='watcher_violation' AND ts > now() - interval '15 minutes'" 2>/dev/null || echo 1) + [ "$V" = "0" ] && dm_urgente "killswitch:manual" "URGENTE: PUBLISH_OFF activo (kill switch MANUAL) — publicación en draft global. Sin violación del watcher que lo explique." + fi +fi +FUGA=$(sudo -n -u postgres psql -d cayce -tAc "SELECT count(*) FROM guard_rejections WHERE type='watcher_violation' AND ts > now() - interval '60 minutes'" 2>/dev/null || echo 0) +[ "$FUGA" != "0" ] && dm_urgente "fuga:watcher" "URGENTE: $FUGA violaciones del watcher en 60 min — posible fuga de publicación. PUBLISH_OFF recomendado." + +# 2) servicios críticos caídos >15 min (estado persistente de primera caída) +while IFS=: read -r TAG UNIT MINS; do + [ "$TAG" = "DOWN" ] && dm_urgente "down:$UNIT" "URGENTE: servicio crítico caído ${MINS} min: $UNIT" +done < <(python3 - "$DOWNST" <<'PY' +import json, subprocess, sys, time +UNITS = ["cayce-nostr.service","hardcap-backend.service","hardcap-frontend.service", + "cayce-bitcoinhotels.service","postgresql@18-main.service","caddy.service"] +st = {} +try: st = json.load(open(sys.argv[1])) +except Exception: pass +now = time.time() +for u in UNITS: + active = subprocess.run(["systemctl","is-active",u], capture_output=True, text=True).stdout.strip() == "active" + if active: + st.pop(u, None) + else: + st.setdefault(u, now) + if now - st[u] > 15*60: + print(f"DOWN:{u}:{int((now-st[u])/60)}") +json.dump(st, open(sys.argv[1], "w")) +PY +) + +# 3) backup fallido 2 días seguidos (r33: un fallo puntual no es urgente) +if systemctl is-failed cayce-backup.service >/dev/null 2>&1; then + FAILS=$(journalctl -u cayce-backup.service --since "48 hours ago" --no-pager 2>/dev/null | grep -c "Failed with result") + if [ "${FAILS:-0}" -ge 2 ]; then + dm_urgente "backup:2días" "URGENTE: cayce-backup fallido ${FAILS} veces en 48h — revisar /var/backups/cayce y el journal." + else + echo "[urgent] backup failed hoy ($FAILS en 48h <2) — va al resumen diario" + fi +fi + +# 4) gasto de Gemini hoy > umbral (tokens reales) +read -r COST CALLS TIN TOUT < <(python3 - "$USAGE" <<'PY' +import json, sys, os, datetime +rin = float(os.environ.get("GEMINI_USD_PER_MTOK_IN", "0.15")) +rout = float(os.environ.get("GEMINI_USD_PER_MTOK_OUT", "0.60")) +today = datetime.datetime.utcnow().date().isoformat() +IN = OUT_ = CALLS = 0 +if os.path.exists(sys.argv[1]): + for line in open(sys.argv[1]): + try: + d = json.loads(line) + if d.get("ts","")[:10] == today: + CALLS += 1; IN += d.get("in",0); OUT_ += d.get("out",0) + except Exception: pass +print(f"{IN/1e6*rin + OUT_/1e6*rout:.4f} {CALLS} {IN} {OUT_}") +PY +) +if python3 -c "exit(0 if float('${COST:-0}') > float('$GEMINI_MAX_USD') else 1)" 2>/dev/null; then + dm_urgente "gasto:gemini" "URGENTE: gasto Gemini de hoy \$${COST} (> \$${GEMINI_MAX_USD}) — ${CALLS} llamadas, ${TIN} in / ${TOUT} out tokens." +fi + +# r33: lo que YA NO manda DM (solo resumen diario de 23:50): +# · ciclos mudos / pausados a propósito (promo hoy) — el resumen diario ya +# trae la sección "Ciclos: piezas producidas" +# · merges que tocan CODEOWNERS — el resumen diario (sección 3) lista los +# merges del día; la sección 3b lista los PRs abiertos +# · rollbacks recuperados — el autodeploy usa dm_once (un aviso por +# incidente) y el resumen diario lista los deploys/rollbacks del journal + +logger -t cayce-urgent-watch "tick ok (gemini=\$${COST:-0})" -- 2.53.0