Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91480ebb1a | ||
|
|
6e16965e9b | ||
|
|
50f1e5e1b7 | ||
|
|
d55d8e91a2 |
@@ -7,6 +7,16 @@
|
|||||||
# Timer: cada 3 min. Fallo → DM al supervisor + logger. Nunca toca un tree sucio.
|
# Timer: cada 3 min. Fallo → DM al supervisor + logger. Nunca toca un tree sucio.
|
||||||
|
|
||||||
say() { echo "[autodeploy] $*"; logger -t cayce-autodeploy "$*"; }
|
say() { echo "[autodeploy] $*"; logger -t cayce-autodeploy "$*"; }
|
||||||
|
# r33: un rollback repetido avisa UNA vez por incidente (subject = repo:causa);
|
||||||
|
# el journal sigue completo (say por ciclo). Un deploy exitoso limpia el subject.
|
||||||
|
dm_once() { # $1 subject, $2 mensaje
|
||||||
|
local flag="/var/lib/cayce/autodeploy-alerts/$1"
|
||||||
|
if [ -e "$flag" ]; then say "DM suprimido (incidente ya avisado): $1"; return 0; fi
|
||||||
|
mkdir -p /var/lib/cayce/autodeploy-alerts 2>/dev/null || true
|
||||||
|
touch "$flag" 2>/dev/null || true
|
||||||
|
dm "$2"
|
||||||
|
}
|
||||||
|
dm_clear() { rm -f "/var/lib/cayce/autodeploy-alerts/$1" 2>/dev/null || true; }
|
||||||
dm() {
|
dm() {
|
||||||
SUP=$(grep -aoE 'SUPERVISOR_NPUB_HEX=[0-9a-f]+' /srv/cayce-nostr/.env 2>/dev/null | cut -d= -f2)
|
SUP=$(grep -aoE 'SUPERVISOR_NPUB_HEX=[0-9a-f]+' /srv/cayce-nostr/.env 2>/dev/null | cut -d= -f2)
|
||||||
[ -n "$SUP" ] && curl -s -X POST http://127.0.0.1:3011/dm -H "Content-Type: application/json" \
|
[ -n "$SUP" ] && curl -s -X POST http://127.0.0.1:3011/dm -H "Content-Type: application/json" \
|
||||||
@@ -116,7 +126,7 @@ if ! git merge-base --is-ancestor origin/main HEAD 2>/dev/null && [ -z "$(git st
|
|||||||
fi
|
fi
|
||||||
if ! (cd frontend && npm run build >/tmp/hc-autodeploy-build.log 2>&1); then
|
if ! (cd frontend && npm run build >/tmp/hc-autodeploy-build.log 2>&1); then
|
||||||
git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service
|
git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service
|
||||||
say "ROLLBACK hardcap-news: fallo el build"; dm "ROLLBACK hardcap-news (build) — main NO desplegado, rollback activo"
|
say "ROLLBACK hardcap-news: fallo el build"; dm_once "hardcap:rollback-build" "ROLLBACK hardcap-news (build) — main NO desplegado, rollback activo (aviso unico)"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
sudo -n systemctl restart hardcap-frontend.service
|
sudo -n systemctl restart hardcap-frontend.service
|
||||||
@@ -124,20 +134,25 @@ if ! git merge-base --is-ancestor origin/main HEAD 2>/dev/null && [ -z "$(git st
|
|||||||
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3003/)
|
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3003/)
|
||||||
if [ "$CODE" != "200" ]; then
|
if [ "$CODE" != "200" ]; then
|
||||||
git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service
|
git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service
|
||||||
say "ROLLBACK hardcap-news: verificacion (code=$CODE)"; dm "ROLLBACK hardcap-news verificacion: code=$CODE — rollback activo"
|
say "ROLLBACK hardcap-news: verificacion (code=$CODE)"; dm_once "hardcap:rollback-code" "ROLLBACK hardcap-news verificacion: code=$CODE — rollback activo (aviso unico)"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
# r24: identidad por build-id (cuando el despliegue trae la función)
|
# r24: identidad por build-id (cuando el despliegue trae la función)
|
||||||
if git grep -q "NEXT_PUBLIC_BUILD_ID" -- frontend/src/app/layout.tsx frontend/next.config.js 2>/dev/null; then
|
# r33: la identidad de hardcap solo se exige si ESTE main renderiza la meta
|
||||||
|
# build-id (gh#10 la anade al layout). Mientras no exista, no exigir identidad
|
||||||
|
# (loop del 2026-11-03: 34 rollbacks grep-eando un build-id que main no servia).
|
||||||
|
if git grep -q "name=.build-id." -- frontend/src/app/layout.tsx 2>/dev/null; then
|
||||||
BUILT=$(curl -s -m 8 http://127.0.0.1:3003/ | grep -oE 'name="build-id" content="[a-f0-9]{8}"' | grep -oE '[a-f0-9]{8}' | head -1)
|
BUILT=$(curl -s -m 8 http://127.0.0.1:3003/ | grep -oE 'name="build-id" content="[a-f0-9]{8}"' | grep -oE '[a-f0-9]{8}' | head -1)
|
||||||
WANT=$(git rev-parse --short=8 HEAD)
|
WANT=$(git rev-parse --short=8 HEAD)
|
||||||
if [ "$BUILT" != "$WANT" ]; then
|
if [ "$BUILT" != "$WANT" ]; then
|
||||||
git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service
|
git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service
|
||||||
say "ROLLBACK hardcap-news: build-id '$BUILT' != '$WANT'"; dm "ROLLBACK hardcap-news: servia build '$BUILT' pero lo desplegado era '$WANT'."
|
say "ROLLBACK hardcap-news: build-id '$BUILT' != '$WANT'"; dm_once "hardcap:build-id" "ROLLBACK hardcap-news: servia build '$BUILT' pero lo desplegado era '$WANT'. Hasta mergear gh#10 (meta build-id) el check de identidad no se exige; aviso unico."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
dm_clear "hardcap:build-id"; dm_clear "hardcap:rollback-build"; dm_clear "hardcap:rollback-code"
|
||||||
say "identidad hardcap OK: servido=$BUILT desplegado=$WANT"
|
say "identidad hardcap OK: servido=$BUILT desplegado=$WANT"
|
||||||
fi
|
fi
|
||||||
|
dm_clear "hardcap:build-id"; dm_clear "hardcap:rollback-build"; dm_clear "hardcap:rollback-code"
|
||||||
say "hardcap-news → $(git rev-parse --short HEAD) (200)"
|
say "hardcap-news → $(git rev-parse --short HEAD) (200)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,19 @@ WHERE ts > now() - interval '24 hours'
|
|||||||
GROUP BY reasons ORDER BY count(*) DESC LIMIT 6;
|
GROUP BY reasons ORDER BY count(*) DESC LIMIT 6;
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "== 2b. DMs urgentes (politica r33) =="
|
||||||
|
NDM=$(journalctl -u cayce-nostr --since "-24 hours" --no-pager 2>/dev/null | grep -c "\[dm\] envio a")
|
||||||
|
UCOUNT=$(python3 -c "import json; print(json.load(open('/srv/cayce-core/.dm-urgent-count.json')).get('$(date -u +%F)', 0))" 2>/dev/null || echo 0)
|
||||||
|
echo " DMs NIP-04 enviados (24h, todas las causas): $NDM · urgentes de hoy (tope 3): $UCOUNT"
|
||||||
|
if [ -s /srv/cayce-core/.dm-deferred.log ]; then
|
||||||
|
echo " diferidos por tope (van aqui en vez de DM):"
|
||||||
|
tail -5 /srv/cayce-core/.dm-deferred.log | sed "s/^/ /"
|
||||||
|
fi
|
||||||
|
if [ -f /srv/cayce-core/PUBLISH_OFF ]; then
|
||||||
|
echo " kill switch: ACTIVO ($(head -c 20 /srv/cayce-core/PUBLISH_OFF 2>/dev/null))"
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "== 3. PRs y deploys =="
|
echo "== 3. PRs y deploys =="
|
||||||
BTOK=$(cat /etc/cayce/ci-bot.token 2>/dev/null)
|
BTOK=$(cat /etc/cayce/ci-bot.token 2>/dev/null)
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# cayce-urgent-watch (r21 → r33) — política de DMs del operador:
|
||||||
|
# DM inmediato SOLO si:
|
||||||
|
# · kill switch activado o fuga detectada
|
||||||
|
# · sitio/servicio público caído >15 min (no rollbacks que se recuperan)
|
||||||
|
# · backup fallido 2 días seguidos
|
||||||
|
# · gasto de Gemini por encima del umbral
|
||||||
|
# Todo lo demás (initiative, rollbacks recuperados, ciclos, publicaciones,
|
||||||
|
# rechazos, merges CODEOWNERS) va SOLO al resumen diario de las 23:50 UTC.
|
||||||
|
# Ciclos pausados a propósito (p.ej. promo) no generan alertas.
|
||||||
|
# Dedup por tipo de incidente + sujeto (24 h). Tope: 3 DMs urgentes/día;
|
||||||
|
# a partir de ahí, una línea en el resumen diario (.dm-deferred.log).
|
||||||
|
# DRYRUN=1 imprime qué haría sin enviar ni contar.
|
||||||
|
DEDUP=/srv/cayce-core/.dm-dedup.json
|
||||||
|
DOWNST=/srv/cayce-core/.critical-down.json
|
||||||
|
COUNTF=/srv/cayce-core/.dm-urgent-count.json
|
||||||
|
DEFER=/srv/cayce-core/.dm-deferred.log
|
||||||
|
USAGE=${LLM_USAGE_LOG:-/srv/cayce-core/.llm-usage.jsonl}
|
||||||
|
GEMINI_MAX_USD=${GEMINI_MAX_USD:-1.0}
|
||||||
|
export TTL_MS=$((24 * 3600 * 1000))
|
||||||
|
DRY=${DRYRUN:-0}
|
||||||
|
|
||||||
|
dm_urgente() { # $1 clave (tipo:sujeto), $2 mensaje
|
||||||
|
python3 - "$1" "$2" "$DEDUP" "$TTL_MS" "$COUNTF" "$DEFER" "$DRY" <<'PY'
|
||||||
|
import json, sys, subprocess, datetime, os
|
||||||
|
key, msg, dedup, ttl, countf, defer, dry = sys.argv[1:8]
|
||||||
|
ttl, dry = int(ttl), int(dry)
|
||||||
|
st = {}
|
||||||
|
try: st = json.load(open(dedup))
|
||||||
|
except Exception: pass
|
||||||
|
now = datetime.datetime.utcnow().timestamp() * 1000
|
||||||
|
if now - st.get(key, 0) < ttl:
|
||||||
|
print(f"[urgent] DM dedup ({key}) — ya alertado en <24h"); sys.exit(0)
|
||||||
|
cnt = {}
|
||||||
|
today = datetime.datetime.utcnow().date().isoformat()
|
||||||
|
try: cnt = json.load(open(countf))
|
||||||
|
except Exception: pass
|
||||||
|
n = cnt.get(today, 0)
|
||||||
|
sup = ""
|
||||||
|
for l in open("/srv/cayce-nostr/.env"):
|
||||||
|
if l.startswith("SUPERVISOR_NPUB_HEX="): sup = l.strip().split("=",1)[1]
|
||||||
|
if not sup:
|
||||||
|
print("[urgent] sin supervisor — solo log"); sys.exit(0)
|
||||||
|
if n >= 3:
|
||||||
|
with open(defer, "a") as f:
|
||||||
|
f.write(f"{datetime.datetime.utcnow().isoformat()} DEFERIDO (tope 3/dia) {key}: {msg}\n")
|
||||||
|
print(f"[urgent] TOPE 3/dia alcanzado — diferido al resumen diario ({key})")
|
||||||
|
sys.exit(0)
|
||||||
|
if dry:
|
||||||
|
print(f"[urgent] DRYRUN: enviaria DM ({key}): {msg[:100]}")
|
||||||
|
sys.exit(0)
|
||||||
|
subprocess.run(["curl","-s","-m","20","-X","POST","http://127.0.0.1:3011/dm",
|
||||||
|
"-H","Content-Type: application/json","-d",json.dumps({"to_hex":sup,"content":msg})],
|
||||||
|
capture_output=True)
|
||||||
|
st[key] = now
|
||||||
|
cnt[today] = n + 1
|
||||||
|
json.dump(st, open(dedup, "w"))
|
||||||
|
json.dump(cnt, open(countf, "w"))
|
||||||
|
print(f"[urgent] DM enviado ({key}) — {cnt[today]}/3 hoy")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
# 1) kill switch manual o fuga: PUBLISH_OFF sin violación reciente del watcher,
|
||||||
|
# o violación del relay-watcher (fuga) en la última hora.
|
||||||
|
if [ -f /srv/cayce-core/PUBLISH_OFF ]; then
|
||||||
|
SIG=$(head -c 9 /srv/cayce-core/PUBLISH_OFF 2>/dev/null)
|
||||||
|
AGE=$(( $(date -u +%s) - $(stat -c %Y /srv/cayce-core/PUBLISH_OFF 2>/dev/null || echo 0) ))
|
||||||
|
if [ "$SIG" != "autodeploy" ] || [ "$AGE" -ge 600 ]; then
|
||||||
|
V=$(sudo -n -u postgres psql -d cayce -tAc "SELECT count(*) FROM guard_rejections WHERE type='watcher_violation' AND ts > now() - interval '15 minutes'" 2>/dev/null || echo 1)
|
||||||
|
[ "$V" = "0" ] && dm_urgente "killswitch:manual" "URGENTE: PUBLISH_OFF activo (kill switch MANUAL) — publicación en draft global. Sin violación del watcher que lo explique."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
FUGA=$(sudo -n -u postgres psql -d cayce -tAc "SELECT count(*) FROM guard_rejections WHERE type='watcher_violation' AND ts > now() - interval '60 minutes'" 2>/dev/null || echo 0)
|
||||||
|
[ "$FUGA" != "0" ] && dm_urgente "fuga:watcher" "URGENTE: $FUGA violaciones del watcher en 60 min — posible fuga de publicación. PUBLISH_OFF recomendado."
|
||||||
|
|
||||||
|
# 2) servicios críticos caídos >15 min (estado persistente de primera caída)
|
||||||
|
while IFS=: read -r TAG UNIT MINS; do
|
||||||
|
[ "$TAG" = "DOWN" ] && dm_urgente "down:$UNIT" "URGENTE: servicio crítico caído ${MINS} min: $UNIT"
|
||||||
|
done < <(python3 - "$DOWNST" <<'PY'
|
||||||
|
import json, subprocess, sys, time
|
||||||
|
UNITS = ["cayce-nostr.service","hardcap-backend.service","hardcap-frontend.service",
|
||||||
|
"cayce-bitcoinhotels.service","postgresql@18-main.service","caddy.service"]
|
||||||
|
st = {}
|
||||||
|
try: st = json.load(open(sys.argv[1]))
|
||||||
|
except Exception: pass
|
||||||
|
now = time.time()
|
||||||
|
for u in UNITS:
|
||||||
|
active = subprocess.run(["systemctl","is-active",u], capture_output=True, text=True).stdout.strip() == "active"
|
||||||
|
if active:
|
||||||
|
st.pop(u, None)
|
||||||
|
else:
|
||||||
|
st.setdefault(u, now)
|
||||||
|
if now - st[u] > 15*60:
|
||||||
|
print(f"DOWN:{u}:{int((now-st[u])/60)}")
|
||||||
|
json.dump(st, open(sys.argv[1], "w"))
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3) backup fallido 2 días seguidos (r33: un fallo puntual no es urgente)
|
||||||
|
if systemctl is-failed cayce-backup.service >/dev/null 2>&1; then
|
||||||
|
FAILS=$(journalctl -u cayce-backup.service --since "48 hours ago" --no-pager 2>/dev/null | grep -c "Failed with result")
|
||||||
|
if [ "${FAILS:-0}" -ge 2 ]; then
|
||||||
|
dm_urgente "backup:2días" "URGENTE: cayce-backup fallido ${FAILS} veces en 48h — revisar /var/backups/cayce y el journal."
|
||||||
|
else
|
||||||
|
echo "[urgent] backup failed hoy ($FAILS en 48h <2) — va al resumen diario"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 4) gasto de Gemini hoy > umbral (tokens reales)
|
||||||
|
read -r COST CALLS TIN TOUT < <(python3 - "$USAGE" <<'PY'
|
||||||
|
import json, sys, os, datetime
|
||||||
|
rin = float(os.environ.get("GEMINI_USD_PER_MTOK_IN", "0.15"))
|
||||||
|
rout = float(os.environ.get("GEMINI_USD_PER_MTOK_OUT", "0.60"))
|
||||||
|
today = datetime.datetime.utcnow().date().isoformat()
|
||||||
|
IN = OUT_ = CALLS = 0
|
||||||
|
if os.path.exists(sys.argv[1]):
|
||||||
|
for line in open(sys.argv[1]):
|
||||||
|
try:
|
||||||
|
d = json.loads(line)
|
||||||
|
if d.get("ts","")[:10] == today:
|
||||||
|
CALLS += 1; IN += d.get("in",0); OUT_ += d.get("out",0)
|
||||||
|
except Exception: pass
|
||||||
|
print(f"{IN/1e6*rin + OUT_/1e6*rout:.4f} {CALLS} {IN} {OUT_}")
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
if python3 -c "exit(0 if float('${COST:-0}') > float('$GEMINI_MAX_USD') else 1)" 2>/dev/null; then
|
||||||
|
dm_urgente "gasto:gemini" "URGENTE: gasto Gemini de hoy \$${COST} (> \$${GEMINI_MAX_USD}) — ${CALLS} llamadas, ${TIN} in / ${TOUT} out tokens."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# r33: lo que YA NO manda DM (solo resumen diario de 23:50):
|
||||||
|
# · ciclos mudos / pausados a propósito (promo hoy) — el resumen diario ya
|
||||||
|
# trae la sección "Ciclos: piezas producidas"
|
||||||
|
# · merges que tocan CODEOWNERS — el resumen diario (sección 3) lista los
|
||||||
|
# merges del día; la sección 3b lista los PRs abiertos
|
||||||
|
# · rollbacks recuperados — el autodeploy usa dm_once (un aviso por
|
||||||
|
# incidente) y el resumen diario lista los deploys/rollbacks del journal
|
||||||
|
|
||||||
|
logger -t cayce-urgent-watch "tick ok (gemini=\$${COST:-0})"
|
||||||
Reference in New Issue
Block a user