From 5cc3fd64195a9db538a6c9d5c496c74a3a03545f Mon Sep 17 00:00:00 2001 From: coder-agent Date: Tue, 3 Nov 2026 03:03:03 +0000 Subject: [PATCH] s1: autodeploy+auto-merge con flock compartido (versionados) + guard de workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Incidente 02:06:29Z: el rollback del autodeploy (git reset --hard $OLD) pisó la rama e2-fuentes-y-sellos porque el árbol cambió de branch a mitad de deploy. Causa: ninguna exclusión mutua entre autodeploy, auto-merger y ops manuales sobre los mismos árboles. - Lock: /run/lock/cayce-git-ops.lock (tmpfiles.d, 0664 root:ubuntu). autodeploy flock -n (salta ciclo), auto-merge flock -w 240 (espera). - Guard en ci.yml: un job cuyo workspace caiga bajo /srv/* revienta. --- .gitea/workflows/ci.yml | 6 + deploy/usr-local-bin/README.md | 11 ++ deploy/usr-local-bin/cayce-auto-merge | 46 +++++++ deploy/usr-local-bin/cayce-autodeploy | 180 ++++++++++++++++++++++++++ 4 files changed, 243 insertions(+) create mode 100644 deploy/usr-local-bin/README.md create mode 100755 deploy/usr-local-bin/cayce-auto-merge create mode 100755 deploy/usr-local-bin/cayce-autodeploy diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8a98361..08b7509 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -15,6 +15,12 @@ jobs: run: shell: bash --noprofile --norc -eo pipefail {0} steps: + - name: "r31 guard: CI nunca opera en /srv/*" + run: | + case "$PWD" in + /srv/*) echo "VIOLACION: workspace de CI en $PWD"; exit 1 ;; + *) echo "workspace OK: $PWD" ;; + esac - uses: actions/checkout@v4 with: fetch-depth: 1 diff --git a/deploy/usr-local-bin/README.md b/deploy/usr-local-bin/README.md new file mode 100644 index 0000000..ca074a6 --- /dev/null +++ b/deploy/usr-local-bin/README.md @@ -0,0 +1,11 @@ +# /usr/local/bin versionado + +Copias exactas de los scripts de infra que viven en /usr/local/bin del VPS. +Actualizar: PR aquí → merge → copiar al VPS (sudo cp) + daemon-reload si toca unit. + +- cayce-autodeploy (r31: flock compartido /run/lock/cayce-git-ops.lock) +- cayce-auto-merge (r31: mismo lock, espera hasta 240s) + +El lockfile lo crea tmpfiles.d: /etc/tmpfiles.d/cayce-locks.conf +(f /run/lock/cayce-git-ops.lock 0664 root ubuntu -). Ops manuales con git +sobre /srv/*: flock /run/lock/cayce-git-ops.lock . diff --git a/deploy/usr-local-bin/cayce-auto-merge b/deploy/usr-local-bin/cayce-auto-merge new file mode 100755 index 0000000..4c0f7be --- /dev/null +++ b/deploy/usr-local-bin/cayce-auto-merge @@ -0,0 +1,46 @@ +#!/bin/bash +# Auto-merge (ronda-9): SOLO PRs con todos los checks en success y que NO +# toquen paths protegidos. Nunca btcpay-events. Nunca credenciales del operador. +# r19: el token persiste en /etc/cayce/ci-bot.token (600). Regenerar con nombre +# ya usado da ERROR y dejaba BOT vacio (bug historico: el script no hacia nada). +# r31 (S1): lock compartido con cayce-autodeploy — nunca compiten sobre los +# mismos arboles/branches (incidente 02:06:29Z del 2026-11-03). +LOCK=/run/lock/cayce-git-ops.lock +exec 9>"$LOCK" || { echo 'FATAL: no se puede abrir lockfile'; exit 1; } +if ! flock -w 240 9; then + echo "[auto-merge] autodeploy mantiene el lock >240s — paso de este ciclo (el proximo mergea)" + exit 0 +fi +BOT=$(cat /etc/cayce/ci-bot.token 2>/dev/null | tr -d '[:space:]') +if [ -z "$BOT" ]; then + BOT=$(sudo -u git /usr/local/bin/gitea --config /etc/gitea/app.ini admin user generate-access-token -u ci-bot -t "auto-merge-$(date +%s)" --scopes write:repository,write:issue 2>/dev/null | grep -aoE "[0-9a-f]{40}" | head -1) + [ -n "$BOT" ] && printf '%s' "$BOT" > /etc/cayce/ci-bot.token && chmod 600 /etc/cayce/ci-bot.token +fi +[ -z "$BOT" ] && { echo 'FATAL: sin token ci-bot'; exit 1; } +G="http://127.0.0.1:3100/api/v1" +# paths protegidos (grep -E contra la lista de ficheros modificados del PR) +PROTECTED="SPEC.md|MANDATE.md|prompts/cayce-identity.md|guard[^/]*\.|relay-watcher|outbox\.mjs|guard-blocklist|check-single-outbox|\.gitea/workflows|\.github/workflows|cayce-auto-merge|act_runner|docker" +for REPO in cayce-ops entity bitcoinhotels; do + curl -s -H "Authorization: token $BOT" "$G/repos/cayce/$REPO/pulls?state=open&limit=20" | python3 -c " +import json,sys +for p in json.load(sys.stdin): + if p.get('title','').upper().startswith('WIP'): continue + print(p['number'], p['head']['sha'])" 2>/dev/null | while read N SHA; do + FILES=$(curl -s -H "Authorization: token $BOT" "$G/repos/cayce/$REPO/pulls/$N/files?limit=100" | python3 -c " +import json,sys +for f in json.load(sys.stdin): print(f.get('filename',''))" 2>/dev/null) + if echo "$FILES" | grep -qE "$PROTECTED"; then + echo "$REPO #$N: TOCA PATH PROTEGIDO — no se auto-mergea" + continue + fi + # r19: /status (combinado) en vez de /statuses — la lista incluye los + # 'pending' historicos y hacia que un PR verde pareciera no verde + OK=$(curl -s -H "Authorization: token $BOT" "$G/repos/cayce/$REPO/commits/$SHA/status" | python3 -c " +import json,sys +d = json.load(sys.stdin) +print('yes' if isinstance(d, dict) and d.get('state') == 'success' else 'no')" 2>/dev/null) + if [ "$OK" = "yes" ]; then + curl -s -X POST -H "Authorization: token $BOT" -H "Content-Type: application/json" -d '{"Do":"merge"}' "$G/repos/cayce/$REPO/pulls/$N/merge" -o /dev/null -w "$REPO #$N (green, sin paths protegidos) merge: %{http_code}\n" + fi + done +done diff --git a/deploy/usr-local-bin/cayce-autodeploy b/deploy/usr-local-bin/cayce-autodeploy new file mode 100755 index 0000000..d4d6152 --- /dev/null +++ b/deploy/usr-local-bin/cayce-autodeploy @@ -0,0 +1,180 @@ +#!/bin/bash +# cayce-autodeploy — r18: despliega main nuevo SIN intervención del operador. +# ~/cayce-ops → git pull de main (docs, prompts; el despliegue lee el tree) +# /srv/cayce-core → checkout de cayce/entity main, con PUBLISH_OFF activo +# durante el swap. Solo se retira si: tests del guard +# 27+/27 y watcher sin VIOLACION. Rollback automatico. +# Timer: cada 3 min. Fallo → DM al supervisor + logger. Nunca toca un tree sucio. + +say() { echo "[autodeploy] $*"; logger -t cayce-autodeploy "$*"; } +dm() { + SUP=$(grep -aoE 'SUPERVISOR_NPUB_HEX=[0-9a-f]+' /srv/cayce-nostr/.env 2>/dev/null | cut -d= -f2) + [ -n "$SUP" ] && curl -s -X POST http://127.0.0.1:3011/dm -H "Content-Type: application/json" \ + -d "{\"to_hex\":\"$SUP\",\"content\":\"AUTODEPLOY: $1\"}" >/dev/null 2>&1 +} + +# r31 (S1): lock compartido con cayce-auto-merge — nunca compiten sobre los +# mismos arboles/branches (incidente 02:06:29Z del 2026-11-03: un rollback +# `git reset --hard $OLD` pisó una rama de trabajo porque el árbol cambió de +# branch a mitad de deploy). Las ops manuales con git sobre /srv/* tambien +# lo toman: flock /run/lock/cayce-git-ops.lock +LOCK=/run/lock/cayce-git-ops.lock +exec 9>"$LOCK" || { say 'FATAL: no se puede abrir lockfile'; exit 1; } +if ! flock -n 9; then + say "auto-merge (u otra op) mantiene el lock — paso de este ciclo" + exit 0 +fi + +# ── 1. cayce-ops (main) ────────────────────────────────────────────────────── +cd /home/ubuntu/cayce-ops || { say "FATAL: sin ~/cayce-ops"; exit 1; } +git fetch -q origin 2>/dev/null +if [ "$(git branch --show-current)" = "main" ] && ! git merge-base --is-ancestor origin/main HEAD 2>/dev/null; then + if [ -n "$(git status --porcelain)" ]; then + say "ABORT: cayce-ops tree sucio"; dm "abortado: cayce-ops dirty" + elif git pull -q --ff-only origin main; then + say "cayce-ops → $(git rev-parse --short HEAD)" + # r21: sin DM (no urgente) — dm "cayce-ops actualizado a $(git rev-parse --short HEAD)" + else + say "ERROR: pull cayce-ops"; dm "error pull cayce-ops" + fi +fi + +# ── 2. entity → /srv/cayce-core (con regla de urgencia) ────────────────────── +cd /srv/cayce-core || { say "FATAL: sin /srv/cayce-core"; exit 1; } +git fetch -q origin 2>/dev/null +# desplegar solo si main tiene commits que NO estan en el checkout +# r20: sin exit 0 — mataba el script y la sección bitcoinhotels nunca corría +if ! git merge-base --is-ancestor origin/main HEAD 2>/dev/null; then +# r25: lessons.md lo muta el ciclo retro en produccion — preservar (stash/pop). +# Cualquier OTRO fichero sucio si aborta. +DIRTY_OTHER=$(git status --porcelain | grep -v " lessons.md" | head -3) +if [ -n "$DIRTY_OTHER" ]; then + say "ABORT: /srv/cayce-core tree sucio: $DIRTY_OTHER"; dm "abortado: /srv/cayce-core dirty ($DIRTY_OTHER) — deploy pendiente" + exit 1 +fi +git stash push -q -m "autodeploy-lessons" -- lessons.md 2>/dev/null || true + +OLD=$(git rev-parse HEAD) +echo "autodeploy $(date -u +%FT%TZ)" > PUBLISH_OFF # r21: firmado para que urgent-watch no lo tome por manual +say "main nuevo detectado: $(git rev-parse --short HEAD) → $(git rev-parse --short origin/main) (PUBLISH_OFF activo)" + +if ! git checkout -q main || ! git reset -q --hard origin/main; then + git reset -q --hard "$OLD" 2>/dev/null; rm -f PUBLISH_OFF + say "FATAL: checkout entity"; dm "FATAL checkout entity — deploy cancelado, sin cambios" + exit 1 +fi + +# verificación 1: tests del guard +OUT=$(cd /srv/cayce-core && node --test guard.test.mjs 2>&1) +PASS=$(echo "$OUT" | grep -c '^ok ') +FAIL=$(echo "$OUT" | grep -c '^not ok ') +if [ "$FAIL" != "0" ] || [ "$PASS" -lt 27 ]; then + git reset -q --hard "$OLD" + git stash pop -q 2>/dev/null || true + say "ROLLBACK: tests $PASS ok / $FAIL fail (<27 o fail>0)" + dm "ROLLBACK en deploy entity: tests $PASS/$FAIL. main NO desplegado; runtime en $(git rev-parse --short HEAD)" + exit 1 +fi + +# verificación 2: daemon + watcher limpio +sudo -n systemctl restart cayce-nostr.service +sudo -n systemctl start cayce-relay-watcher.service +W=0 +for i in 1 2 3 4 5 6; do + sleep 5 + [ "$(systemctl is-active cayce-relay-watcher.service)" = "inactive" ] && break +done +if systemctl is-failed cayce-relay-watcher.service >/dev/null 2>&1 \ + || journalctl -u cayce-relay-watcher -n 8 --no-pager 2>/dev/null | grep -q VIOLACION; then + git reset -q --hard "$OLD" + sudo -n systemctl restart cayce-nostr.service + say "ROLLBACK: watcher con VIOLACION — PUBLISH_OFF queda ACTIVO" + dm "ROLLBACK watcher en deploy entity. PUBLISH_OFF QUEDA ACTIVO — revision manual" + exit 1 # ojo: PUBLISH_OFF se queda (fail-safe deliberado) +fi + +rm -f PUBLISH_OFF +git stash pop -q 2>/dev/null || true # lessons.md del ciclo retro +say "entity → $(git rev-parse --short HEAD) (tests $PASS/27, watcher OK, live)" + # r21: sin DM (no urgente) — dm "entity desplegado: $(git rev-parse --short HEAD) — tests $PASS/27, watcher OK" +fi # fin sección entity + +# ── 2b. hardcap-news → /srv/hardcap-news (GitHub; web, sin PUBLISH_OFF) ───── +cd /srv/hardcap-news || { say "FATAL: sin /srv/hardcap-news"; exit 1; } +git fetch -q origin 2>/dev/null +if ! git merge-base --is-ancestor origin/main HEAD 2>/dev/null && [ -z "$(git status --porcelain)" ]; then + OLD=$(git rev-parse HEAD) + if ! git checkout -q main || ! git reset -q --hard origin/main; then + git reset -q --hard "$OLD" 2>/dev/null; say "FATAL: checkout hardcap-news"; exit 1 + fi + if git diff --name-only "$OLD" HEAD 2>/dev/null | grep -q "^frontend/package-lock.json$"; then + (cd frontend && npm ci --ignore-scripts >/dev/null 2>&1) || say "WARN: npm ci hardcap" + fi + if ! (cd frontend && npm run build >/tmp/hc-autodeploy-build.log 2>&1); then + git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service + say "ROLLBACK hardcap-news: fallo el build"; dm "ROLLBACK hardcap-news (build) — main NO desplegado, rollback activo" + exit 1 + fi + sudo -n systemctl restart hardcap-frontend.service + sleep 6 + CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3003/) + if [ "$CODE" != "200" ]; then + git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service + say "ROLLBACK hardcap-news: verificacion (code=$CODE)"; dm "ROLLBACK hardcap-news verificacion: code=$CODE — rollback activo" + exit 1 + fi + # r24: identidad por build-id (cuando el despliegue trae la función) + if git grep -q "NEXT_PUBLIC_BUILD_ID" -- frontend/src/app/layout.tsx frontend/next.config.js 2>/dev/null; then + BUILT=$(curl -s -m 8 http://127.0.0.1:3003/ | grep -oE 'name="build-id" content="[a-f0-9]{8}"' | grep -oE '[a-f0-9]{8}' | head -1) + WANT=$(git rev-parse --short=8 HEAD) + if [ "$BUILT" != "$WANT" ]; then + git reset -q --hard "$OLD"; (cd frontend && npm run build >/dev/null 2>&1); sudo -n systemctl restart hardcap-frontend.service + say "ROLLBACK hardcap-news: build-id '$BUILT' != '$WANT'"; dm "ROLLBACK hardcap-news: servia build '$BUILT' pero lo desplegado era '$WANT'." + exit 1 + fi + say "identidad hardcap OK: servido=$BUILT desplegado=$WANT" + fi + say "hardcap-news → $(git rev-parse --short HEAD) (200)" +fi + +# ── 3. bitcoinhotels → /srv/bitcoinhotels (web; sin PUBLISH_OFF) ──────────── +cd /srv/bitcoinhotels || { say "FATAL: sin /srv/bitcoinhotels"; exit 1; } +git fetch -q origin 2>/dev/null +if ! git merge-base --is-ancestor origin/main HEAD 2>/dev/null && [ -z "$(git status --porcelain)" ]; then + OLD=$(git rev-parse HEAD) + if ! git checkout -q main || ! git reset -q --hard origin/main; then + git reset -q --hard "$OLD" 2>/dev/null; say "FATAL: checkout bitcoinhotels"; dm "FATAL checkout bitcoinhotels — sin cambios"; exit 1 + fi + if git diff --name-only "$OLD" HEAD 2>/dev/null | grep -q "^package-lock.json$"; then + npm ci --ignore-scripts >/dev/null 2>&1 || { say "WARN: npm ci bitcoinhotels"; } + fi + if ! npm run build >/tmp/bh-autodeploy-build.log 2>&1; then + git reset -q --hard "$OLD"; npm run build >/dev/null 2>&1; systemctl restart cayce-bitcoinhotels + say "ROLLBACK bitcoinhotels: fallo el build"; dm "ROLLBACK bitcoinhotels (build) — main NO desplegado, rollback activo" + exit 1 + fi + sudo -n systemctl restart cayce-bitcoinhotels + sleep 6 + CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3002/) + UMAMI=$(curl -s http://127.0.0.1:3002/ | grep -c umami) + MARCAS=$(curl -s http://127.0.0.1:3002/guides/how-to-pay-with-lightning-abroad | grep -cE "Trezor|Ledger|BitBox|Coldcard") + if [ "$CODE" != "200" ] || [ "$UMAMI" = "0" ] || [ "$MARCAS" != "0" ]; then + git reset -q --hard "$OLD"; npm run build >/dev/null 2>&1; systemctl restart cayce-bitcoinhotels + say "ROLLBACK bitcoinhotels: verificacion (code=$CODE umami=$UMAMI marcas=$MARCAS)" + dm "ROLLBACK bitcoinhotels verificacion: code=$CODE umami=$UMAMI marcas=$MARCAS — rollback activo" + exit 1 + fi + say "bitcoinhotels → $(git rev-parse --short HEAD) (200 + umami + 0 marcas)" + # r22: verificación de IDENTIDAD — lo servido debe ser lo desplegado + if git grep -q "NEXT_PUBLIC_BUILD_ID" -- src/app/layout.tsx 2>/dev/null; then + BUILT=$(curl -s -m 8 http://127.0.0.1:3002/ | grep -oE 'name="build-id" content="[a-f0-9]{8}"' | grep -oE '[a-f0-9]{8}' | head -1) + WANT=$(git rev-parse --short=8 HEAD) + if [ "$BUILT" != "$WANT" ]; then + git reset -q --hard "$OLD"; npm run build >/dev/null 2>&1; sudo -n systemctl restart cayce-bitcoinhotels + say "ROLLBACK bitcoinhotels: build-id servido '$BUILT' != desplegado '$WANT'" + dm "ROLLBACK bitcoinhotels: el sitio servia build '$BUILT' pero lo desplegado era '$WANT' — rollback activo." + exit 1 + fi + say "identidad OK: servido=$BUILT desplegado=$WANT" + fi +fi