#!/usr/bin/env bash
# Deploy the command-stream stack to a CPU streaming server (London VPS).
# Run from the developer machine. The box must already be provisioned
# (packages, wine, runtime, engine profiles, replays, service user).
set -euo pipefail

APP_ROOT=$(cd "$(dirname "$0")/.." && pwd)      # experiments/d3d9-command-stream
REPO_ROOT=$(cd "$APP_ROOT/../.." && pwd)
# How many seats this box serves. One number drives the prefixes, the gate
# slot directories, the systemd instances, the claim image matrix, nginx's
# seat map and the page's seat probe — they used to be four hand-kept lists.
# Seat N binds relay port 8144+N, so 6 is NOT usable here: 8150 belongs to
# w3cs-web. Measured on the 4 vCPU / 7.8 GB London box: engine ~610 MB and
# relay ~130-220 MB per session, and games plus relays reach ~2.9 of 4
# cores at six sessions. Five is the practical ceiling; beyond it frame
# rate falls below the 40 fps target and the box starts swapping.
SEAT_COUNT=${W3CS_SEAT_COUNT:-5}
SEATS=$(seq 1 "$SEAT_COUNT")
HOST=${W3CS_CPU_HOST:-danny@82.39.133.153}
# The box's public DNS name: nginx server_name and the Let's Encrypt cert.
DOMAIN=${W3CS_CPU_DOMAIN:-london-cpu-worker.war3replays.com}
# Every CPU worker in the pool, space separated. Written to /workers.json on
# this box so its pages can spread new viewers over all workers (most free
# seats wins; see web/worker-pool.js). Same list on every box.
WORKER_HOSTS=${W3CS_WORKER_HOSTS:-"london-cpu-worker.war3replays.com london-cpu-worker-2.war3replays.com"}
SSH=(ssh -o BatchMode=yes "$HOST")
RSYNC=(rsync -a --delete -e "ssh -o BatchMode=yes" --rsync-path="sudo rsync")

echo "=== sync app tree"
"${RSYNC[@]}" --exclude .git --exclude "*.log" --exclude __pycache__ \
  "$APP_ROOT/" "$HOST:/home/ubuntu/w3cs-lab/app/"

echo "=== sync infra (criu build inputs)"
"${SSH[@]}" sudo mkdir -p /home/ubuntu/war3-repo/infra/runtime \
  /home/ubuntu/war3-repo/infra/patches
"${RSYNC[@]}" --exclude __pycache__ "$REPO_ROOT/infra/runtime/" \
  "$HOST:/home/ubuntu/war3-repo/infra/runtime/"
"${RSYNC[@]}" "$REPO_ROOT/infra/patches/" \
  "$HOST:/home/ubuntu/war3-repo/infra/patches/"

echo "=== remote build + install"
# Quoted heredoc: nothing expands locally, so the seat count travels as an
# argument rather than as an interpolated value.
# ssh joins its arguments into one command string that the remote shell
# re-parses: quote every argument.
# shellcheck disable=SC2086
REMOTE_ARGS=$(printf '%q ' "$SEAT_COUNT" "$DOMAIN" "$(echo $WORKER_HOSTS)")
"${SSH[@]}" "sudo bash -s $REMOTE_ARGS" <<'REMOTE'
set -euo pipefail
SEAT_COUNT=${1:?seat count}
DOMAIN=${2:?domain}
WORKER_HOSTS=${3:?worker hosts}
SEATS=$(seq 1 "$SEAT_COUNT")
LAB=/home/ubuntu/w3cs-lab
APP=$LAB/app

# Relay (native) and the game-side proxy DLL.
make -C "$APP/native" d3d9.dll
g++ -O3 -g0 -std=c++20 -Wall -Wextra -Werror -pthread \
  -DGST_USE_UNSTABLE_API "$APP/native/w3cs_webrtc_relay.cpp" \
  -o "$APP/native/w3cs-webrtc-relay" \
  $(pkg-config --cflags --libs gstreamer-1.0 gstreamer-webrtc-1.0 \
    gstreamer-sdp-1.0 libsoup-3.0 json-glib-1.0 x11 xfixes xtst zlib libzstd)

# WebTransport bridge: browser QUIC sessions onto the relay's local
# WebSocket planes. Needs the Go toolchain (one-time provisioning at
# /usr/local/go); module downloads hit the local cache after first build.
export PATH=$PATH:/usr/local/go/bin
if command -v go >/dev/null; then
  (cd "$APP/native/wt-bridge" && GOFLAGS=-mod=mod go build \
    -o /usr/local/sbin/w3cs-wt-bridge .)
  install -m 0644 "$APP/deploy/w3cs-wt-bridge.service" /etc/systemd/system/
  ufw allow 4443/udp >/dev/null
else
  echo "WARN: Go toolchain missing at /usr/local/go; wt-bridge not built"
fi

# lab-live.sh expects its collaborators flat in LAB_ROOT.
install -m 0755 "$APP/native/w3cs-webrtc-relay" "$LAB/w3cs-webrtc-relay"
install -m 0644 "$APP/native/d3d9.dll" "$LAB/d3d9.dll"
install -m 0755 "$APP/native/lab-live.sh" "$APP/native/lab-game-session.sh" \
  "$APP/native/lab-switch-replay.sh" "$LAB/"
install -m 0644 "$APP/native/stage-replay-session.py" \
  "$APP/native/materialize-catalog-replay.py" \
  "$APP/lab_live_bridge.py" "$APP/protocol.py" "$APP/relay.py" "$LAB/"

# Test index data: one entry per staged replay, grouped by engine profile.
python3 - <<'PY'
import json, glob, os
out = []
for p in sorted(glob.glob('/home/ubuntu/w3cs-lab/replays/*.json')):
    rid = os.path.basename(p)[:-5]
    with open(p) as f:
        d = json.load(f)
    entry = {'id': rid,
             'profile': d.get('engineProfile', 'native-1285'),
             'map': d.get('mapPath', '')}
    if d.get('displayName'):
        entry['name'] = d['displayName']
    out.append(entry)
with open('/home/ubuntu/w3cs-lab/app/replays.json', 'w') as f:
    json.dump(out, f, indent=1)
print(f"replays.json: {len(out)} entries")
PY
# Worker pool for the pages (see WORKER_HOSTS above).
python3 - "$SEAT_COUNT" $WORKER_HOSTS <<'PY'
import json, sys
seats = int(sys.argv[1])
workers = [{"host": host, "seats": seats} for host in sys.argv[2:]]
with open('/home/ubuntu/w3cs-lab/app/workers.json', 'w') as f:
    json.dump({"workers": workers}, f, indent=1)
print("workers.json:", ", ".join(w["host"] for w in workers))
PY

# TLS front (secure context for WebGPU): nginx + self-signed IP cert until
# a real domain points at the box.
command -v nginx >/dev/null || {
  DEBIAN_FRONTEND=noninteractive apt-get -yq install nginx >/dev/null
}
mkdir -p /etc/ssl/w3cs /var/www/certbot
if [ -d "/etc/letsencrypt/live/$DOMAIN" ]; then
  ln -sf "/etc/letsencrypt/live/$DOMAIN/fullchain.pem" /etc/ssl/w3cs/w3cs.crt
  ln -sf "/etc/letsencrypt/live/$DOMAIN/privkey.pem" /etc/ssl/w3cs/w3cs.key
elif [ ! -f /etc/ssl/w3cs/w3cs.crt ]; then
  BOX_IP=$(ip -4 -br addr show scope global | awk '{print $3}' | cut -d/ -f1 | head -1)
  openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \
    -keyout /etc/ssl/w3cs/w3cs.key -out /etc/ssl/w3cs/w3cs.crt \
    -days 3650 -nodes -subj "/CN=w3cs-cpu-server" \
    -addext "subjectAltName=IP:$BOX_IP" 2>/dev/null
  chmod 600 /etc/ssl/w3cs/w3cs.key
fi
# Per-IP viewer gate allowlist (geo include in nginx-w3cs.conf). Box-local:
# the deploy never overwrites it, only guarantees the include target exists
# so nginx -t passes on a fresh box. Add developer addresses by hand, see
# deploy/w3cs-trusted-ips.conf.example.
[ -f /etc/nginx/w3cs-trusted-ips.conf ] || \
  install -m 0644 /dev/null /etc/nginx/w3cs-trusted-ips.conf
# The view-cap allowlist is operator state (test IPs), not repo state:
# create it empty when missing, never overwrite it.
install -d -m 0755 /etc/w3cs
[ -f /etc/w3cs/w3cs-view-allowlist.conf ] || : > /etc/w3cs/w3cs-view-allowlist.conf
# The repo config names the first London box; every sibling gets its own
# server_name here.
sed "s/london-cpu-worker\.war3replays\.com/$DOMAIN/" \
  "$APP/deploy/nginx-w3cs.conf" > /etc/nginx/conf.d/w3cs.conf
chmod 0644 /etc/nginx/conf.d/w3cs.conf
rm -f /etc/nginx/sites-enabled/default
nginx -t
systemctl enable --now nginx >/dev/null 2>&1
systemctl reload nginx
# First deploy on a new box: obtain the Let's Encrypt cert now that nginx
# answers HTTP-01 on port 80 (self-signed until here), then switch to it.
# Same registration as the first box (no contact email); renewals reload
# nginx through the deploy hook. W3CS_CERTBOT=0 skips this.
if [ ! -d "/etc/letsencrypt/live/$DOMAIN" ] && command -v certbot >/dev/null \
    && [ "${W3CS_CERTBOT:-1}" != 0 ]; then
  if certbot certonly --webroot -w /var/www/certbot -d "$DOMAIN" \
      --non-interactive --agree-tos --register-unsafely-without-email \
      --deploy-hook "systemctl reload nginx"; then
    ln -sf "/etc/letsencrypt/live/$DOMAIN/fullchain.pem" /etc/ssl/w3cs/w3cs.crt
    ln -sf "/etc/letsencrypt/live/$DOMAIN/privkey.pem" /etc/ssl/w3cs/w3cs.key
    nginx -t && systemctl reload nginx
  else
    echo "WARN: certbot failed; nginx keeps the self-signed certificate"
  fi
fi

install -m 0755 "$APP/deploy/w3cs-watchdog.sh" /usr/local/sbin/w3cs-watchdog.sh
install -m 0755 "$APP/deploy/w3cs-bundle-upload.sh" /usr/local/sbin/w3cs-bundle-upload.sh
install -m 0755 "$APP/deploy/w3cs-pack-build.py" /usr/local/sbin/w3cs-pack-build.py
install -m 0755 "$APP/deploy/w3cs-seat-launch.sh" /usr/local/sbin/w3cs-seat-launch.sh
# On-demand catalog: the compact serving index comes from the R2 mirror and
# is refreshed by a timer; the first fetch runs now so the seats can serve
# any catalog sha1 right after this deploy.
install -m 0755 "$APP/deploy/w3cs-catalog-index.sh" /usr/local/sbin/w3cs-catalog-index.sh
install -m 0644 "$APP/deploy/w3cs-catalog-index.service" \
  "$APP/deploy/w3cs-catalog-index.timer" /etc/systemd/system/
systemctl daemon-reload
systemctl enable --now w3cs-catalog-index.timer >/dev/null 2>&1
/usr/local/sbin/w3cs-catalog-index.sh || echo "WARN: serving index refresh failed"

# CRIU claim fast path: the shared worker (from infra/runtime), the layer
# base builder, this pipeline's image-matrix builder, and the sudoers rule
# that lets the (ubuntu) session scripts restore and stop claims. Image
# builds are fingerprinted, so the matrix run below only rebuilds what the
# new proxy DLL or seat environment actually invalidates.
# The infra tree is synced to /home/ubuntu/war3-repo (see the rsync above),
# NOT beside the app tree: "$APP/../../infra" resolved to /home/ubuntu/infra,
# which does not exist, so this block was silently skipped on every deploy
# and the installed zoom patcher / CRIU worker stayed at whatever was last
# copied by hand (found 2026-08-21 when the 1.10 lane's image build ran an
# old patcher and aborted the whole matrix).
INFRA_RUNTIME=/home/ubuntu/war3-repo/infra/runtime
if [ -f "$INFRA_RUNTIME/classic-criu-worker.sh" ]; then
  install -m 0755 "$INFRA_RUNTIME/classic-criu-worker.sh" \
    /usr/local/sbin/war3-classic-criu
  install -m 0755 "$INFRA_RUNTIME/build-classic-layer-base.sh" \
    /usr/local/sbin/war3-build-classic-layer-base
  install -d -m 0755 /usr/local/libexec/war3
  install -m 0755 "$INFRA_RUNTIME/w3cs-camera-zoom-patch.py" \
    /usr/local/libexec/war3/w3cs-camera-zoom-patch.py
else
  echo "WARN: $INFRA_RUNTIME/classic-criu-worker.sh missing; claim tooling not refreshed"
fi
install -m 0755 "$APP/deploy/w3cs-criu-build-images.sh" \
  /usr/local/sbin/w3cs-criu-build-images
cat > /etc/sudoers.d/w3cs-criu <<'SUDOERS'
# The w3cs session scripts (running as ubuntu) restore and stop CRIU claim
# checkpoints; criu itself requires root. SETENV lets the caller pass the
# W3_* configuration; the worker script validates every path and mode it
# accepts and refuses anything outside its contract.
ubuntu ALL=(root) NOPASSWD:SETENV: /usr/local/sbin/war3-classic-criu
SUDOERS
chmod 440 /etc/sudoers.d/w3cs-criu
visudo -c -f /etc/sudoers.d/w3cs-criu >/dev/null
# tmpfs directories the claim path needs at boot. Without this contract
# the replay-open gate directory only existed as a leftover of the last
# image build, and the first reboot left every boot-time claim dying on
# the missing release-marker path (seats up, engines absent). The
# restore worker also recreates its own slot directory; this covers the
# first boot before any restore has run.
{
  echo "d /run/war3-replay-gate 0755 root root -"
  for SEAT in $SEATS; do
    echo "d /run/war3-replay-gate/slot$SEAT 0755 ubuntu ubuntu -"
  done
  echo "d /run/war3-no-dbus 0755 root root -"
} > /etc/tmpfiles.d/w3cs-war3.conf
systemd-tmpfiles --create /etc/tmpfiles.d/w3cs-war3.conf
install -m 0644 "$APP/deploy/w3cs-live@.service" \
  "$APP/deploy/w3cs-pulse.service" \
  "$APP/deploy/w3cs-web.service" "$APP/deploy/w3cs-watchdog.service" \
  "$APP/deploy/w3cs-watchdog.timer" \
  "$APP/deploy/w3cs-bundle-upload.service" \
  "$APP/deploy/w3cs-bundle-upload.timer" /etc/systemd/system/

# Per-seat wine prefixes (independent games need independent prefixes; the
# Documents/Replays staging path and the wineserver are per prefix).
for SEAT in $SEATS; do
  P32="/opt/war3-runtime/classic/prefix-seat$SEAT"
  if [ ! -d "$P32" ]; then
    cp -a /opt/war3-runtime/classic/prefix "$P32"
    chown -R ubuntu:ubuntu "$P32"
  fi
  P64="/opt/war3-runtime/classic-131/prefix64-seat$SEAT"
  if [ ! -d "$P64" ] && [ -d /opt/war3-runtime/classic-131/prefix64 ]; then
    cp -a /opt/war3-runtime/classic-131/prefix64 "$P64"
    chown -R ubuntu:ubuntu "$P64"
  fi
done

ufw allow 40000:40599/udp >/dev/null

chown -R ubuntu:ubuntu "$LAB" /home/ubuntu/war3-repo

systemctl daemon-reload
# The singleton unit is replaced by per-seat template instances.
systemctl disable --now w3cs-live.service >/dev/null 2>&1 || true
rm -f /etc/systemd/system/w3cs-live.service
systemctl daemon-reload
SEAT_UNITS=$(for SEAT in $SEATS; do printf "w3cs-live@%s " "$SEAT"; done)
systemctl enable w3cs-pulse w3cs-web $SEAT_UNITS \
  w3cs-watchdog.timer >/dev/null 2>&1
# Bundle uploads need /etc/w3cs/rclone.conf (root-only R2 keys, provisioned
# out of band); the timer is harmless without it - the unit just fails.
if [ -f /etc/w3cs/rclone.conf ]; then
  systemctl enable --now w3cs-bundle-upload.timer >/dev/null 2>&1
fi
systemctl restart w3cs-pulse
if [ -x /usr/local/sbin/w3cs-wt-bridge ]; then
  systemctl enable w3cs-wt-bridge >/dev/null 2>&1
  systemctl restart w3cs-wt-bridge
fi
systemctl restart w3cs-web
systemctl restart $SEAT_UNITS
systemctl start w3cs-watchdog.timer
# Rebuild claim images the new proxy DLL or seat environment invalidated
# (fingerprinted no-ops otherwise). Serializes per seat with its own
# stop/start, so this runs after the seats are already serving.
if [ -x /usr/local/libexec/war3/criu ]; then
  /usr/local/sbin/w3cs-criu-build-images || \
    echo "WARN: claim image build failed; seats fall back to cold launches"
fi
sleep 2
systemctl --no-pager --plain status w3cs-web "w3cs-live@*" \
  | grep -E "service|Active"
REMOTE

echo "=== deployed. Test page: https://$DOMAIN/poc/cpu-test-index.html"