שרת לסוכני AI שלא נופל ולא נפרץ

ssh-keygen -t ed25519 -C "my-laptop"
ssh-copy-id -i ~/.ssh/id_ed25519.pub root@SERVER_IP

sudo adduser --gecos "" ops
sudo usermod -aG sudo ops
sudo install -d -m 700 -o ops -g ops /home/ops/.ssh
sudo install -m 600 -o ops -g ops ~/.ssh/authorized_keys /home/ops/.ssh/authorized_keys

# /etc/ssh/sshd_config.d/00-hardening.conf
# sshd uses the FIRST value it reads, and files here are read in name order,
# so "00-" wins over a cloud image's "50-cloud-init.conf".
PasswordAuthentication no
KbdInteractiveAuthentication no
PubkeyAuthentication yes
PermitRootLogin prohibit-password
MaxAuthTries 3
LoginGraceTime 30
X11Forwarding no

sudo sshd -t && sudo systemctl restart ssh

sudo sshd -T | grep -E '^(passwordauthentication|kbdinteractiveauthentication|permitrootlogin)'
# permitrootlogin prohibit-password    (או no)
# passwordauthentication no
# kbdinteractiveauthentication no

ssh -o PubkeyAuthentication=no -o PreferredAuthentications=password ops@SERVER_IP
# ops@SERVER_IP: Permission denied (publickey).

sudo ufw allow OpenSSH
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw enable

# /etc/fail2ban/jail.local
[DEFAULT]
bantime = 1h
findtime = 10m
maxretry = 5
backend = systemd

[sshd]
enabled = true

sudo apt install -y fail2ban
sudo cp jail.local /etc/fail2ban/jail.local
sudo systemctl restart fail2ban

// /etc/apt/apt.conf.d/20auto-upgrades
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
APT::Periodic::AutocleanInterval "7";

sudo apt install -y unattended-upgrades
sudo cp 20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades

apt-config dump | grep Periodic::Unattended-Upgrade
# APT::Periodic::Unattended-Upgrade "1";
sudo unattended-upgrade --dry-run; echo "exit=$?"
# exit=0

sudo install -d -m 755 /etc/my-agent
sudo install -m 600 -o root -g root /dev/null /etc/my-agent/agent.env
sudoedit /etc/my-agent/agent.env

# /etc/my-agent/agent.env   owner root, chmod 600. Never in git, never in code, never printed to logs.
TELEGRAM_TOKEN=put-your-bot-token-here
API_KEY=put-your-model-api-key-here
HEARTBEAT_FILE=/var/lib/my-agent/heartbeat

.env
.env.*
*.pem
*.key
secrets/

sudo stat -c '%a %U %n' /etc/my-agent/agent.env
# 600 root /etc/my-agent/agent.env
git check-ignore -v .env
# .gitignore:1:.env	.env
git grep -nIE '(api[_-]?key|token|secret|password)\s*[:=]\s*["'\'']?[A-Za-z0-9_\-:]{16,}'
# (בלי פלט)

sudo useradd --system --create-home --home-dir /opt/my-agent --shell /usr/sbin/nologin agent
sudo install -d -o agent -g agent /var/lib/my-agent

#!/usr/bin/env bash
set -euo pipefail
cd /opt/my-agent
exec python3 bot.py        # או: exec node index.js

[Unit]
Description=My AI agent
Wants=network-online.target
After=network-online.target
# Stop trying after 5 failed starts in 10 minutes (the watchdog will tell you)
StartLimitIntervalSec=600
StartLimitBurst=5

[Service]
Type=simple
User=agent
Group=agent
WorkingDirectory=/opt/my-agent
EnvironmentFile=/etc/my-agent/agent.env
ExecStart=/opt/my-agent/run.sh
Restart=on-failure
RestartSec=10
# Keep one runaway process from taking the whole server down
MemoryMax=1500M
MemorySwapMax=0
# Basic sandboxing: the agent may write only where it needs to
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ProtectHome=read-only
ReadWritePaths=/opt/my-agent /var/lib/my-agent

[Install]
WantedBy=multi-user.target

sudo chmod 755 /opt/my-agent/run.sh
sudo cp my-agent.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now my-agent

systemctl is-enabled my-agent; systemctl is-active my-agent
# enabled
# active
sudo systemctl kill -s KILL my-agent; sleep 15; systemctl is-active my-agent
# active        (systemd הקים אותו אחרי ההריגה)
sudo reboot     # ואחרי שהשרת עולה:
systemctl is-active my-agent
# active

# bash
touch "$HEARTBEAT_FILE"
# Python
pathlib.Path(os.environ["HEARTBEAT_FILE"]).touch()
# Node.js
fs.writeFileSync(process.env.HEARTBEAT_FILE, String(Date.now()))

[Unit]
Description=My AI agent (interactive, inside tmux)
Wants=network-online.target
After=network-online.target

[Service]
Type=forking
User=agent
Group=agent
WorkingDirectory=/opt/my-agent
EnvironmentFile=/etc/my-agent/agent.env
# Attach later with: sudo -u agent tmux attach -t agent
ExecStart=/usr/bin/tmux new-session -d -s agent /opt/my-agent/run.sh
ExecStop=/usr/bin/tmux kill-session -t agent
# tmux exits cleanly when the agent quits, so restart on any exit
Restart=always
RestartSec=15

[Install]
WantedBy=multi-user.target

#!/usr/bin/env bash
# Agent watchdog: checks the agent and the server, alerts on Telegram once per problem,
# repeats only every REPEAT_MIN minutes, and sends a recovery message when it is fixed.
# Config: /etc/my-agent/watchdog.env (chmod 600). DRY_RUN=1 prints instead of sending.
set -uo pipefail

SERVICE="${SERVICE:-my-agent.service}"            # the agent's systemd unit
HEARTBEAT_FILE="${HEARTBEAT_FILE:-/var/lib/my-agent/heartbeat}"
HEARTBEAT_MAX_MIN="${HEARTBEAT_MAX_MIN:-15}"      # the agent touches this file at least this often
HEALTH_URL="${HEALTH_URL:-}"                      # optional: an HTTP health endpoint of the agent
JOB_UNIT="${JOB_UNIT:-}"                          # optional: the scheduled job whose last result to check
DISK_MAX_PCT="${DISK_MAX_PCT:-85}"
MEM_MIN_AVAIL_MB="${MEM_MIN_AVAIL_MB:-300}"
SWAP_MAX_PCT="${SWAP_MAX_PCT:-80}"
REPEAT_MIN="${REPEAT_MIN:-60}"                    # repeat an open alert at most this often
AUTO_RESTART="${AUTO_RESTART:-1}"                 # restart the agent once when it is down or silent
STATE_DIR="${STATE_DIR:-/var/lib/my-agent-watchdog}"
MEMINFO="${MEMINFO:-/proc/meminfo}"
HOST_LABEL="${HOST_LABEL:-$(hostname -s)}"
DRY_RUN="${DRY_RUN:-0}"

mkdir -p "$STATE_DIR"

send() {
  local text="$1"
  if [[ "$DRY_RUN" == "1" ]]; then
    printf 'DRY-RUN message: %s\n' "$text"
    return 0
  fi
  if [[ -z "${TELEGRAM_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then
    echo "watchdog: TELEGRAM_TOKEN or TELEGRAM_CHAT_ID missing" >&2
    return 1
  fi
  # The URL (with the token) goes through stdin, so it never shows up in `ps` or in logs
  printf 'url = "https://api.telegram.org/bot%s/sendMessage"\n' "$TELEGRAM_TOKEN" |
    curl -sS --max-time 15 -o /dev/null -w '%{http_code}' -K - \
      --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
      --data-urlencode "text=${text}" | grep -q '^200$' || echo "watchdog: Telegram send failed" >&2
}

# Remember each open problem in a file: alert when it opens, repeat every REPEAT_MIN, recover when it closes
problem() {   # problem <key> <message>
  local key="$1" msg="$2" f="$STATE_DIR/$1.open"
  if [[ ! -f "$f" ]] || [[ -n "$(find "$f" -mmin +"$REPEAT_MIN" 2>/dev/null)" ]]; then
    send "⚠️ ${HOST_LABEL}: ${msg}" && printf '%s\n' "$msg" > "$f"
  fi
  OPEN+=("$key")
}
resolved() {  # resolved <key> <message>
  local f="$STATE_DIR/$1.open"
  if [[ -f "$f" ]]; then
    send "✅ ${HOST_LABEL}: ${2}" && rm -f "$f"
  fi
}

OPEN=()
restart_once() {
  [[ "$AUTO_RESTART" == "1" ]] || return 0
  local f="$STATE_DIR/restarted"
  # at most one automatic restart per REPEAT_MIN, so a broken agent is not restarted in a loop
  if [[ ! -f "$f" ]] || [[ -n "$(find "$f" -mmin +"$REPEAT_MIN" 2>/dev/null)" ]]; then
    touch "$f"
    if [[ "$DRY_RUN" == "1" ]]; then echo "DRY-RUN restart: $SERVICE"; else systemctl restart "$SERVICE"; fi
  fi
}

# 1. The service is running
if systemctl is-active --quiet "$SERVICE"; then
  resolved service "$SERVICE is running again"
else
  problem service "$SERVICE is not running ($(systemctl is-active "$SERVICE" 2>/dev/null))"
  restart_once
fi

# 2. The agent is really alive: a fresh heartbeat file (and the health URL, if set)
if [[ -f "$HEARTBEAT_FILE" ]] && [[ -z "$(find "$HEARTBEAT_FILE" -mmin +"$HEARTBEAT_MAX_MIN" 2>/dev/null)" ]]; then
  resolved heartbeat "the agent's heartbeat is fresh again"
else
  problem heartbeat "no heartbeat from the agent for over ${HEARTBEAT_MAX_MIN} minutes"
  restart_once
fi
if [[ -n "$HEALTH_URL" ]]; then
  if curl -fsS --max-time 10 -o /dev/null "$HEALTH_URL"; then
    resolved health "the health check answers again"
  else
    problem health "the health check at $HEALTH_URL does not answer"
  fi
fi

# 3. Disk
disk_pct="$(df --output=pcent / | tail -1 | tr -dc '0-9')"
if (( disk_pct >= DISK_MAX_PCT )); then
  problem disk "disk is ${disk_pct}% full (limit ${DISK_MAX_PCT}%)"
else
  resolved disk "disk is back to ${disk_pct}%"
fi

# 4. Memory and swap
mem_avail_kb="$(awk '/^MemAvailable:/ {print $2}' "$MEMINFO")"
swap_total_kb="$(awk '/^SwapTotal:/ {print $2}' "$MEMINFO")"
swap_free_kb="$(awk '/^SwapFree:/ {print $2}' "$MEMINFO")"
mem_avail_mb=$(( ${mem_avail_kb:-0} / 1024 ))
if (( mem_avail_mb < MEM_MIN_AVAIL_MB )); then
  problem memory "only ${mem_avail_mb} MB RAM available (limit ${MEM_MIN_AVAIL_MB} MB)"
else
  resolved memory "RAM is back to ${mem_avail_mb} MB available"
fi
if (( ${swap_total_kb:-0} > 0 )); then
  swap_pct=$(( (swap_total_kb - swap_free_kb) * 100 / swap_total_kb ))
  if (( swap_pct >= SWAP_MAX_PCT )); then
    problem swap "swap is ${swap_pct}% used (limit ${SWAP_MAX_PCT}%)"
  else
    resolved swap "swap is back to ${swap_pct}%"
  fi
fi

# 5. The last run of the scheduled job
if [[ -n "$JOB_UNIT" ]]; then
  result="$(systemctl show -p Result --value "$JOB_UNIT" 2>/dev/null)"
  if [[ -z "$result" || "$result" == "success" ]]; then
    resolved job "$JOB_UNIT succeeded again"
  else
    problem job "the last run of $JOB_UNIT ended with: $result"
  fi
fi

if (( ${#OPEN[@]} == 0 )); then echo "watchdog: all ok"; else echo "watchdog: open: ${OPEN[*]}"; fi
exit 0

[Unit]
Description=Watchdog for my AI agent

[Service]
Type=oneshot
EnvironmentFile=/etc/my-agent/watchdog.env
ExecStart=/usr/local/bin/my-agent-watchdog
TimeoutStartSec=120

[Unit]
Description=Run the agent watchdog every 5 minutes

[Timer]
OnBootSec=3min
OnUnitActiveSec=5min
AccuracySec=30s

[Install]
WantedBy=timers.target

sudo install -m 755 watchdog.sh /usr/local/bin/my-agent-watchdog
sudo install -m 600 -o root -g root /dev/null /etc/my-agent/watchdog.env
sudoedit /etc/my-agent/watchdog.env

TELEGRAM_TOKEN=token-of-your-alerts-bot
TELEGRAM_CHAT_ID=your-chat-id
JOB_UNIT=my-job.service

sudo cp my-agent-watchdog.service my-agent-watchdog.timer /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now my-agent-watchdog.timer

sudo bash -c 'set -a; . /etc/my-agent/watchdog.env; DRY_RUN=1 /usr/local/bin/my-agent-watchdog'
# watchdog: all ok
systemctl list-timers my-agent-watchdog.timer --no-pager
# שורה עם NEXT בעוד פחות מ-5 דקות
sudo systemctl stop my-agent && sudo systemctl start my-agent-watchdog.service
# בטלגרם: ⚠️ ...: my-agent.service is not running (inactive)
# כלב השמירה מפעיל את הסוכן מחדש, ובהרצה הבאה: ✅ ...: my-agent.service is running again

sudo fallocate -l 2G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab

# /etc/sysctl.d/99-agent-swap.conf
# Use swap only when RAM is really short, instead of pushing the agent out early
vm.swappiness = 10

sudo cp 99-agent-swap.conf /etc/sysctl.d/ && sudo sysctl --system

# כמשתמש שלכם
systemd-run --user --scope -p MemoryMax=1500M -p MemorySwapMax=0 npm run build
# כ-root
sudo systemd-run --scope -p MemoryMax=1500M -p MemorySwapMax=0 apt-get -y upgrade

systemd-run --user --scope -p MemoryMax=200M -p MemorySwapMax=0 python3 -c "x = bytearray(400*1024*1024)"; echo "exit=$?"
# Killed
# exit=137        (והשרת ממשיך לעבוד)

journalctl -k --since "7 days ago" | grep -iE 'out of memory|oom-kill|killed process'
journalctl -u my-agent --since "7 days ago" | grep -i oom

[Unit]
Description=My scheduled agent job
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
User=agent
Group=agent
WorkingDirectory=/opt/my-agent
EnvironmentFile=/etc/my-agent/agent.env
# flock -n: if the previous run still holds the lock, this run exits (code 75) instead of overlapping,
# and 75 counts as success, so a skipped run is not reported as a failure
ExecStart=/usr/bin/flock -n -E 75 /var/lib/my-agent/my-job.lock /opt/my-agent/job.sh
SuccessExitStatus=75
# Kill a run that hangs, so the next one can start
TimeoutStartSec=30min
MemoryMax=1200M
MemorySwapMax=0

[Unit]
Description=Run my agent job every weekday at 07:30

[Timer]
OnCalendar=Mon..Fri 07:30
# Set the server's time zone, or write the zone here, e.g. OnCalendar=Mon..Fri 07:30 Asia/Jerusalem
# Run a missed run after boot (the server was off at 07:30)
Persistent=true
RandomizedDelaySec=2min

[Install]
WantedBy=timers.target

sudo cp my-job.service my-job.timer /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now my-job.timer

systemctl list-timers my-job.timer --no-pager
# NEXT: ביום העבודה הבא ב-07:30
sudo systemctl start my-job.service; journalctl -u my-job -n 20 --no-pager
# הפלט של המשימה, ובסוף: Finished my-job.service
systemctl show -p Result --value my-job.service
# success

{
  "permissions": {
    "allow": [
      "Bash(git status)",
      "Bash(git diff *)",
      "Bash(git log *)",
      "Bash(npm test *)",
      "Bash(npm run build *)"
    ],
    "deny": [
      "Bash(git push *)",
      "Bash(gh pr merge *)",
      "Bash(rm -rf *)",
      "Bash(sudo *)",
      "Bash(curl *)",
      "Bash(ssh *)",
      "Bash(scp *)",
      "Read(./.env)",
      "Read(./.env.*)",
      "Read(//etc/my-agent/**)"
    ]
  }
}

#!/usr/bin/env bash
# Human approval gate for outward actions (send email, post, merge, pay).
# The agent never runs these itself: it writes one file per action into the queue,
#   /var/lib/my-agent/outbox/<id>.cmd   (one command line)
#   /var/lib/my-agent/outbox/<id>.why   (one line: why)
# and you approve from your own shell:  approve.sh            (list)
#                                       approve.sh <id>       (show, ask, run, log)
# Install it so the agent cannot run it: sudo install -m 750 -o root -g sudo approve.sh /usr/local/bin/approve
set -euo pipefail
OUTBOX="${OUTBOX:-/var/lib/my-agent/outbox}"
AUDIT="${AUDIT:-/var/log/my-agent/audit.log}"
AGENT_USER="${AGENT_USER:-agent}"
RUN_AS="${RUN_AS:-$AGENT_USER}"   # approved commands run with the agent's rights, not yours
mkdir -p "$OUTBOX" "$(dirname "$AUDIT")"

# The agent must never approve its own actions
if [[ "$(id -un)" == "$AGENT_USER" ]]; then echo "refused: the agent user cannot approve"; exit 3; fi

log() { printf '%s\t%s\t%s\t%s\n' "$(date -u +%FT%TZ)" "${SUDO_USER:-$USER}" "$1" "$2" >> "$AUDIT"; }

if [[ $# -eq 0 ]]; then
  shopt -s nullglob
  for f in "$OUTBOX"/*.cmd; do
    id="$(basename "$f" .cmd)"
    printf '%s\t%s\n' "$id" "$(cat "$OUTBOX/$id.why" 2>/dev/null || echo '-')"
  done
  exit 0
fi

id="$1"
[[ "$id" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "bad id"; exit 2; }
cmd_file="$OUTBOX/$id.cmd"
[[ -f "$cmd_file" ]] || { echo "no such action: $id"; exit 2; }
echo "Why:     $(cat "$OUTBOX/$id.why" 2>/dev/null || echo '-')"
echo "Command: $(cat "$cmd_file")"
# A person at a terminal types the answer; piped input is refused
[[ -t 0 || "${APPROVE_TEST_NO_TTY:-0}" == "1" ]] || { echo "refused: run this from a terminal"; exit 3; }
read -r -p "Run it? Type yes to approve: " answer
if [[ "$answer" != "yes" ]]; then
  log "$id" "rejected: $(cat "$cmd_file")"
  mv "$cmd_file" "$cmd_file.rejected"
  echo "rejected"
  exit 0
fi
log "$id" "approved: $(cat "$cmd_file")"
if [[ "$RUN_AS" == "$(id -un)" ]]; then runner=(bash -c); else runner=(sudo -u "$RUN_AS" bash -c); fi
if "${runner[@]}" "$(cat "$cmd_file")"; then log "$id" "done"; mv "$cmd_file" "$cmd_file.done"; else log "$id" "failed"; exit 1; fi

sudo install -m 750 -o root -g sudo approve.sh /usr/local/bin/approve
sudo install -d -o agent -g agent /var/lib/my-agent/outbox
sudo install -d -m 750 /var/log/my-agent

sudo -u agent sudo -n true
# sudo: a password is required    (או: agent is not in the sudoers file)
sudo -u agent touch /etc/should-fail
# touch: cannot touch '/etc/should-fail': Permission denied
sudo approve                      # רשימת הפעולות שמחכות
sudo approve <id>                 # מציג, שואל, מריץ, רושם
sudo tail -3 /var/log/my-agent/audit.log

sudo apt install -y rclone
sudo rclone config          # יוצרים remote בשם offsite אצל ספק האחסון שבחרתם
sudo install -m 600 -o root -g root /dev/null /etc/my-agent/backup.pass
sudoedit /etc/my-agent/backup.pass     # משפט סיסמה ארוך

#!/usr/bin/env bash
# Nightly backup: configs, env files and data -> one encrypted archive -> off-server storage (rclone).
# Config: /etc/my-agent/backup.env (chmod 600): BACKUP_PATHS, RCLONE_REMOTE, PASSPHRASE_FILE, KEEP_DAYS
set -euo pipefail

BACKUP_PATHS="${BACKUP_PATHS:-/etc/my-agent /opt/my-agent /var/lib/my-agent /etc/systemd/system}"
RCLONE_REMOTE="${RCLONE_REMOTE:?set RCLONE_REMOTE, e.g. offsite:my-agent-backups}"
PASSPHRASE_FILE="${PASSPHRASE_FILE:-/etc/my-agent/backup.pass}"
KEEP_DAYS="${KEEP_DAYS:-30}"
WORK_DIR="${WORK_DIR:-/var/tmp/my-agent-backup}"
STAMP="$(date -u +%Y-%m-%dT%H%MZ)"
NAME="backup-${HOSTNAME:-host}-${STAMP}.tar.gz.gpg"

umask 077
mkdir -p "$WORK_DIR"
trap 'rm -f "$WORK_DIR/$NAME"' EXIT

# shellcheck disable=SC2086  # BACKUP_PATHS is a space-separated list on purpose
tar --create --gzip --file - --exclude='*.lock' --exclude='node_modules' --exclude='.cache' $BACKUP_PATHS 2>/dev/null |
  gpg --batch --yes --quiet --symmetric --cipher-algo AES256 --passphrase-file "$PASSPHRASE_FILE" \
      --output "$WORK_DIR/$NAME"

rclone copy --quiet "$WORK_DIR/$NAME" "$RCLONE_REMOTE/"
rclone delete --quiet --min-age "${KEEP_DAYS}d" "$RCLONE_REMOTE/"
echo "backup ok: $RCLONE_REMOTE/$NAME ($(du -h "$WORK_DIR/$NAME" | cut -f1))"

[Unit]
Description=Nightly encrypted backup of my AI agent
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
EnvironmentFile=/etc/my-agent/backup.env
ExecStart=/usr/local/bin/my-agent-backup
TimeoutStartSec=1h
Nice=10
IOSchedulingClass=idle

[Unit]
Description=Nightly backup at 03:15

[Timer]
OnCalendar=*-*-* 03:15
Persistent=true
RandomizedDelaySec=10min

[Install]
WantedBy=timers.target

sudo install -m 755 backup.sh /usr/local/bin/my-agent-backup
sudo install -m 755 restore-test.sh /usr/local/bin/my-agent-restore-test
sudo cp my-agent-backup.service my-agent-backup.timer /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now my-agent-backup.timer

#!/usr/bin/env bash
# Restore test: download the newest backup, decrypt it, unpack it into a scratch folder and check it.
# Nothing on the live system is touched. Run it by hand once a month.
set -euo pipefail

RCLONE_REMOTE="${RCLONE_REMOTE:?set RCLONE_REMOTE}"
PASSPHRASE_FILE="${PASSPHRASE_FILE:-/etc/my-agent/backup.pass}"
CHECK_FILE="${CHECK_FILE:-etc/my-agent/agent.env}"   # a file that must be in every backup (path without the leading /)
DEST="$(mktemp -d /var/tmp/restore-test.XXXXXX)"
trap 'rm -rf "$DEST"' EXIT

latest="$(rclone lsf --files-only "$RCLONE_REMOTE/" | grep '\.tar\.gz\.gpg$' | sort | tail -1)"
[[ -n "$latest" ]] || { echo "restore test FAILED: no backups found"; exit 1; }
rclone copy --quiet "$RCLONE_REMOTE/$latest" "$DEST/"
gpg --batch --quiet --decrypt --passphrase-file "$PASSPHRASE_FILE" "$DEST/$latest" | tar --extract --gzip --file - -C "$DEST"
[[ -s "$DEST/$CHECK_FILE" ]] || { echo "restore test FAILED: $CHECK_FILE missing in $latest"; exit 1; }
echo "restore test ok: $latest, $(find "$DEST" -type f | wc -l) files, $CHECK_FILE present"

sudo systemctl start my-agent-backup.service; journalctl -u my-agent-backup -n 3 --no-pager
# backup ok: offsite:my-agent-backups/backup-...tar.gz.gpg (...)
sudo bash -c 'set -a; . /etc/my-agent/backup.env; /usr/local/bin/my-agent-restore-test'
# restore test ok: backup-...tar.gz.gpg, ... files, etc/my-agent/agent.env present