173 lines
6.1 KiB
Bash
Executable File
173 lines
6.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# One-shot recovery orchestrator for system-disk, data-disk and total loss.
|
|
# It never partitions, formats, reboots or shuts down the host.
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SCENARIO=""
|
|
ARCHIVE=/data/docker-backups/athena-latest.tar.gz
|
|
RECOVERY_CONFIG=""
|
|
INSTALL_CONFIG=""
|
|
SNAPSHOT=latest
|
|
PORTABLE=""
|
|
AGE_IDENTITY=""
|
|
WORK=""
|
|
|
|
log() { printf '\n==> %s\n' "$*"; }
|
|
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
|
cleanup() { [[ -z $WORK ]] || rm -rf "$WORK"; }
|
|
trap cleanup EXIT
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Systemplatte defekt, vorhandene /data-Platte:
|
|
sudo ./disaster-recovery.sh --scenario system \
|
|
--archive /data/docker-backups/athena-latest.tar.gz
|
|
|
|
Datenplatte defekt, Systemplatte vorhanden:
|
|
sudo ./disaster-recovery.sh --scenario data \
|
|
--config /root/athena-recovery.env
|
|
|
|
Beide Platten neu:
|
|
sudo ./disaster-recovery.sh --scenario all \
|
|
--config /root/athena-recovery.env
|
|
|
|
Alternativ bei Daten-/Totalausfall mit einem zuvor heruntergeladenen Paket:
|
|
sudo ./disaster-recovery.sh --scenario all \
|
|
--portable /pfad/athena-portable-....tar.zst.age \
|
|
--identity /root/athena-recovery-key.txt
|
|
|
|
Voraussetzung: Debian ist installiert und die richtige, bereits formatierte
|
|
Datenpartition ist separat unter /data eingehängt. Dieses Skript formatiert
|
|
keine Datenträger und führt niemals selbst einen Neustart aus.
|
|
EOF
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--scenario) SCENARIO=${2:-}; shift 2 ;;
|
|
--archive) ARCHIVE=${2:-}; shift 2 ;;
|
|
--config) RECOVERY_CONFIG=${2:-}; shift 2 ;;
|
|
--install-config) INSTALL_CONFIG=${2:-}; shift 2 ;;
|
|
--snapshot) SNAPSHOT=${2:-}; shift 2 ;;
|
|
--portable) PORTABLE=${2:-}; shift 2 ;;
|
|
--identity) AGE_IDENTITY=${2:-}; shift 2 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) die "Unbekanntes Argument: $1" ;;
|
|
esac
|
|
done
|
|
|
|
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
|
[[ $SCENARIO == system || $SCENARIO == data || $SCENARIO == all ]] || \
|
|
die "--scenario muss system, data oder all sein."
|
|
mountpoint -q /data || die "/data ist kein eigener Mountpoint. Abbruch zum Schutz der Systemplatte."
|
|
|
|
install_bootstrap_packages() {
|
|
apt-get update
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
|
ca-certificates gzip rsync tar restic
|
|
}
|
|
|
|
copy_tree() {
|
|
local source=$1 target=$2
|
|
[[ -d $source ]] || return 0
|
|
install -d -m 0755 "$target"
|
|
rsync -a "$source/" "$target/"
|
|
}
|
|
|
|
restore_local_bootstrap() {
|
|
[[ -s $ARCHIVE ]] || die "Lokales Backup fehlt: $ARCHIVE"
|
|
gzip -t "$ARCHIVE" || die "Lokales Backup ist beschädigt."
|
|
WORK=$(mktemp -d /tmp/athena-system-recovery.XXXXXX)
|
|
tar -xzf "$ARCHIVE" -C "$WORK"
|
|
[[ -d $WORK/backup/etc-mike-ai ]] || die "Backup enthält /etc/mike-ai nicht."
|
|
copy_tree "$WORK/backup/etc-mike-ai" /etc/mike-ai
|
|
if [[ -d $WORK/backup/opt-mike-ai ]]; then
|
|
copy_tree "$WORK/backup/opt-mike-ai" /opt/mike-ai
|
|
elif [[ -d $WORK/backup/stack ]]; then
|
|
copy_tree "$WORK/backup/stack" /opt/mike-ai/stack
|
|
fi
|
|
install -d -m 0700 /var/lib/mike-ai-disaster-backup/latest
|
|
install -m 0600 "$ARCHIVE" \
|
|
/var/lib/mike-ai-disaster-backup/latest/docker-state.tar.gz
|
|
}
|
|
|
|
restore_external_snapshot() {
|
|
[[ -r $RECOVERY_CONFIG ]] || die "Externe Recovery-Konfiguration fehlt: $RECOVERY_CONFIG"
|
|
# shellcheck disable=SC1090
|
|
source "$RECOVERY_CONFIG"
|
|
[[ -n ${RESTIC_REPOSITORY:-} ]] || die "RESTIC_REPOSITORY fehlt."
|
|
[[ -n ${RESTIC_PASSWORD_FILE:-} && -r $RESTIC_PASSWORD_FILE ]] || die \
|
|
"Der separat aufzubewahrende Restic-Schlüssel fehlt."
|
|
WORK=$(mktemp -d /tmp/athena-offsite-recovery.XXXXXX)
|
|
restic restore "$SNAPSHOT" --tag "${RESTIC_TAG:-athena-disaster}" --target "$WORK"
|
|
[[ -d $WORK/data ]] || die "Snapshot enthält keine Athena-Daten."
|
|
copy_tree "$WORK/data" /data
|
|
if [[ $SCENARIO == all ]]; then
|
|
copy_tree "$WORK/etc/mike-ai" /etc/mike-ai
|
|
copy_tree "$WORK/opt/mike-ai" /opt/mike-ai
|
|
fi
|
|
}
|
|
|
|
restore_portable_archive() {
|
|
[[ -s $PORTABLE ]] || die "Portables Backup fehlt: $PORTABLE"
|
|
[[ -r $AGE_IDENTITY ]] || die "Age-Identität fehlt: $AGE_IDENTITY"
|
|
WORK=$(mktemp -d /tmp/athena-portable-recovery.XXXXXX)
|
|
age --decrypt -i "$AGE_IDENTITY" "$PORTABLE" | zstd -d | tar -xf - -C "$WORK"
|
|
[[ -d $WORK/data ]] || die "Portables Backup enthält keine Athena-Daten."
|
|
copy_tree "$WORK/data" /data
|
|
if [[ $SCENARIO == all ]]; then
|
|
copy_tree "$WORK/etc/mike-ai" /etc/mike-ai
|
|
copy_tree "$WORK/opt/mike-ai" /opt/mike-ai
|
|
fi
|
|
}
|
|
|
|
run_installer() {
|
|
local config=${INSTALL_CONFIG:-/etc/mike-ai/install.env}
|
|
[[ -r $config ]] || die \
|
|
"Installationskonfiguration fehlt: $config (alternativ --install-config angeben)."
|
|
chmod 0600 "$config"
|
|
[[ -x /opt/mike-ai/stack/install.sh ]] || die "Wiederhergestellter Stack fehlt."
|
|
set +e
|
|
/opt/mike-ai/stack/install.sh --config "$config"
|
|
local rc=$?
|
|
set -e
|
|
if [[ $rc == 20 || $rc == 21 ]]; then
|
|
printf '\nEin kontrollierter Neustart ist für Treiber/Netzwerk nötig.\n'
|
|
printf 'Danach exakt denselben Disaster-Recovery-Befehl erneut ausführen.\n'
|
|
exit "$rc"
|
|
fi
|
|
[[ $rc == 0 ]] || die "Installer fehlgeschlagen (Exit $rc)."
|
|
}
|
|
|
|
restore_docker_state() {
|
|
local state=/var/lib/mike-ai-disaster-backup/latest/docker-state.tar.gz
|
|
if [[ ! -s $state && -n $WORK ]]; then
|
|
state=$(find "$WORK/var/lib/mike-ai-disaster-backup/latest" \
|
|
-maxdepth 1 -name docker-state.tar.gz -type f -print -quit 2>/dev/null || true)
|
|
fi
|
|
[[ -s $state ]] || die "Docker-Zustandsarchiv fehlt im Backup."
|
|
/opt/mike-ai/stack/restore.sh --check "$state"
|
|
/opt/mike-ai/stack/restore.sh "$state"
|
|
}
|
|
|
|
install_bootstrap_packages
|
|
if [[ $SCENARIO == system ]]; then
|
|
restore_local_bootstrap
|
|
elif [[ -n $PORTABLE ]]; then
|
|
restore_portable_archive
|
|
else
|
|
restore_external_snapshot
|
|
fi
|
|
|
|
# In the data-only case the source/configuration remain on the system disk.
|
|
[[ -x /opt/mike-ai/stack/install.sh ]] || die "/opt/mike-ai/stack fehlt."
|
|
run_installer
|
|
restore_docker_state
|
|
/opt/mike-ai/stack/platform/recovery/rebuild-specialized.sh
|
|
/opt/mike-ai/stack/smoke-test.sh
|
|
|
|
printf '\nATHENA_DISASTER_RECOVERY_OK scenario=%s\n' "$SCENARIO"
|
|
printf 'Athena läuft im LLM-Standardmodus; Spezial-GPU-Worker bleiben gestoppt.\n'
|