Add three-scenario disaster recovery
This commit is contained in:
Mikei386
Executable
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-shot recovery orchestrator for system-disk, data-disk and total loss.
|
||||
# It never partitions, formats, reboots or shuts down the host.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCENARIO=""
|
||||
ARCHIVE=/data/docker-backups/athena-latest.tar.gz
|
||||
RECOVERY_CONFIG=""
|
||||
INSTALL_CONFIG=""
|
||||
SNAPSHOT=latest
|
||||
PORTABLE=""
|
||||
AGE_IDENTITY=""
|
||||
WORK=""
|
||||
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
cleanup() { [[ -z $WORK ]] || rm -rf "$WORK"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Systemplatte defekt, vorhandene /data-Platte:
|
||||
sudo ./disaster-recovery.sh --scenario system \
|
||||
--archive /data/docker-backups/athena-latest.tar.gz
|
||||
|
||||
Datenplatte defekt, Systemplatte vorhanden:
|
||||
sudo ./disaster-recovery.sh --scenario data \
|
||||
--config /root/athena-recovery.env
|
||||
|
||||
Beide Platten neu:
|
||||
sudo ./disaster-recovery.sh --scenario all \
|
||||
--config /root/athena-recovery.env
|
||||
|
||||
Alternativ bei Daten-/Totalausfall mit einem zuvor heruntergeladenen Paket:
|
||||
sudo ./disaster-recovery.sh --scenario all \
|
||||
--portable /pfad/athena-portable-....tar.zst.age \
|
||||
--identity /root/athena-recovery-key.txt
|
||||
|
||||
Voraussetzung: Debian ist installiert und die richtige, bereits formatierte
|
||||
Datenpartition ist separat unter /data eingehängt. Dieses Skript formatiert
|
||||
keine Datenträger und führt niemals selbst einen Neustart aus.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--scenario) SCENARIO=${2:-}; shift 2 ;;
|
||||
--archive) ARCHIVE=${2:-}; shift 2 ;;
|
||||
--config) RECOVERY_CONFIG=${2:-}; shift 2 ;;
|
||||
--install-config) INSTALL_CONFIG=${2:-}; shift 2 ;;
|
||||
--snapshot) SNAPSHOT=${2:-}; shift 2 ;;
|
||||
--portable) PORTABLE=${2:-}; shift 2 ;;
|
||||
--identity) AGE_IDENTITY=${2:-}; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) die "Unbekanntes Argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ $SCENARIO == system || $SCENARIO == data || $SCENARIO == all ]] || \
|
||||
die "--scenario muss system, data oder all sein."
|
||||
mountpoint -q /data || die "/data ist kein eigener Mountpoint. Abbruch zum Schutz der Systemplatte."
|
||||
|
||||
install_bootstrap_packages() {
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates gzip rsync tar restic
|
||||
}
|
||||
|
||||
copy_tree() {
|
||||
local source=$1 target=$2
|
||||
[[ -d $source ]] || return 0
|
||||
install -d -m 0755 "$target"
|
||||
rsync -a "$source/" "$target/"
|
||||
}
|
||||
|
||||
restore_local_bootstrap() {
|
||||
[[ -s $ARCHIVE ]] || die "Lokales Backup fehlt: $ARCHIVE"
|
||||
gzip -t "$ARCHIVE" || die "Lokales Backup ist beschädigt."
|
||||
WORK=$(mktemp -d /tmp/athena-system-recovery.XXXXXX)
|
||||
tar -xzf "$ARCHIVE" -C "$WORK"
|
||||
[[ -d $WORK/backup/etc-mike-ai ]] || die "Backup enthält /etc/mike-ai nicht."
|
||||
copy_tree "$WORK/backup/etc-mike-ai" /etc/mike-ai
|
||||
if [[ -d $WORK/backup/opt-mike-ai ]]; then
|
||||
copy_tree "$WORK/backup/opt-mike-ai" /opt/mike-ai
|
||||
elif [[ -d $WORK/backup/stack ]]; then
|
||||
copy_tree "$WORK/backup/stack" /opt/mike-ai/stack
|
||||
fi
|
||||
install -d -m 0700 /var/lib/mike-ai-disaster-backup/latest
|
||||
install -m 0600 "$ARCHIVE" \
|
||||
/var/lib/mike-ai-disaster-backup/latest/docker-state.tar.gz
|
||||
}
|
||||
|
||||
restore_external_snapshot() {
|
||||
[[ -r $RECOVERY_CONFIG ]] || die "Externe Recovery-Konfiguration fehlt: $RECOVERY_CONFIG"
|
||||
# shellcheck disable=SC1090
|
||||
source "$RECOVERY_CONFIG"
|
||||
[[ -n ${RESTIC_REPOSITORY:-} ]] || die "RESTIC_REPOSITORY fehlt."
|
||||
[[ -n ${RESTIC_PASSWORD_FILE:-} && -r $RESTIC_PASSWORD_FILE ]] || die \
|
||||
"Der separat aufzubewahrende Restic-Schlüssel fehlt."
|
||||
WORK=$(mktemp -d /tmp/athena-offsite-recovery.XXXXXX)
|
||||
restic restore "$SNAPSHOT" --tag "${RESTIC_TAG:-athena-disaster}" --target "$WORK"
|
||||
[[ -d $WORK/data ]] || die "Snapshot enthält keine Athena-Daten."
|
||||
copy_tree "$WORK/data" /data
|
||||
if [[ $SCENARIO == all ]]; then
|
||||
copy_tree "$WORK/etc/mike-ai" /etc/mike-ai
|
||||
copy_tree "$WORK/opt/mike-ai" /opt/mike-ai
|
||||
fi
|
||||
}
|
||||
|
||||
restore_portable_archive() {
|
||||
[[ -s $PORTABLE ]] || die "Portables Backup fehlt: $PORTABLE"
|
||||
[[ -r $AGE_IDENTITY ]] || die "Age-Identität fehlt: $AGE_IDENTITY"
|
||||
WORK=$(mktemp -d /tmp/athena-portable-recovery.XXXXXX)
|
||||
age --decrypt -i "$AGE_IDENTITY" "$PORTABLE" | zstd -d | tar -xf - -C "$WORK"
|
||||
[[ -d $WORK/data ]] || die "Portables Backup enthält keine Athena-Daten."
|
||||
copy_tree "$WORK/data" /data
|
||||
if [[ $SCENARIO == all ]]; then
|
||||
copy_tree "$WORK/etc/mike-ai" /etc/mike-ai
|
||||
copy_tree "$WORK/opt/mike-ai" /opt/mike-ai
|
||||
fi
|
||||
}
|
||||
|
||||
run_installer() {
|
||||
local config=${INSTALL_CONFIG:-/etc/mike-ai/install.env}
|
||||
[[ -r $config ]] || die \
|
||||
"Installationskonfiguration fehlt: $config (alternativ --install-config angeben)."
|
||||
chmod 0600 "$config"
|
||||
[[ -x /opt/mike-ai/stack/install.sh ]] || die "Wiederhergestellter Stack fehlt."
|
||||
set +e
|
||||
/opt/mike-ai/stack/install.sh --config "$config"
|
||||
local rc=$?
|
||||
set -e
|
||||
if [[ $rc == 20 || $rc == 21 ]]; then
|
||||
printf '\nEin kontrollierter Neustart ist für Treiber/Netzwerk nötig.\n'
|
||||
printf 'Danach exakt denselben Disaster-Recovery-Befehl erneut ausführen.\n'
|
||||
exit "$rc"
|
||||
fi
|
||||
[[ $rc == 0 ]] || die "Installer fehlgeschlagen (Exit $rc)."
|
||||
}
|
||||
|
||||
restore_docker_state() {
|
||||
local state=/var/lib/mike-ai-disaster-backup/latest/docker-state.tar.gz
|
||||
if [[ ! -s $state && -n $WORK ]]; then
|
||||
state=$(find "$WORK/var/lib/mike-ai-disaster-backup/latest" \
|
||||
-maxdepth 1 -name docker-state.tar.gz -type f -print -quit 2>/dev/null || true)
|
||||
fi
|
||||
[[ -s $state ]] || die "Docker-Zustandsarchiv fehlt im Backup."
|
||||
/opt/mike-ai/stack/restore.sh --check "$state"
|
||||
/opt/mike-ai/stack/restore.sh "$state"
|
||||
}
|
||||
|
||||
install_bootstrap_packages
|
||||
if [[ $SCENARIO == system ]]; then
|
||||
restore_local_bootstrap
|
||||
elif [[ -n $PORTABLE ]]; then
|
||||
restore_portable_archive
|
||||
else
|
||||
restore_external_snapshot
|
||||
fi
|
||||
|
||||
# In the data-only case the source/configuration remain on the system disk.
|
||||
[[ -x /opt/mike-ai/stack/install.sh ]] || die "/opt/mike-ai/stack fehlt."
|
||||
run_installer
|
||||
restore_docker_state
|
||||
/opt/mike-ai/stack/platform/recovery/rebuild-specialized.sh
|
||||
/opt/mike-ai/stack/smoke-test.sh
|
||||
|
||||
printf '\nATHENA_DISASTER_RECOVERY_OK scenario=%s\n' "$SCENARIO"
|
||||
printf 'Athena läuft im LLM-Standardmodus; Spezial-GPU-Worker bleiben gestoppt.\n'
|
||||
Reference in New Issue
Block a user