Simplify Athena stack and recovery
This commit is contained in:
Mikei386 committed
1 parent
c4851305d1
commit
069da8b4f0
70 files changed
+887
-5806
No files matched your search
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
OUTPUT=${1:-}
|
||||
RECIPIENT_FILE=${AGE_RECIPIENT_FILE:-/etc/mike-ai/recovery.age-recipient}
|
||||
OPENWEBUI_VOLUME=${OPENWEBUI_VOLUME:-mike-ai_open-webui-data}
|
||||
OPENWEBUI_CONTAINER=${OPENWEBUI_CONTAINER:-mike-ai-open-webui}
|
||||
STACK_DIR=${STACK_DIR:-/opt/mike-ai/stack}
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -n $OUTPUT ]] || die "Aufruf: $0 /sicheres/offhost-ziel/athena-recovery-YYYYMMDD.tar.age"
|
||||
[[ -s $RECIPIENT_FILE ]] || die "Age-Empfängerdatei fehlt: $RECIPIENT_FILE"
|
||||
command -v age >/dev/null || die "age ist nicht installiert."
|
||||
command -v docker >/dev/null || die "Docker ist nicht installiert."
|
||||
|
||||
recipient=$(awk '/^age1[[:alnum:]]+$/ {print; exit}' "$RECIPIENT_FILE")
|
||||
[[ -n $recipient ]] || die "Keine gültige öffentliche age-Adresse gefunden."
|
||||
install -d -m 0700 "$(dirname "$OUTPUT")"
|
||||
[[ ! -e $OUTPUT ]] || die "Zieldatei existiert bereits: $OUTPUT"
|
||||
|
||||
stage=$(mktemp -d /tmp/mike-ai-recovery.XXXXXX)
|
||||
sqlite_snapshot=""
|
||||
cleanup() {
|
||||
[[ -z $sqlite_snapshot ]] || rm -f -- "$sqlite_snapshot"
|
||||
rm -rf "$stage"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$stage/rootfs" "$stage/payload"
|
||||
|
||||
for source in \
|
||||
/etc/mike-ai \
|
||||
/root/mike-ai-install.env \
|
||||
/opt/mike-ai/stack/docs \
|
||||
/data/mike-ai-platform-context \
|
||||
/data/hermes \
|
||||
/data/hermes-webui/state; do
|
||||
[[ -e $source ]] || continue
|
||||
rsync -aR "$source" "$stage/rootfs/"
|
||||
done
|
||||
|
||||
tar -C "$stage/rootfs" -czf "$stage/payload/host-config.tar.gz" .
|
||||
volume_path=$(docker volume inspect -f '{{.Mountpoint}}' "$OPENWEBUI_VOLUME")
|
||||
[[ -s $volume_path/webui.db ]] || die "OpenWebUI-Datenbank fehlt oder ist leer."
|
||||
|
||||
# OpenWebUI uses SQLite. The online backup API creates a transactionally
|
||||
# consistent snapshot while the service remains available. The archive omits
|
||||
# the live DB/WAL/SHM and stores that snapshot under the canonical DB name.
|
||||
[[ $(docker inspect -f '{{.State.Running}}' "$OPENWEBUI_CONTAINER" 2>/dev/null || true) == true ]] || \
|
||||
die "OpenWebUI läuft nicht; Online-Datenbanksicherung nicht möglich."
|
||||
sqlite_snapshot="$volume_path/.mike-ai-recovery-webui.db"
|
||||
rm -f -- "$sqlite_snapshot"
|
||||
docker exec -i "$OPENWEBUI_CONTAINER" python - <<'PY'
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
source = "/app/backend/data/webui.db"
|
||||
snapshot = "/app/backend/data/.mike-ai-recovery-webui.db"
|
||||
if os.path.exists(snapshot):
|
||||
os.unlink(snapshot)
|
||||
with sqlite3.connect(source) as src, sqlite3.connect(snapshot) as dst:
|
||||
src.backup(dst)
|
||||
with sqlite3.connect(snapshot) as check:
|
||||
result = check.execute("PRAGMA integrity_check").fetchone()
|
||||
if not result or result[0] != "ok":
|
||||
raise SystemExit("SQLite integrity_check failed")
|
||||
PY
|
||||
[[ -s $sqlite_snapshot ]] || die "Konsistenter OpenWebUI-Snapshot wurde nicht erzeugt."
|
||||
tar -C "$volume_path" \
|
||||
--exclude='./webui.db' \
|
||||
--exclude='./webui.db-wal' \
|
||||
--exclude='./webui.db-shm' \
|
||||
--transform='s#\.mike-ai-recovery-webui\.db#webui.db#' \
|
||||
-czf "$stage/payload/openwebui-data.tar.gz" .
|
||||
tar -tzf "$stage/payload/openwebui-data.tar.gz" ./webui.db >/dev/null 2>&1 || \
|
||||
die "OpenWebUI-Archiv enthält den konsistenten Datenbanksnapshot nicht."
|
||||
|
||||
source_commit=unknown
|
||||
[[ ! -s $STACK_DIR/.mike-ai-source-commit ]] || source_commit=$(<"$STACK_DIR/.mike-ai-source-commit")
|
||||
cat >"$stage/payload/METADATA" <<EOF
|
||||
created_utc=$(date -u +%FT%TZ)
|
||||
hostname=$(hostname)
|
||||
source_commit=$source_commit
|
||||
openwebui_volume=$OPENWEBUI_VOLUME
|
||||
EOF
|
||||
(
|
||||
cd "$stage/payload"
|
||||
sha256sum host-config.tar.gz openwebui-data.tar.gz METADATA >SHA256SUMS
|
||||
tar -czf "$stage/bundle.tar.gz" \
|
||||
host-config.tar.gz openwebui-data.tar.gz METADATA SHA256SUMS
|
||||
)
|
||||
|
||||
age -r "$recipient" -o "$OUTPUT.partial" "$stage/bundle.tar.gz"
|
||||
mv "$OUTPUT.partial" "$OUTPUT"
|
||||
chmod 0600 "$OUTPUT"
|
||||
printf 'RECOVERY_BUNDLE_OK %s\n' "$OUTPUT"
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
RECOVERY_BUNDLE=${1:-}
|
||||
AGE_IDENTITY=${2:-}
|
||||
SOURCE_BUNDLE=${3:-}
|
||||
RELEASE_DIR=${4:-}
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $RECOVERY_BUNDLE ]] || die "Recovery-Bundle fehlt."
|
||||
[[ -s $AGE_IDENTITY ]] || die "age-Identität fehlt."
|
||||
[[ -s $SOURCE_BUNDLE ]] || die "Git-Quellbundle fehlt."
|
||||
[[ -n $RELEASE_DIR && $RELEASE_DIR == /data/* ]] || \
|
||||
die "Release-Ziel muss ein eindeutiger Pfad unter /data sein."
|
||||
[[ ! -e $RELEASE_DIR ]] || die "Release-Ziel existiert bereits."
|
||||
command -v age >/dev/null || die "age fehlt."
|
||||
command -v git >/dev/null || die "git fehlt."
|
||||
|
||||
stage=$(mktemp -d /tmp/mike-ai-data-kit.XXXXXX)
|
||||
trap 'rm -rf "$stage"' EXIT
|
||||
age -d -i "$AGE_IDENTITY" -o "$stage/bundle.tar.gz" "$RECOVERY_BUNDLE"
|
||||
tar -C "$stage" -xzf "$stage/bundle.tar.gz" METADATA SHA256SUMS \
|
||||
host-config.tar.gz openwebui-data.tar.gz
|
||||
(cd "$stage" && sha256sum -c SHA256SUMS)
|
||||
commit=$(sed -n 's/^source_commit=//p' "$stage/METADATA" | head -n 1)
|
||||
[[ $commit =~ ^[0-9a-f]{40}$ ]] || die "Recovery-Commit in METADATA ist ungültig."
|
||||
git clone -q "$SOURCE_BUNDLE" "$stage/source-check"
|
||||
git -C "$stage/source-check" cat-file -e "$commit^{commit}"
|
||||
|
||||
install -d -m 0700 "$RELEASE_DIR"
|
||||
install -m 0600 "$RECOVERY_BUNDLE" "$RELEASE_DIR/recovery.tar.age"
|
||||
install -m 0600 "$AGE_IDENTITY" "$RELEASE_DIR/recovery.agekey"
|
||||
install -m 0600 "$SOURCE_BUNDLE" "$RELEASE_DIR/source.git.bundle"
|
||||
install -m 0700 "$ROOT_DIR/platform/recovery/reinstall-from-data.sh" \
|
||||
"$RELEASE_DIR/reinstall-athena.sh"
|
||||
cat >"$RELEASE_DIR/kit.env" <<EOF
|
||||
RECOVERY_COMMIT=$commit
|
||||
RECOVERY_BUNDLE=recovery.tar.age
|
||||
SOURCE_BUNDLE=source.git.bundle
|
||||
AGE_IDENTITY=recovery.agekey
|
||||
EOF
|
||||
cat >"$RELEASE_DIR/README.txt" <<'EOF'
|
||||
ATHENA SELF-CONTAINED DATA-DISK RECOVERY
|
||||
|
||||
Auf einem frischen Debian die bestehende Data-SSD unter /data einhängen und
|
||||
als root ausführen:
|
||||
|
||||
/data/mike-ai-recovery-kit/reinstall-athena.sh
|
||||
|
||||
Das Skript prüft alle Dateien, stellt einen persistenten /data-Mount her,
|
||||
installiert minimale Werkzeuge und rekonstruiert den vollständigen MikeAI-
|
||||
Stack. Erforderliche Treiber-Neustarts werden höchstens dreimal automatisch
|
||||
fortgesetzt.
|
||||
|
||||
SICHERHEIT: Dieses Kit enthält den Entschlüsselungsschlüssel. Wer die Data-SSD
|
||||
lesen kann, kann daher auch die enthaltenen Infrastruktur-Secrets entschlüsseln.
|
||||
EOF
|
||||
chmod 0600 "$RELEASE_DIR/kit.env" "$RELEASE_DIR/README.txt"
|
||||
(
|
||||
cd "$RELEASE_DIR"
|
||||
sha256sum recovery.tar.age recovery.agekey source.git.bundle \
|
||||
reinstall-athena.sh kit.env README.txt >SHA256SUMS
|
||||
)
|
||||
chmod 0600 "$RELEASE_DIR/SHA256SUMS"
|
||||
|
||||
link=/data/mike-ai-recovery-kit
|
||||
[[ ! -e $link || -L $link ]] || die "$link existiert und ist kein verwalteter Symlink."
|
||||
ln -sfn "$(basename "$RELEASE_DIR")" "$link"
|
||||
printf 'SELF_CONTAINED_DATA_KIT_OK %s -> %s\n' "$link" "$RELEASE_DIR"
|
||||
@@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Self-contained Athena reinstall entry point. This file is copied into a
|
||||
# root-only recovery kit on /data; it is not run during normal installation.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
KIT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
CONFIG="$KIT_DIR/kit.env"
|
||||
WORK_DIR=/var/lib/mike-ai-data-reinstall
|
||||
SERVICE=/etc/systemd/system/mike-ai-data-reinstall.service
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $CONFIG ]] || die "kit.env fehlt im Recovery-Kit."
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG"
|
||||
|
||||
required=(RECOVERY_COMMIT RECOVERY_BUNDLE SOURCE_BUNDLE AGE_IDENTITY)
|
||||
for name in "${required[@]}"; do
|
||||
[[ -n ${!name:-} ]] || die "Pflichtwert $name fehlt."
|
||||
done
|
||||
for file in "$RECOVERY_BUNDLE" "$SOURCE_BUNDLE" "$AGE_IDENTITY"; do
|
||||
[[ -s $KIT_DIR/$file ]] || die "Recovery-Datei fehlt: $file"
|
||||
done
|
||||
|
||||
log "Recovery-Kit kryptografisch prüfen"
|
||||
(cd "$KIT_DIR" && sha256sum -c SHA256SUMS)
|
||||
|
||||
log "Persistenten Mount für die Data-SSD sicherstellen"
|
||||
[[ $(findmnt -n -o TARGET --target "$KIT_DIR") == /data ]] || \
|
||||
die "Recovery-Kit liegt nicht auf dem eingehängten /data-Dateisystem."
|
||||
if ! findmnt -s -n -o TARGET | grep -qx /data; then
|
||||
source_device=$(findmnt -n -o SOURCE --target "$KIT_DIR")
|
||||
source_uuid=$(blkid -s UUID -o value "$source_device")
|
||||
source_type=$(findmnt -n -o FSTYPE --target "$KIT_DIR")
|
||||
[[ -n $source_uuid && -n $source_type ]] || \
|
||||
die "UUID oder Dateisystemtyp der Data-SSD konnte nicht bestimmt werden."
|
||||
printf 'UUID=%s /data %s defaults,nofail,x-systemd.device-timeout=30 0 2\n' \
|
||||
"$source_uuid" "$source_type" >>/etc/fstab
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
|
||||
log "Minimale Wiederherstellungswerkzeuge installieren"
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
age ca-certificates git rsync
|
||||
|
||||
install -d -m 0700 "$WORK_DIR"
|
||||
if [[ ! -d $WORK_DIR/repository/.git ]]; then
|
||||
git clone "$KIT_DIR/$SOURCE_BUNDLE" "$WORK_DIR/repository"
|
||||
fi
|
||||
git -C "$WORK_DIR/repository" checkout --detach "$RECOVERY_COMMIT"
|
||||
[[ $(git -C "$WORK_DIR/repository" rev-parse HEAD) == "$RECOVERY_COMMIT" ]] || \
|
||||
die "Der geforderte Recovery-Commit ist nicht ausgecheckt."
|
||||
|
||||
attempt=0
|
||||
[[ ! -s $WORK_DIR/attempt ]] || attempt=$(<"$WORK_DIR/attempt")
|
||||
(( attempt += 1 ))
|
||||
printf '%s\n' "$attempt" >"$WORK_DIR/attempt"
|
||||
(( attempt <= 3 )) || die "Mehr als drei automatische Recovery-Versuche; Abbruch gegen Bootschleife."
|
||||
|
||||
log "Bare-Metal-Wiederherstellung ausführen (Versuch $attempt/3)"
|
||||
set +e
|
||||
"$WORK_DIR/repository/platform/recovery/restore-recovery-bundle.sh" \
|
||||
"$KIT_DIR/$RECOVERY_BUNDLE" "$KIT_DIR/$AGE_IDENTITY"
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ $status == 20 || $status == 21 ]]; then
|
||||
log "Ein kontrollierter Treiber-/Netzwerk-Neustart ist erforderlich"
|
||||
cat >"$SERVICE" <<EOF
|
||||
[Unit]
|
||||
Description=Resume MikeAI data-disk disaster recovery
|
||||
After=network-online.target local-fs.target
|
||||
Wants=network-online.target
|
||||
RequiresMountsFor=/data
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=$KIT_DIR/reinstall-athena.sh --resume
|
||||
StandardOutput=append:/var/log/mike-ai-data-reinstall.log
|
||||
StandardError=append:/var/log/mike-ai-data-reinstall.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable mike-ai-data-reinstall.service
|
||||
sync
|
||||
systemctl reboot
|
||||
exit 0
|
||||
fi
|
||||
[[ $status == 0 ]] || die "Wiederherstellung ist mit Status $status fehlgeschlagen."
|
||||
|
||||
systemctl disable mike-ai-data-reinstall.service >/dev/null 2>&1 || true
|
||||
rm -f "$SERVICE" "$WORK_DIR/attempt"
|
||||
systemctl daemon-reload
|
||||
printf '\nDATA_DISK_REINSTALL_OK\n'
|
||||
printf 'OpenWebUI, Router, Modelle und MCPs wurden wiederhergestellt.\n'
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
BUNDLE=${1:-}
|
||||
IDENTITY=${2:-}
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
OPENWEBUI_VOLUME=${OPENWEBUI_VOLUME:-mike-ai_open-webui-data}
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $BUNDLE ]] || die "Recovery-Bundle fehlt."
|
||||
[[ -s $IDENTITY ]] || die "Age-Identität fehlt."
|
||||
|
||||
if ! command -v age >/dev/null || ! command -v rsync >/dev/null; then
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends age rsync
|
||||
fi
|
||||
|
||||
stage=$(mktemp -d /tmp/mike-ai-restore.XXXXXX)
|
||||
trap 'rm -rf "$stage"' EXIT
|
||||
age -d -i "$IDENTITY" -o "$stage/bundle.tar.gz" "$BUNDLE"
|
||||
tar -C "$stage" -xzf "$stage/bundle.tar.gz"
|
||||
(
|
||||
cd "$stage"
|
||||
sha256sum -c SHA256SUMS
|
||||
)
|
||||
tar -tzf "$stage/openwebui-data.tar.gz" ./webui.db >/dev/null 2>&1 || \
|
||||
die "OpenWebUI-Archiv enthält keine Datenbank."
|
||||
|
||||
recorded_commit=$(sed -n 's/^source_commit=//p' "$stage/METADATA" | head -n 1)
|
||||
current_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || true)
|
||||
if [[ -n $recorded_commit && $recorded_commit != unknown && \
|
||||
$current_commit != "$recorded_commit" ]]; then
|
||||
die "Repository-Commit stimmt nicht mit dem Backup überein: erwartet $recorded_commit"
|
||||
fi
|
||||
|
||||
log "Root-only Konfiguration und freigegebene Secrets wiederherstellen"
|
||||
mkdir -p "$stage/rootfs"
|
||||
tar -C "$stage/rootfs" -xzf "$stage/host-config.tar.gz"
|
||||
[[ -s $stage/rootfs/root/mike-ai-install.env ]] || \
|
||||
die "Installationskonfiguration fehlt im Bundle."
|
||||
install -d -m 0700 /etc/mike-ai
|
||||
rsync -a "$stage/rootfs/etc/mike-ai/" /etc/mike-ai/
|
||||
install -m 0600 "$stage/rootfs/root/mike-ai-install.env" /root/mike-ai-install.env
|
||||
|
||||
log "Reproduzierbaren Host-Installer ausführen"
|
||||
set +e
|
||||
"$ROOT_DIR/install.sh" --config /root/mike-ai-install.env
|
||||
status=$?
|
||||
set -e
|
||||
if [[ $status == 20 || $status == 21 ]]; then
|
||||
printf 'REBOOT_REQUIRED code=%s\n' "$status"
|
||||
printf 'Nach dem Neustart denselben Restore-Befehl erneut ausführen.\n'
|
||||
exit "$status"
|
||||
fi
|
||||
[[ $status == 0 ]] || die "Host-Installer ist mit Status $status fehlgeschlagen."
|
||||
|
||||
log "Gesicherten Platform-Kontext und lokale Dokumentationspflege wiederherstellen"
|
||||
if [[ -d $stage/rootfs/opt/mike-ai/stack/docs ]]; then
|
||||
rsync -a "$stage/rootfs/opt/mike-ai/stack/docs/" /opt/mike-ai/stack/docs/
|
||||
fi
|
||||
if [[ -d $stage/rootfs/data/mike-ai-platform-context ]]; then
|
||||
install -d -o 10001 -g 10001 -m 0750 /data/mike-ai-platform-context
|
||||
rsync -a "$stage/rootfs/data/mike-ai-platform-context/" /data/mike-ai-platform-context/
|
||||
chown -R 10001:10001 /data/mike-ai-platform-context
|
||||
fi
|
||||
|
||||
log "OpenWebUI-Zustand atomar wiederherstellen"
|
||||
volume_path=$(docker volume inspect -f '{{.Mountpoint}}' "$OPENWEBUI_VOLUME")
|
||||
[[ -d $volume_path && $volume_path == /* && $volume_path != / && \
|
||||
$volume_path != /data && $volume_path != /var && \
|
||||
$volume_path != /var/lib && $volume_path != /var/lib/docker ]] || \
|
||||
die "Unsicherer Docker-Volume-Pfad: $volume_path"
|
||||
fallback=/data/openwebui-before-disaster-restore-$(date +%Y%m%d-%H%M%S).tar.gz
|
||||
docker stop mike-ai-open-webui >/dev/null 2>&1 || true
|
||||
tar -C "$volume_path" -czf "$fallback" .
|
||||
find "$volume_path" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
tar -C "$volume_path" -xzf "$stage/openwebui-data.tar.gz"
|
||||
docker start mike-ai-open-webui >/dev/null
|
||||
|
||||
deadline=$((SECONDS + 240))
|
||||
until [[ $(docker inspect -f '{{.State.Health.Status}}' mike-ai-open-webui 2>/dev/null || true) == healthy ]]; do
|
||||
(( SECONDS < deadline )) || die "OpenWebUI wurde nicht rechtzeitig gesund."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
log "Versionierte Modelle, Filter und Tool-Verbindungen nachziehen"
|
||||
"$ROOT_DIR/platform/openwebui/install-models.sh"
|
||||
"$ROOT_DIR/platform/openwebui/install-filters.sh"
|
||||
"$ROOT_DIR/platform/mcp/install-tools.sh"
|
||||
if [[ -s /etc/mike-ai/navidrome-mcp.env ]]; then
|
||||
"$ROOT_DIR/platform/mcp/verify-navidrome.sh"
|
||||
fi
|
||||
"$ROOT_DIR/dev/verify_mcp_catalogs.sh"
|
||||
|
||||
printf 'BARE_METAL_RECOVERY_OK\n'
|
||||
printf 'Rückfallsicherung des leeren OpenWebUI-Stands: %s\n' "$fallback"
|
||||
Reference in new issue
Block a user