Add encrypted bare-metal recovery workflow
This commit is contained in:
Mikei386
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
CONTAINER=${OPENWEBUI_CONTAINER:-mike-ai-open-webui}
|
||||
ENV_FILE=${NAVIDROME_MCP_ENV_FILE:-/etc/mike-ai/navidrome-mcp.env}
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $ENV_FILE ]] || die "Navidrome-Secret-Datei fehlt."
|
||||
[[ $(docker inspect -f '{{.State.Health.Status}}' mike-ai-mcp-navidrome 2>/dev/null || true) == healthy ]] || \
|
||||
die "Navidrome-MCP ist nicht gesund."
|
||||
[[ $(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER" 2>/dev/null || true) == healthy ]] || \
|
||||
die "OpenWebUI ist nicht gesund."
|
||||
|
||||
expect_lastfm=false
|
||||
grep -q '^LASTFM_API_KEY=..' "$ENV_FILE" && expect_lastfm=true
|
||||
|
||||
result=$(docker exec -i -e EXPECT_LASTFM="$expect_lastfm" "$CONTAINER" python - <<'PY'
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from mcp import ClientSession
|
||||
from mcp.client.streamable_http import streamablehttp_client
|
||||
|
||||
EXPECTED_LASTFM = {
|
||||
"get_similar_artists", "get_similar_tracks", "get_artist_info",
|
||||
"get_top_tracks_by_artist", "get_trending_music", "get_artist_albums",
|
||||
"get_album_info",
|
||||
}
|
||||
PLAYBACK = {"play_songs", "pause", "set_volume"}
|
||||
|
||||
def find_unanchored(value, path=""):
|
||||
bad = []
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
here = f"{path}.{key}" if path else key
|
||||
if key == "pattern" and (
|
||||
not isinstance(child, str)
|
||||
or not child.startswith("^")
|
||||
or not child.endswith("$")
|
||||
):
|
||||
bad.append(here)
|
||||
bad.extend(find_unanchored(child, here))
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
bad.extend(find_unanchored(child, f"{path}[{index}]"))
|
||||
return bad
|
||||
|
||||
async def verify():
|
||||
async with streamablehttp_client(
|
||||
"http://mike-ai-mcp-navidrome:3000/mcp"
|
||||
) as (read, write, _):
|
||||
async with ClientSession(read, write) as session:
|
||||
await session.initialize()
|
||||
result = await session.list_tools()
|
||||
names = {tool.name for tool in result.tools}
|
||||
bad = []
|
||||
for tool in result.tools:
|
||||
bad.extend(find_unanchored(tool.inputSchema, tool.name))
|
||||
if bad:
|
||||
raise SystemExit("Unverankerte JSON-Schema-Patterns: " + ", ".join(bad))
|
||||
if PLAYBACK & names:
|
||||
raise SystemExit("Playback-Werkzeuge sind auf dem Headless-Host aktiv.")
|
||||
expect_lastfm = os.environ.get("EXPECT_LASTFM") == "true"
|
||||
if expect_lastfm and not EXPECTED_LASTFM <= names:
|
||||
raise SystemExit("Last.fm-Werkzeugkatalog ist unvollständig.")
|
||||
if not expect_lastfm and EXPECTED_LASTFM & names:
|
||||
raise SystemExit("Last.fm-Werkzeuge sind ohne konfigurierten Schlüssel aktiv.")
|
||||
if expect_lastfm:
|
||||
# Public metadata only. Do not print the returned chart data.
|
||||
response = await session.call_tool(
|
||||
"get_trending_music", {"type": "artists", "limit": 1}
|
||||
)
|
||||
if response.isError:
|
||||
raise SystemExit("Öffentliche Last.fm-Testabfrage ist fehlgeschlagen.")
|
||||
|
||||
con = sqlite3.connect("/app/backend/data/webui.db")
|
||||
row = con.execute(
|
||||
"select value from config where key=?", ("tool_server.connections",)
|
||||
).fetchone()
|
||||
connections = json.loads(row[0]) if row else []
|
||||
ids = {
|
||||
str((connection.get("info") or {}).get("id", ""))
|
||||
for connection in connections if isinstance(connection, dict)
|
||||
}
|
||||
if "navidrome-local" not in ids:
|
||||
raise SystemExit("OpenWebUI-Verbindung navidrome-local fehlt.")
|
||||
print(f"NAVIDROME_ACCEPTANCE_OK tools={len(names)} lastfm={str(expect_lastfm).lower()}")
|
||||
|
||||
asyncio.run(verify())
|
||||
PY
|
||||
)
|
||||
[[ $result == NAVIDROME_ACCEPTANCE_OK\ * ]] || \
|
||||
die "Navidrome-Abnahme lieferte keinen gültigen Erfolgsmarker."
|
||||
printf '%s\n' "$result"
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
OUTPUT=${1:-}
|
||||
RECIPIENT_FILE=${AGE_RECIPIENT_FILE:-/etc/mike-ai/recovery.age-recipient}
|
||||
OPENWEBUI_VOLUME=${OPENWEBUI_VOLUME:-mike-ai_open-webui-data}
|
||||
OPENWEBUI_CONTAINER=${OPENWEBUI_CONTAINER:-mike-ai-open-webui}
|
||||
STACK_DIR=${STACK_DIR:-/opt/mike-ai/stack}
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -n $OUTPUT ]] || die "Aufruf: $0 /sicheres/offhost-ziel/athena-recovery-YYYYMMDD.tar.age"
|
||||
[[ -s $RECIPIENT_FILE ]] || die "Age-Empfängerdatei fehlt: $RECIPIENT_FILE"
|
||||
command -v age >/dev/null || die "age ist nicht installiert."
|
||||
command -v docker >/dev/null || die "Docker ist nicht installiert."
|
||||
|
||||
recipient=$(awk '/^age1[[:alnum:]]+$/ {print; exit}' "$RECIPIENT_FILE")
|
||||
[[ -n $recipient ]] || die "Keine gültige öffentliche age-Adresse gefunden."
|
||||
install -d -m 0700 "$(dirname "$OUTPUT")"
|
||||
[[ ! -e $OUTPUT ]] || die "Zieldatei existiert bereits: $OUTPUT"
|
||||
|
||||
stage=$(mktemp -d /tmp/mike-ai-recovery.XXXXXX)
|
||||
openwebui_was_running=false
|
||||
cleanup() {
|
||||
if $openwebui_was_running; then
|
||||
docker start "$OPENWEBUI_CONTAINER" >/dev/null 2>&1 || \
|
||||
printf 'WARNUNG: OpenWebUI konnte nach dem Backup nicht gestartet werden.\n' >&2
|
||||
fi
|
||||
rm -rf "$stage"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$stage/rootfs" "$stage/payload"
|
||||
|
||||
for source in \
|
||||
/etc/mike-ai \
|
||||
/root/mike-ai-install.env \
|
||||
/usr/local/bin/runraid; do
|
||||
[[ -e $source ]] || continue
|
||||
rsync -aR "$source" "$stage/rootfs/"
|
||||
done
|
||||
|
||||
tar -C "$stage/rootfs" -czf "$stage/payload/host-config.tar.gz" .
|
||||
volume_path=$(docker volume inspect -f '{{.Mountpoint}}' "$OPENWEBUI_VOLUME")
|
||||
[[ -s $volume_path/webui.db ]] || die "OpenWebUI-Datenbank fehlt oder ist leer."
|
||||
|
||||
# OpenWebUI uses SQLite. A filesystem archive while the database is being
|
||||
# written can be internally inconsistent even if tar itself succeeds. Keep the
|
||||
# interruption short, but always restart the container through the EXIT trap.
|
||||
if [[ $(docker inspect -f '{{.State.Running}}' "$OPENWEBUI_CONTAINER" 2>/dev/null || true) == true ]]; then
|
||||
openwebui_was_running=true
|
||||
docker stop --time 30 "$OPENWEBUI_CONTAINER" >/dev/null
|
||||
fi
|
||||
tar -C "$volume_path" -czf "$stage/payload/openwebui-data.tar.gz" .
|
||||
if $openwebui_was_running; then
|
||||
docker start "$OPENWEBUI_CONTAINER" >/dev/null
|
||||
openwebui_was_running=false
|
||||
fi
|
||||
|
||||
source_commit=unknown
|
||||
[[ ! -s $STACK_DIR/.mike-ai-source-commit ]] || source_commit=$(<"$STACK_DIR/.mike-ai-source-commit")
|
||||
cat >"$stage/payload/METADATA" <<EOF
|
||||
created_utc=$(date -u +%FT%TZ)
|
||||
hostname=$(hostname)
|
||||
source_commit=$source_commit
|
||||
openwebui_volume=$OPENWEBUI_VOLUME
|
||||
EOF
|
||||
(
|
||||
cd "$stage/payload"
|
||||
sha256sum host-config.tar.gz openwebui-data.tar.gz METADATA >SHA256SUMS
|
||||
tar -czf "$stage/bundle.tar.gz" \
|
||||
host-config.tar.gz openwebui-data.tar.gz METADATA SHA256SUMS
|
||||
)
|
||||
|
||||
age -r "$recipient" -o "$OUTPUT.partial" "$stage/bundle.tar.gz"
|
||||
mv "$OUTPUT.partial" "$OUTPUT"
|
||||
chmod 0600 "$OUTPUT"
|
||||
printf 'RECOVERY_BUNDLE_OK %s\n' "$OUTPUT"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
BUNDLE=${1:-}
|
||||
IDENTITY=${2:-}
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
OPENWEBUI_VOLUME=${OPENWEBUI_VOLUME:-mike-ai_open-webui-data}
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $BUNDLE ]] || die "Recovery-Bundle fehlt."
|
||||
[[ -s $IDENTITY ]] || die "Age-Identität fehlt."
|
||||
|
||||
if ! command -v age >/dev/null || ! command -v rsync >/dev/null; then
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends age rsync
|
||||
fi
|
||||
|
||||
stage=$(mktemp -d /tmp/mike-ai-restore.XXXXXX)
|
||||
trap 'rm -rf "$stage"' EXIT
|
||||
age -d -i "$IDENTITY" -o "$stage/bundle.tar.gz" "$BUNDLE"
|
||||
tar -C "$stage" -xzf "$stage/bundle.tar.gz"
|
||||
(
|
||||
cd "$stage"
|
||||
sha256sum -c SHA256SUMS
|
||||
)
|
||||
tar -tzf "$stage/openwebui-data.tar.gz" | grep -Eq '(^|/)webui\.db$' || \
|
||||
die "OpenWebUI-Archiv enthält keine Datenbank."
|
||||
|
||||
recorded_commit=$(sed -n 's/^source_commit=//p' "$stage/METADATA" | head -n 1)
|
||||
current_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || true)
|
||||
if [[ -n $recorded_commit && $recorded_commit != unknown && \
|
||||
$current_commit != "$recorded_commit" ]]; then
|
||||
die "Repository-Commit stimmt nicht mit dem Backup überein: erwartet $recorded_commit"
|
||||
fi
|
||||
|
||||
log "Root-only Konfiguration und freigegebene Secrets wiederherstellen"
|
||||
mkdir -p "$stage/rootfs"
|
||||
tar -C "$stage/rootfs" -xzf "$stage/host-config.tar.gz"
|
||||
[[ -s $stage/rootfs/root/mike-ai-install.env ]] || \
|
||||
die "Installationskonfiguration fehlt im Bundle."
|
||||
install -d -m 0700 /etc/mike-ai
|
||||
rsync -a "$stage/rootfs/etc/mike-ai/" /etc/mike-ai/
|
||||
install -m 0600 "$stage/rootfs/root/mike-ai-install.env" /root/mike-ai-install.env
|
||||
if [[ -x $stage/rootfs/usr/local/bin/runraid ]]; then
|
||||
install -m 0755 "$stage/rootfs/usr/local/bin/runraid" /usr/local/bin/runraid
|
||||
fi
|
||||
|
||||
log "Reproduzierbaren Host-Installer ausführen"
|
||||
set +e
|
||||
"$ROOT_DIR/install.sh" --config /root/mike-ai-install.env
|
||||
status=$?
|
||||
set -e
|
||||
if [[ $status == 20 || $status == 21 ]]; then
|
||||
printf 'REBOOT_REQUIRED code=%s\n' "$status"
|
||||
printf 'Nach dem Neustart denselben Restore-Befehl erneut ausführen.\n'
|
||||
exit "$status"
|
||||
fi
|
||||
[[ $status == 0 ]] || die "Host-Installer ist mit Status $status fehlgeschlagen."
|
||||
|
||||
log "OpenWebUI-Zustand atomar wiederherstellen"
|
||||
volume_path=$(docker volume inspect -f '{{.Mountpoint}}' "$OPENWEBUI_VOLUME")
|
||||
[[ -d $volume_path && $volume_path == /* && $volume_path != / && \
|
||||
$volume_path != /data && $volume_path != /var && \
|
||||
$volume_path != /var/lib && $volume_path != /var/lib/docker ]] || \
|
||||
die "Unsicherer Docker-Volume-Pfad: $volume_path"
|
||||
fallback=/data/openwebui-before-disaster-restore-$(date +%Y%m%d-%H%M%S).tar.gz
|
||||
docker stop mike-ai-open-webui >/dev/null 2>&1 || true
|
||||
tar -C "$volume_path" -czf "$fallback" .
|
||||
find "$volume_path" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
tar -C "$volume_path" -xzf "$stage/openwebui-data.tar.gz"
|
||||
docker start mike-ai-open-webui >/dev/null
|
||||
|
||||
deadline=$((SECONDS + 240))
|
||||
until [[ $(docker inspect -f '{{.State.Health.Status}}' mike-ai-open-webui 2>/dev/null || true) == healthy ]]; do
|
||||
(( SECONDS < deadline )) || die "OpenWebUI wurde nicht rechtzeitig gesund."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
log "Versionierte Modelle, Filter und Tool-Verbindungen nachziehen"
|
||||
"$ROOT_DIR/platform/openwebui/install-models.sh"
|
||||
"$ROOT_DIR/platform/openwebui/install-filters.sh"
|
||||
"$ROOT_DIR/platform/mcp/install-tools.sh"
|
||||
if [[ -s /etc/mike-ai/navidrome-mcp.env ]]; then
|
||||
"$ROOT_DIR/platform/mcp/verify-navidrome.sh"
|
||||
fi
|
||||
|
||||
printf 'BARE_METAL_RECOVERY_OK\n'
|
||||
printf 'Rückfallsicherung des leeren OpenWebUI-Stands: %s\n' "$fallback"
|
||||
Reference in New Issue
Block a user