Synchronize repository with Athena deployment
This commit is contained in:
Mikei386
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Encrypted off-host backup for data-disk and total-loss recovery.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
CONFIG=${DISASTER_BACKUP_CONFIG:-/etc/mike-ai/disaster-backup.env}
|
||||
STATE=/var/lib/mike-ai-disaster-backup
|
||||
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -r $CONFIG ]] || die "Konfiguration fehlt: $CONFIG"
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG"
|
||||
[[ ${DISASTER_BACKUP_ENABLED:-false} == true ]] || die \
|
||||
"Externes Backup ist noch nicht freigeschaltet (DISASTER_BACKUP_ENABLED=true)."
|
||||
[[ -n ${RESTIC_REPOSITORY:-} ]] || die "RESTIC_REPOSITORY fehlt."
|
||||
if [[ -n ${RESTIC_REQUIRE_MOUNT:-} ]]; then
|
||||
mountpoint -q "$RESTIC_REQUIRE_MOUNT" || die \
|
||||
"Externes Backupziel ist nicht eingehängt: $RESTIC_REQUIRE_MOUNT"
|
||||
fi
|
||||
[[ -n ${RESTIC_PASSWORD_FILE:-} && -r $RESTIC_PASSWORD_FILE ]] || die \
|
||||
"RESTIC_PASSWORD_FILE fehlt oder ist nicht lesbar."
|
||||
command -v restic >/dev/null || die "restic ist nicht installiert."
|
||||
command -v docker >/dev/null || die "Docker ist nicht installiert."
|
||||
exec 9>/run/lock/athena-disaster-backup.lock
|
||||
flock -n 9 || die "Ein Disaster-Backup läuft bereits."
|
||||
|
||||
install -d -m 0700 "$STATE/latest"
|
||||
|
||||
log "Konsistentes Docker-Schnellbackup erzeugen"
|
||||
docker inspect mike-ai-backup >/dev/null 2>&1 || die "mike-ai-backup fehlt."
|
||||
docker exec mike-ai-backup backup
|
||||
latest=$(readlink -f /data/docker-backups/athena-latest.tar.gz)
|
||||
[[ -s $latest ]] || die "Lokales Docker-Backup wurde nicht erzeugt."
|
||||
gzip -t "$latest" || die "Lokales Docker-Backup ist beschädigt."
|
||||
install -m 0600 "$latest" "$STATE/latest/docker-state.tar.gz"
|
||||
sha256sum "$STATE/latest/docker-state.tar.gz" >"$STATE/latest/docker-state.tar.gz.sha256"
|
||||
|
||||
log "Wiederaufbau-Metadaten erfassen"
|
||||
{
|
||||
printf 'created_utc=%s\n' "$(date -u +%FT%TZ)"
|
||||
printf 'hostname=%s\n' "$(hostname)"
|
||||
printf 'source_commit=%s\n' "$(git -C /opt/mike-ai/stack rev-parse HEAD 2>/dev/null || printf unknown)"
|
||||
findmnt -rn -o SOURCE,UUID,FSTYPE,TARGET / /data 2>/dev/null || true
|
||||
} >"$STATE/latest/manifest.txt"
|
||||
find /data/models -type f -printf '%P\t%s\n' 2>/dev/null | sort \
|
||||
>"$STATE/latest/model-manifest.tsv"
|
||||
docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' \
|
||||
>"$STATE/latest/container-manifest.tsv"
|
||||
|
||||
paths=(/etc/mike-ai /opt/mike-ai "$STATE/latest")
|
||||
for path in \
|
||||
/data/voice /data/music /data/audio /data/llama-dashboard \
|
||||
/data/mike-ai-operator /data/benchmarks /data/model-benchmarks \
|
||||
/data/image-comparison /data/backups /data/deploy-backups; do
|
||||
[[ ! -e $path ]] || paths+=("$path")
|
||||
done
|
||||
|
||||
tag=${RESTIC_TAG:-athena-disaster}
|
||||
log "Verschlüsseltes externes Backup schreiben"
|
||||
if ! restic snapshots >/dev/null 2>&1; then
|
||||
log "Neues Restic-Repository initialisieren"
|
||||
restic init
|
||||
fi
|
||||
restic backup --tag "$tag" "${paths[@]}"
|
||||
|
||||
log "Aufbewahrung anwenden"
|
||||
restic forget --tag "$tag" \
|
||||
--keep-daily "${RESTIC_KEEP_DAILY:-14}" \
|
||||
--keep-weekly "${RESTIC_KEEP_WEEKLY:-8}" \
|
||||
--keep-monthly "${RESTIC_KEEP_MONTHLY:-12}" --prune
|
||||
|
||||
log "Letzten Snapshot verifizieren"
|
||||
restic snapshots --tag "$tag" --latest 1
|
||||
restic check
|
||||
printf 'ATHENA_DISASTER_BACKUP_OK\n'
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Encrypted off-host disaster backup for Athena
|
||||
After=docker.service network-online.target
|
||||
Wants=network-online.target
|
||||
ConditionPathExists=/etc/mike-ai/disaster-backup.env
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/athena-disaster-backup
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Nightly Athena off-host disaster backup
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:15:00
|
||||
Persistent=true
|
||||
RandomizedDelaySec=30m
|
||||
Unit=athena-disaster-backup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a browser-downloadable, encrypted archive of irreplaceable Athena data.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
OUTPUT_DIR=${ATHENA_EXPORT_DIR:-/data/emergency-backups}
|
||||
RECIPIENT_FILE=${ATHENA_AGE_RECIPIENT_FILE:-/etc/mike-ai/recovery.age-recipient}
|
||||
STATE=/var/lib/mike-ai-disaster-backup/latest
|
||||
KEEP=${ATHENA_EXPORT_KEEP:-5}
|
||||
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $RECIPIENT_FILE ]] || die "Age-Empfänger fehlt: $RECIPIENT_FILE"
|
||||
[[ $KEEP =~ ^[1-9][0-9]*$ ]] || die "ATHENA_EXPORT_KEEP muss positiv sein."
|
||||
for command in age zstd tar docker sha256sum flock; do
|
||||
command -v "$command" >/dev/null || die "$command fehlt."
|
||||
done
|
||||
exec 9>/run/lock/athena-export-backup.lock
|
||||
flock -n 9 || die "Ein exportierbares Backup läuft bereits."
|
||||
|
||||
install -d -m 0755 "$OUTPUT_DIR"
|
||||
install -d -m 0700 "$STATE"
|
||||
|
||||
log "Aktuellen Docker-Zustand sichern"
|
||||
docker exec mike-ai-backup backup
|
||||
latest=$(readlink -f /data/docker-backups/athena-latest.tar.gz)
|
||||
[[ -s $latest ]] || die "Docker-Zustandsbackup fehlt."
|
||||
gzip -t "$latest" || die "Docker-Zustandsbackup ist beschädigt."
|
||||
install -m 0600 "$latest" "$STATE/docker-state.tar.gz"
|
||||
|
||||
stamp=$(date -u +%Y-%m-%dT%H-%M-%SZ)
|
||||
name="athena-portable-$stamp.tar.zst.age"
|
||||
partial="$OUTPUT_DIR/.$name.partial"
|
||||
target="$OUTPUT_DIR/$name"
|
||||
list=$(mktemp /tmp/athena-export-list.XXXXXX)
|
||||
trap 'rm -f "$list" "$partial"' EXIT
|
||||
|
||||
add_path() {
|
||||
local path=${1#/}
|
||||
[[ ! -e /$path ]] || printf '%s\0' "$path" >>"$list"
|
||||
}
|
||||
|
||||
# Reproducible model/HF caches are deliberately omitted. Everything below is
|
||||
# either a host configuration, project source, user input or generated result.
|
||||
add_path /etc/mike-ai
|
||||
add_path /opt/mike-ai
|
||||
add_path /var/lib/mike-ai-disaster-backup/latest
|
||||
add_path /data/voice/applio/logs
|
||||
add_path /data/voice/applio/datasets
|
||||
add_path /data/voice/applio/config.json
|
||||
add_path /data/voice/omnivoice/output
|
||||
add_path /data/voice/xvc/output
|
||||
add_path /data/voice/studio
|
||||
add_path /data/music
|
||||
add_path /data/audio
|
||||
add_path /data/llama-dashboard
|
||||
add_path /data/mike-ai-operator
|
||||
add_path /data/benchmarks
|
||||
add_path /data/model-benchmarks
|
||||
add_path /data/image-comparison
|
||||
add_path /data/backups
|
||||
add_path /data/deploy-backups
|
||||
|
||||
log "Portables, verschlüsseltes Backup erzeugen"
|
||||
tar --create --numeric-owner --acls --xattrs -C / --null --files-from="$list" \
|
||||
| zstd -T0 -3 \
|
||||
| age -R "$RECIPIENT_FILE" -o "$partial"
|
||||
chmod 0644 "$partial"
|
||||
mv "$partial" "$target"
|
||||
sha256sum "$target" >"$target.sha256"
|
||||
chmod 0644 "$target.sha256"
|
||||
|
||||
log "Nur die letzten $KEEP Generationen behalten"
|
||||
mapfile -t old < <(find "$OUTPUT_DIR" -maxdepth 1 -type f \
|
||||
-name 'athena-portable-*.tar.zst.age' -printf '%T@ %p\n' | sort -rn | tail -n +$((KEEP + 1)) | cut -d' ' -f2-)
|
||||
for archive in "${old[@]}"; do
|
||||
rm -f -- "$archive" "$archive.sha256"
|
||||
done
|
||||
|
||||
printf 'ATHENA_EXPORT_BACKUP_OK file=%s bytes=%s\n' "$target" "$(stat -c %s "$target")"
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Create encrypted downloadable Athena recovery package
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
ConditionPathExists=/etc/mike-ai/recovery.age-recipient
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/athena-export-backup
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Create an Athena recovery package every five hours
|
||||
|
||||
[Timer]
|
||||
OnBootSec=45m
|
||||
OnUnitActiveSec=5h
|
||||
Persistent=true
|
||||
RandomizedDelaySec=10m
|
||||
Unit=athena-export-backup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user