Synchronize repository with Athena deployment

2026-09-13 20:01:36 +02:00
parent 040a2df48b
commit fce9900389
60 changed files with 2570 additions and 607 deletions
@@ -37,11 +37,14 @@ if (( (8#$config_mode & 077) != 0 )); then
 fi
 # shellcheck disable=SC1090
 source "$CONFIG"
+if [[ -n ${HF_TOKEN_FILE:-} && ! -r ${HF_TOKEN_FILE:-} && \
+      -r /etc/mike-ai/huggingface-token ]]; then
+  HF_TOKEN_FILE=/etc/mike-ai/huggingface-token
+fi

 required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE
  FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL
  MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256
-  BETA1_MODEL_FILE BETA1_MODEL_URL BETA1_MODEL_SHA256
  ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256
  UNCENSORED_MODEL_FILE UNCENSORED_MODEL_URL UNCENSORED_MODEL_SHA256
  UNCENSORED_PROJECTOR_FILE UNCENSORED_PROJECTOR_URL
@@ -70,7 +73,7 @@ install_base_packages() {
  apt-get update
  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
    ca-certificates curl git gnupg jq openssl wireguard-tools iptables \
-    iproute2 pciutils rsync unattended-upgrades ethtool age
+    iproute2 pciutils rsync unattended-upgrades ethtool age restic zstd
 }

 setup_stable_network_name() {
@@ -304,6 +307,19 @@ install_stack_files() {
    printf '%s\n' "$source_head" >"$STACK_DIR/.mike-ai-source-commit"
  fi
  install -d -m 0700 "$SECRETS_DIR"
+  # Keep the exact host bootstrap inputs with the protected system
+  # configuration. This breaks the former recovery cycle in which a fresh
+  # host needed a lost /root-only install file before it could restore backup.
+  if [[ $(realpath "$CONFIG") != $(realpath -m "$SECRETS_DIR/install.env") ]]; then
+    install -m 0600 "$CONFIG" "$SECRETS_DIR/install.env"
+  else
+    chmod 0600 "$SECRETS_DIR/install.env"
+  fi
+  if [[ -n ${HF_TOKEN_FILE:-} && -r $HF_TOKEN_FILE && \
+        $(realpath "$HF_TOKEN_FILE") != $(realpath -m "$SECRETS_DIR/huggingface-token") ]]; then
+    install -m 0600 "$HF_TOKEN_FILE" "$SECRETS_DIR/huggingface-token"
+    HF_TOKEN_FILE=$SECRETS_DIR/huggingface-token
+  fi
  [[ -s $SECRETS_DIR/router-api-key ]] || openssl rand -base64 48 >$SECRETS_DIR/router-api-key
  [[ -s $SECRETS_DIR/controller-token ]] || openssl rand -base64 48 >$SECRETS_DIR/controller-token
  chmod 0600 "$SECRETS_DIR"/*
@@ -314,8 +330,6 @@ MODEL_DIR=$MODEL_DIR
 WIREGUARD_CONFIG_FILE=${WIREGUARD_CONFIG_FILE:-/etc/mike-ai/wireguard/fritz-athena.conf}
 ROUTER_API_KEY=$(<$SECRETS_DIR/router-api-key)
 CONTROLLER_TOKEN=$(<$SECRETS_DIR/controller-token)
-PIPER_TTS_VERSION=${PIPER_TTS_VERSION:-1.6.0}
-PIPER_VOICE=${PIPER_VOICE:-de_DE-thorsten-high}
 QWEN3_TTS_IMAGE=${QWEN3_TTS_IMAGE:-ghcr.io/malaiwah/qwen3-tts-server:latest@sha256:b363a01d08b1bbecbfc3ca6f585368fae2cfdc591f9ecca6643738369f9a9d98}
 QWEN3_TTS_CACHE_DIR=$QWEN3_TTS_CACHE_DIR
 QWEN3_TTS_VOICES_DIR=$QWEN3_TTS_VOICES_DIR
@@ -323,7 +337,6 @@ QWEN3_TTS_GPU_DEVICE=${QWEN3_TTS_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d4
 AI_DNS=${WG_DNS:-1.1.1.1}
 FAST_MODEL_FILE=$FAST_MODEL_FILE
 MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE
-BETA1_MODEL_FILE=$BETA1_MODEL_FILE
 LARGE_MODEL_FILE=$LARGE_MODEL_FILE
 ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE
 UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE
@@ -337,10 +350,6 @@ MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000}
 MEDIUM_BATCH_SIZE=${MEDIUM_BATCH_SIZE:-2048}
 MEDIUM_UBATCH_SIZE=${MEDIUM_UBATCH_SIZE:-128}
 MEDIUM_PARALLEL_SLOTS=${MEDIUM_PARALLEL_SLOTS:-1}
-BETA1_CONTEXT=${BETA1_CONTEXT:-192000}
-BETA1_BATCH_SIZE=${BETA1_BATCH_SIZE:-2048}
-BETA1_UBATCH_SIZE=${BETA1_UBATCH_SIZE:-128}
-BETA1_PARALLEL_SLOTS=${BETA1_PARALLEL_SLOTS:-1}
 LARGE_CONTEXT=${LARGE_CONTEXT:-192000}
 LARGE_BATCH_SIZE=${LARGE_BATCH_SIZE:-2048}
 LARGE_UBATCH_SIZE=${LARGE_UBATCH_SIZE:-128}
@@ -356,7 +365,6 @@ UNCENSORED_PARALLEL_SLOTS=${UNCENSORED_PARALLEL_SLOTS:-1}
 FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 MEDIUM_TENSOR_SPLIT=${MEDIUM_TENSOR_SPLIT:-90,10}
-BETA1_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 LARGE_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 LARGE_TENSOR_SPLIT=${LARGE_TENSOR_SPLIT:-86,14}
 ULTRA_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
@@ -365,7 +373,8 @@ UNCENSORED_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDAR
 UNCENSORED_TENSOR_SPLIT=${UNCENSORED_TENSOR_SPLIT:-90,10}
 UNCENSORED_MTP_MAX=${UNCENSORED_MTP_MAX:-2}
 IMAGE_GPU_DEVICES=${IMAGE_GPU_DEVICES:-${TEXT_GPU_DEVICES:-0}}
-FLUX_MODEL_DIR=${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}
+FLUX_COMPONENT_DIR=${FLUX_COMPONENT_DIR:-/data/models/FLUX.2-klein-9B-components}
+FLUX_TRANSFORMER_DIR=${FLUX_TRANSFORMER_DIR:-/data/models/FLUX.2-klein-9B-fp8}
 LLAMA_THREADS=${LLAMA_THREADS:-6}
 LLAMA_THREADS_BATCH=${LLAMA_THREADS_BATCH:-6}
 LLAMA_CACHE_RAM_MIB=${LLAMA_CACHE_RAM_MIB:-32768}
@@ -374,6 +383,29 @@ EOF
  chmod 0600 $SECRETS_DIR/stack.env
 }

+install_disaster_backup() {
+  log "Externes Disaster-Backup installieren"
+  install -m 0755 "$ROOT_DIR/platform/backup/athena-disaster-backup" \
+    /usr/local/sbin/athena-disaster-backup
+  install -m 0644 "$ROOT_DIR/platform/backup/athena-disaster-backup.service" \
+    /etc/systemd/system/athena-disaster-backup.service
+  install -m 0644 "$ROOT_DIR/platform/backup/athena-disaster-backup.timer" \
+    /etc/systemd/system/athena-disaster-backup.timer
+  install -m 0755 "$ROOT_DIR/platform/backup/athena-export-backup" \
+    /usr/local/sbin/athena-export-backup
+  install -m 0644 "$ROOT_DIR/platform/backup/athena-export-backup.service" \
+    /etc/systemd/system/athena-export-backup.service
+  install -m 0644 "$ROOT_DIR/platform/backup/athena-export-backup.timer" \
+    /etc/systemd/system/athena-export-backup.timer
+  if [[ ! -e $SECRETS_DIR/disaster-backup.env ]]; then
+    install -m 0600 "$ROOT_DIR/config/disaster-backup.env.example" \
+      "$SECRETS_DIR/disaster-backup.env.example"
+  fi
+  systemctl daemon-reload
+  systemctl enable --now athena-disaster-backup.timer
+  systemctl enable --now athena-export-backup.timer
+}
+
 download_one() {
  local relative=$1 url=$2 expected=$3 target="$MODEL_DIR/$1"
  install -d -m 0755 "$(dirname "$target")"
@@ -404,7 +436,6 @@ download_models() {
  done <<EOF
 $FAST_MODEL_FILE|$FAST_MODEL_URL|$FAST_MODEL_SHA256
 $MEDIUM_MODEL_FILE|$MEDIUM_MODEL_URL|$MEDIUM_MODEL_SHA256
-$BETA1_MODEL_FILE|$BETA1_MODEL_URL|$BETA1_MODEL_SHA256
 $LARGE_MODEL_FILE|$LARGE_MODEL_URL|$LARGE_MODEL_SHA256
 $ULTRA_MODEL_FILE|$ULTRA_MODEL_URL|$ULTRA_MODEL_SHA256
 $UNCENSORED_MODEL_FILE|$UNCENSORED_MODEL_URL|$UNCENSORED_MODEL_SHA256
@@ -486,25 +517,38 @@ build_and_start() {
  docker build --progress=plain --build-arg LLAMA_CPP_COMMIT="$commit" \
    -f platform/docker/llama-cpp/Dockerfile -t mike-ai/llama.cpp:local .
  docker compose --env-file "$SECRETS_DIR/stack.env" --profile image build image-worker
-  if [[ ! -s ${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}/model_index.json ]]; then
-    log "FLUX.2-klein-4B laden"
-    install -d -m 0755 "${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}"
-    docker run --rm --entrypoint python \
-      -v "${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}:/download" \
+  local flux_components=${FLUX_COMPONENT_DIR:-/data/models/FLUX.2-klein-9B-components}
+  local flux_transformer=${FLUX_TRANSFORMER_DIR:-/data/models/FLUX.2-klein-9B-fp8}
+  local hf_token_file=${HF_TOKEN_FILE:-/root/.cache/huggingface/token}
+  if [[ ! -s $flux_components/model_index.json || \
+        ! -s $flux_transformer/flux-2-klein-9b-fp8.safetensors ]]; then
+    [[ -r $hf_token_file ]] || die \
+      "Hugging-Face-Token fehlt: $hf_token_file (FLUX.2 Klein 9B ist gated)"
+    log "FLUX.2 Klein 9B Komponenten und FP8-Transformer laden"
+    install -d -m 0755 "$flux_components" "$flux_transformer"
+    docker run --rm --entrypoint /opt/image-venv/bin/python \
+      -e HF_TOKEN_PATH=/run/secrets/hf-token \
+      -v "$hf_token_file:/run/secrets/hf-token:ro" \
+      -v "$flux_components:/download" \
      mike-ai/image-worker:local -c \
-      "from huggingface_hub import snapshot_download; snapshot_download('black-forest-labs/FLUX.2-klein-4B', revision='e7b7dc27f91deacad38e78976d1f2b499d76a294', local_dir='/download')"
-    chmod -R a-w "${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}"
+      "from huggingface_hub import snapshot_download; snapshot_download('black-forest-labs/FLUX.2-klein-9B', revision='92196c8e11f7b6cf2b7493e037d8c5345c559216', local_dir='/download', allow_patterns=['model_index.json', 'scheduler/*', 'text_encoder/*', 'tokenizer/*', 'transformer/config.json', 'vae/*'])"
+    docker run --rm --entrypoint /opt/image-venv/bin/python \
+      -e HF_TOKEN_PATH=/run/secrets/hf-token \
+      -v "$hf_token_file:/run/secrets/hf-token:ro" \
+      -v "$flux_transformer:/download" \
+      mike-ai/image-worker:local -c \
+      "from huggingface_hub import snapshot_download; snapshot_download('black-forest-labs/FLUX.2-klein-9b-fp8', revision='902d9d510b51533e07729f19211414a3648b77d2', local_dir='/download', allow_patterns=['flux-2-klein-9b-fp8.safetensors', 'README.md', 'LICENSE.md'])"
+    chmod -R a-w "$flux_components" "$flux_transformer"
  fi
  # Creates the tools network and deploys the only host-bound MCP: Operator.
  # Portable MCPs and Hermes live on Unraid and are restored through Appdata.
  "$STACK_DIR/platform/mcp/install-tools.sh"
  docker compose --env-file "$SECRETS_DIR/stack.env" --profile inference create \
-    llama-fast llama-medium llama-beta1 llama-large llama-ultra llama-uncensored
+    llama-fast llama-medium llama-large llama-ultra llama-uncensored
  docker compose --env-file "$SECRETS_DIR/stack.env" --profile image create image-worker
  docker volume inspect portainer_data >/dev/null 2>&1 || docker volume create portainer_data >/dev/null
-  docker compose --env-file "$SECRETS_DIR/stack.env" stop llama-dashboard portainer
  docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \
-    wireguard-gateway qwen3-tts piper tts-gateway profile-controller router llama-dashboard portainer backup
+    wireguard-gateway qwen3-tts tts-gateway profile-controller router llama-dashboard portainer backup
  if [[ ${WIREGUARD_MODE:-container} == container ]]; then
    systemctl restart mike-ai-container-vpn-guard.service
  fi
@@ -565,6 +609,7 @@ install_stack_files
 download_models
 install_routing_guard
 build_and_start
+install_disaster_backup

 log "Installation abgeschlossen"
 if [[ ${WIREGUARD_MODE:-container} == container ]]; then