Fix Athena recovery and installation consistency
This commit is contained in:
Mikei386
+6
-6
@@ -4,7 +4,7 @@ MODEL_DIR=/data/models
|
||||
ROUTER_API_KEY=GENERATED_BY_INSTALLER
|
||||
CONTROLLER_TOKEN=GENERATED_BY_INSTALLER
|
||||
FLUX_MODEL_DIR=/data/models/FLUX.2-klein-4B
|
||||
IMAGE_GPU_DEVICES=1
|
||||
IMAGE_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe
|
||||
PIPER_TTS_VERSION=1.6.0
|
||||
PIPER_VOICE=de_DE-thorsten-high
|
||||
XTTS_IMAGE=ghcr.io/coqui-ai/xtts-streaming-server:latest-cuda121@sha256:f7fb3b1f9d4bc88af94da1b5959d8002f1e0b003c97557164034eb8a29f01b90
|
||||
@@ -36,14 +36,14 @@ ULTRA_UBATCH_SIZE=128
|
||||
UNCENSORED_CONTEXT=80000
|
||||
UNCENSORED_BATCH_SIZE=2048
|
||||
UNCENSORED_UBATCH_SIZE=128
|
||||
FAST_GPU_DEVICES=0,1
|
||||
MEDIUM_GPU_DEVICES=0,1
|
||||
FAST_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
|
||||
MEDIUM_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
|
||||
MEDIUM_TENSOR_SPLIT=85,15
|
||||
LARGE_GPU_DEVICES=0,1
|
||||
LARGE_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
|
||||
LARGE_TENSOR_SPLIT=86,14
|
||||
ULTRA_GPU_DEVICES=0,1
|
||||
ULTRA_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
|
||||
ULTRA_TENSOR_SPLIT=80,20
|
||||
UNCENSORED_GPU_DEVICES=0,1
|
||||
UNCENSORED_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
|
||||
UNCENSORED_TENSOR_SPLIT=90,10
|
||||
UNCENSORED_MTP_MAX=2
|
||||
LLAMA_THREADS=6
|
||||
|
||||
@@ -109,6 +109,7 @@ Nach einer frischen Debian-Installation und erneut eingehängtem `/data`:
|
||||
|
||||
```bash
|
||||
sudo ./install.sh --config /root/mike-ai-install.env
|
||||
sudo ./restore.sh --check /data/docker-backups/athena-latest.tar.gz
|
||||
sudo ./restore.sh /data/docker-backups/athena-latest.tar.gz
|
||||
sudo ./smoke-test.sh
|
||||
```
|
||||
|
||||
@@ -12,9 +12,12 @@ INSTALL_NVIDIA_DRIVER=true
|
||||
# festlegen (z. B. 610). Leer lassen, um dem aktuellen stabilen Zweig zu folgen.
|
||||
NVIDIA_DRIVER_BRANCH=
|
||||
NVIDIA_MIN_DRIVER_MAJOR=570
|
||||
TEXT_GPU_DEVICES=0
|
||||
SECONDARY_GPU_DEVICES=1
|
||||
IMAGE_GPU_DEVICES=1
|
||||
# Stable UUID order: CUDA0 = RTX 5080, CUDA1 = RTX 3060. Positional host
|
||||
# indices are intentionally avoided because nvidia-smi currently enumerates
|
||||
# the cards in the opposite order.
|
||||
TEXT_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe
|
||||
SECONDARY_GPU_DEVICES=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
|
||||
IMAGE_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe
|
||||
FLUX_MODEL_DIR=/data/models/FLUX.2-klein-4B
|
||||
|
||||
# Headless remote reachability. Firmware power-loss recovery is configured
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import io
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class RecoveryScriptTests(unittest.TestCase):
|
||||
def _archive(self, members: list[str]) -> Path:
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False)
|
||||
tmp.close()
|
||||
archive = Path(tmp.name)
|
||||
with tarfile.open(archive, "w:gz") as handle:
|
||||
for name in members:
|
||||
payload = b"test\n"
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(payload)
|
||||
handle.addfile(info, io.BytesIO(payload))
|
||||
self.addCleanup(archive.unlink, missing_ok=True)
|
||||
return archive
|
||||
|
||||
def test_current_backup_layout_is_accepted(self) -> None:
|
||||
archive = self._archive([
|
||||
"/backup/etc-mike-ai/stack.env",
|
||||
"/backup/volumes/router-state/state.json",
|
||||
])
|
||||
result = subprocess.run(
|
||||
[str(ROOT / "restore.sh"), "--check", str(archive)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("ATHENA_BACKUP_CHECK_OK", result.stdout)
|
||||
|
||||
def test_unexpected_tree_is_rejected(self) -> None:
|
||||
archive = self._archive([
|
||||
"/backup/etc-mike-ai/stack.env",
|
||||
"/backup/volumes/router-state/state.json",
|
||||
"/etc/shadow",
|
||||
])
|
||||
result = subprocess.run(
|
||||
[str(ROOT / "restore.sh"), "--check", str(archive)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("Unerwarteter Pfad", result.stderr)
|
||||
|
||||
def test_backup_prefix_extracts_to_restore_layout(self) -> None:
|
||||
archive = self._archive([
|
||||
"/backup/etc-mike-ai/stack.env",
|
||||
"/backup/volumes/router-state/state.json",
|
||||
])
|
||||
with tempfile.TemporaryDirectory() as target:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"tar", "-xzf", str(archive), "-C", target,
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
root = Path(target, "backup")
|
||||
self.assertTrue(Path(root, "etc-mike-ai", "stack.env").is_file())
|
||||
self.assertTrue(Path(root, "volumes", "router-state", "state.json").is_file())
|
||||
|
||||
def test_installer_uses_current_profiles(self) -> None:
|
||||
installer = (ROOT / "install.sh").read_text(encoding="utf-8")
|
||||
self.assertNotIn("llama-experimental", installer)
|
||||
for profile in ("fast", "medium", "large", "ultra", "uncensored"):
|
||||
self.assertIn(f"llama-{profile}", installer)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,6 +1,6 @@
|
||||
# Aktueller temporärer Laufzustand
|
||||
# Aktueller produktiver Laufzustand
|
||||
|
||||
Stand: 31. August 2026
|
||||
Stand: 1. September 2026
|
||||
|
||||
## Produktive llama.cpp-Runtime
|
||||
|
||||
@@ -20,6 +20,8 @@ beendet.
|
||||
Dabei gilt ausdrücklich:
|
||||
|
||||
- Der Gesamtkontext bleibt bei **160.000 Tokens**.
|
||||
- Das Profil verwendet wieder **einen Slot**. Die getestete Zwei-Slot-Variante
|
||||
ist nicht produktiv.
|
||||
- **MTP / Speculative Decoding bleibt aktiviert**; MTP wurde nicht entfernt.
|
||||
- Modell, Quantisierung, GPU-Aufteilung und KV-Cache-Quantisierung bleiben
|
||||
unverändert.
|
||||
|
||||
@@ -21,9 +21,12 @@ Full-context results with the selected settings:
|
||||
|
||||
The larger logical batches 3072 and 4096 did not improve Medium at ubatch 128. The original one-slot benchmark selected 2048 / 128 at 90:10.
|
||||
|
||||
## Medium two-slot benchmark
|
||||
## Historischer Medium-Zwei-Slot-Test
|
||||
|
||||
Medium now uses two parallel slots with unified KV, so both chats dynamically share one total 160K-token pool. The model weights remain loaded only once. To fit the additional scheduler buffers, the production GPU split is 85:15 while batch / ubatch remains 2048 / 128.
|
||||
This was an A/B candidate, not the current production configuration. Production
|
||||
was returned to **one slot** because concurrent Hermes requests did not behave
|
||||
reliably enough. The 85:15 GPU split and batch / ubatch 2048 / 128 remain in
|
||||
production because they also work with the single-slot profile.
|
||||
|
||||
Identical fresh 100,297-token prompt with a deterministic 256-token completion:
|
||||
|
||||
|
||||
+7
-1
@@ -33,10 +33,16 @@ Router, Modelle oder Dashboard-Daten zu verlieren.
|
||||
4. Letztes Datenarchiv einspielen:
|
||||
|
||||
```bash
|
||||
sudo ./restore.sh --check /data/docker-backups/athena-latest.tar.gz
|
||||
sudo ./restore.sh /data/docker-backups/athena-latest.tar.gz
|
||||
sudo ./smoke-test.sh
|
||||
```
|
||||
|
||||
`--check` liest das komplette gzip-Archiv und prüft dessen sichere
|
||||
`/backup`-Struktur sowie die benötigten Konfigurations- und Volume-Bäume, ohne
|
||||
Container oder Dateien zu verändern. Der reguläre Restore extrahiert und
|
||||
verwendet anschließend ausschließlich diesen einen geprüften Baum.
|
||||
|
||||
Das Restore verändert weder SSH noch LAN, WireGuard, Kernel, Partitionen oder
|
||||
Mounts.
|
||||
|
||||
@@ -58,7 +64,7 @@ Appdata-Restore plus Neuerstellung über die jeweilige Template-XML.
|
||||
|
||||
```bash
|
||||
docker compose --env-file /etc/mike-ai/stack.env ps
|
||||
test -s /data/docker-backups/athena-latest.tar.gz
|
||||
sudo ./restore.sh --check /data/docker-backups/athena-latest.tar.gz
|
||||
curl -fsS http://192.168.1.212:8099/health
|
||||
sudo ./smoke-test.sh
|
||||
```
|
||||
|
||||
+21
-1
@@ -42,6 +42,9 @@ required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE
|
||||
FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL
|
||||
MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256
|
||||
ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256
|
||||
UNCENSORED_MODEL_FILE UNCENSORED_MODEL_URL UNCENSORED_MODEL_SHA256
|
||||
UNCENSORED_PROJECTOR_FILE UNCENSORED_PROJECTOR_URL
|
||||
UNCENSORED_PROJECTOR_SHA256
|
||||
VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256)
|
||||
for name in "${required[@]}"; do
|
||||
[[ -n "${!name:-}" ]] || die "Pflichtwert $name fehlt."
|
||||
@@ -322,10 +325,25 @@ UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE
|
||||
UNCENSORED_PROJECTOR_FILE=$UNCENSORED_PROJECTOR_FILE
|
||||
VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE
|
||||
FAST_CONTEXT=${FAST_CONTEXT:-76800}
|
||||
FAST_BATCH_SIZE=${FAST_BATCH_SIZE:-64}
|
||||
FAST_UBATCH_SIZE=${FAST_UBATCH_SIZE:-32}
|
||||
FAST_PARALLEL_SLOTS=${FAST_PARALLEL_SLOTS:-1}
|
||||
MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000}
|
||||
MEDIUM_BATCH_SIZE=${MEDIUM_BATCH_SIZE:-2048}
|
||||
MEDIUM_UBATCH_SIZE=${MEDIUM_UBATCH_SIZE:-128}
|
||||
MEDIUM_PARALLEL_SLOTS=${MEDIUM_PARALLEL_SLOTS:-1}
|
||||
LARGE_CONTEXT=${LARGE_CONTEXT:-192000}
|
||||
LARGE_BATCH_SIZE=${LARGE_BATCH_SIZE:-2048}
|
||||
LARGE_UBATCH_SIZE=${LARGE_UBATCH_SIZE:-128}
|
||||
LARGE_PARALLEL_SLOTS=${LARGE_PARALLEL_SLOTS:-1}
|
||||
ULTRA_CONTEXT=${ULTRA_CONTEXT:-262144}
|
||||
ULTRA_BATCH_SIZE=${ULTRA_BATCH_SIZE:-2048}
|
||||
ULTRA_UBATCH_SIZE=${ULTRA_UBATCH_SIZE:-128}
|
||||
ULTRA_PARALLEL_SLOTS=${ULTRA_PARALLEL_SLOTS:-1}
|
||||
UNCENSORED_CONTEXT=${UNCENSORED_CONTEXT:-80000}
|
||||
UNCENSORED_BATCH_SIZE=${UNCENSORED_BATCH_SIZE:-2048}
|
||||
UNCENSORED_UBATCH_SIZE=${UNCENSORED_UBATCH_SIZE:-128}
|
||||
UNCENSORED_PARALLEL_SLOTS=${UNCENSORED_PARALLEL_SLOTS:-1}
|
||||
FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
|
||||
MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
|
||||
MEDIUM_TENSOR_SPLIT=${MEDIUM_TENSOR_SPLIT:-90,10}
|
||||
@@ -340,6 +358,8 @@ IMAGE_GPU_DEVICES=${IMAGE_GPU_DEVICES:-${TEXT_GPU_DEVICES:-0}}
|
||||
FLUX_MODEL_DIR=${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}
|
||||
LLAMA_THREADS=${LLAMA_THREADS:-6}
|
||||
LLAMA_THREADS_BATCH=${LLAMA_THREADS_BATCH:-6}
|
||||
LLAMA_CACHE_RAM_MIB=${LLAMA_CACHE_RAM_MIB:-24576}
|
||||
DEFAULT_REASONING_EFFORT=${DEFAULT_REASONING_EFFORT:-off}
|
||||
EOF
|
||||
chmod 0600 $SECRETS_DIR/stack.env
|
||||
}
|
||||
@@ -468,7 +488,7 @@ build_and_start() {
|
||||
# Portable MCPs and Hermes live on Unraid and are restored through Appdata.
|
||||
"$STACK_DIR/platform/mcp/install-tools.sh"
|
||||
docker compose --env-file "$SECRETS_DIR/stack.env" --profile inference create \
|
||||
llama-fast llama-medium llama-large llama-ultra llama-experimental
|
||||
llama-fast llama-medium llama-large llama-ultra llama-uncensored
|
||||
docker compose --env-file "$SECRETS_DIR/stack.env" --profile image create image-worker
|
||||
docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \
|
||||
wireguard-gateway xtts piper tts-gateway profile-controller router llama-dashboard backup
|
||||
|
||||
+45
-11
@@ -4,23 +4,57 @@ set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CHECK_ONLY=false
|
||||
if [[ ${1:-} == --check ]]; then
|
||||
CHECK_ONLY=true
|
||||
shift
|
||||
fi
|
||||
ARCHIVE=${1:-/data/docker-backups/athena-latest.tar.gz}
|
||||
|
||||
die() { printf 'FEHLER: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
[[ -s $ARCHIVE ]] || die "Backup fehlt: $ARCHIVE"
|
||||
|
||||
validate_archive() {
|
||||
local listing normalized entry required
|
||||
gzip -t "$ARCHIVE" || die "Backup ist beschädigt: $ARCHIVE"
|
||||
listing=$(tar -tzf "$ARCHIVE") || die "Backup-Inhalt kann nicht gelesen werden."
|
||||
|
||||
# docker-volume-backup stores the mounted sources below /backup. GNU tar
|
||||
# removes the leading slash while extracting; accept exactly that tree and
|
||||
# reject every unrelated or traversing member before extracting as root.
|
||||
while IFS= read -r entry; do
|
||||
normalized=${entry#./}
|
||||
normalized=${normalized#/}
|
||||
[[ $normalized == backup || $normalized == backup/* ]] || \
|
||||
die "Unerwarteter Pfad im Backup: $entry"
|
||||
[[ /$normalized/ != */../* ]] || die "Unsicherer Pfad im Backup: $entry"
|
||||
done <<<"$listing"
|
||||
|
||||
for required in backup/etc-mike-ai backup/volumes; do
|
||||
grep -Eq "^/?${required}(/|$)" <<<"$listing" || \
|
||||
die "Backup enthält $required nicht."
|
||||
done
|
||||
}
|
||||
|
||||
validate_archive
|
||||
if [[ $CHECK_ONLY == true ]]; then
|
||||
printf 'ATHENA_BACKUP_CHECK_OK %s\n' "$ARCHIVE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ $EUID -eq 0 ]] || die "Bitte als root ausführen."
|
||||
command -v docker >/dev/null || die "Docker fehlt. Zuerst ./install.sh ausführen."
|
||||
|
||||
work=$(mktemp -d /tmp/athena-restore.XXXXXX)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
|
||||
# Refuse absolute paths and parent traversal before extracting as root.
|
||||
if tar -tzf "$ARCHIVE" | grep -Eq '(^/|(^|/)\.\.(/|$))'; then
|
||||
die "Unsichere Pfade im Backup."
|
||||
fi
|
||||
# GNU tar removes the leading slash and extracts the single validated tree as
|
||||
# $work/backup. Keeping that root explicit avoids platform-dependent
|
||||
# --strip-components behaviour between GNU tar and bsdtar.
|
||||
tar -xzf "$ARCHIVE" -C "$work"
|
||||
[[ -d $work/etc-mike-ai ]] || die "Backup enthält etc-mike-ai nicht."
|
||||
[[ -d $work/volumes ]] || die "Backup enthält keine Docker-Volumes."
|
||||
backup_root=$work/backup
|
||||
[[ -d $backup_root/etc-mike-ai ]] || die "Backup enthält etc-mike-ai nicht."
|
||||
[[ -d $backup_root/volumes ]] || die "Backup enthält keine Docker-Volumes."
|
||||
|
||||
# Stop only users of the restored volumes. WireGuard, SSH and networking stay up.
|
||||
for container in mike-ai-router mike-ai-profile-controller mike-ai-piper; do
|
||||
@@ -30,7 +64,7 @@ for container in mike-ai-router mike-ai-profile-controller mike-ai-piper; do
|
||||
done
|
||||
|
||||
install -d -m 0700 /etc/mike-ai
|
||||
rsync -a --delete "$work/etc-mike-ai/" /etc/mike-ai/
|
||||
rsync -a --delete "$backup_root/etc-mike-ai/" /etc/mike-ai/
|
||||
|
||||
restore_volume() {
|
||||
local volume=$1 source=$2 mountpoint
|
||||
@@ -41,9 +75,9 @@ restore_volume() {
|
||||
rsync -a --delete "$source/" "$mountpoint/"
|
||||
}
|
||||
|
||||
restore_volume mike-ai_piper-data "$work/volumes/piper-data"
|
||||
restore_volume mike-ai_router-state "$work/volumes/router-state"
|
||||
restore_volume mike-ai_router-images "$work/volumes/router-images"
|
||||
restore_volume mike-ai_piper-data "$backup_root/volumes/piper-data"
|
||||
restore_volume mike-ai_router-state "$backup_root/volumes/router-state"
|
||||
restore_volume mike-ai_router-images "$backup_root/volumes/router-images"
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
./manage.sh deploy core
|
||||
|
||||
+2
-1
@@ -83,6 +83,7 @@ pass "Router-Liveness und OpenAI-Modellliste funktionieren"
|
||||
|
||||
latest=/data/docker-backups/athena-latest.tar.gz
|
||||
[[ -s $latest ]] || fail "Kein gueltiges Athena-Backup: $latest"
|
||||
pass "Automatisches Athena-Backup ist vorhanden"
|
||||
./restore.sh --check "$latest" >/dev/null || fail "Athena-Backup ist nicht wiederherstellbar"
|
||||
pass "Automatisches Athena-Backup ist vorhanden und strukturell wiederherstellbar"
|
||||
|
||||
printf '\nATHENA_E2E_OK\n'
|
||||
|
||||
Reference in New Issue
Block a user