michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tested 256K Ultra profile

Browse Source
This commit is contained in:
Mikei386
2026-08-22 10:44:14 +02:00
parent a194a2941d
commit f52cf14069
12 changed files with 111 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
install.sh
+7 -1
View File
@@ -41,6 +41,7 @@ source "$CONFIG"
required=(AI_HOSTNAME ADMIN_USER AI_BIND_ADDRESS MODEL_DIR FAST_MODEL_FILE
FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL
MEDIUM_MODEL_SHA256 LONG_MODEL_FILE LONG_MODEL_URL LONG_MODEL_SHA256
ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256
EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256
VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256)
for name in "${required[@]}"; do
@@ -209,15 +210,19 @@ AI_DNS=${WG_DNS:-1.1.1.1}
FAST_MODEL_FILE=$FAST_MODEL_FILE
MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE
LONG_MODEL_FILE=$LONG_MODEL_FILE
ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE
EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE
VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE
FAST_CONTEXT=${FAST_CONTEXT:-76800}
MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-94208}
LONG_CONTEXT=${LONG_CONTEXT:-131072}
ULTRA_CONTEXT=${ULTRA_CONTEXT:-262144}
EXPERIMENTAL_CONTEXT=${EXPERIMENTAL_CONTEXT:-76800}
FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
LONG_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
ULTRA_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
ULTRA_TENSOR_SPLIT=${ULTRA_TENSOR_SPLIT:-80,20}
EXPERIMENTAL_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
LLAMA_THREADS=${LLAMA_THREADS:-6}
LLAMA_THREADS_BATCH=${LLAMA_THREADS_BATCH:-6}
@@ -256,6 +261,7 @@ download_models() {
$FAST_MODEL_FILE|$FAST_MODEL_URL|$FAST_MODEL_SHA256
$MEDIUM_MODEL_FILE|$MEDIUM_MODEL_URL|$MEDIUM_MODEL_SHA256
$LONG_MODEL_FILE|$LONG_MODEL_URL|$LONG_MODEL_SHA256
$ULTRA_MODEL_FILE|$ULTRA_MODEL_URL|$ULTRA_MODEL_SHA256
$EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256
$VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256
EOF
@@ -323,7 +329,7 @@ build_and_start() {
# secret files and required local artifacts are present.
"$STACK_DIR/platform/mcp/install-tools.sh"
docker compose --env-file "$SECRETS_DIR/stack.env" --profile inference create \
llama-fast llama-medium llama-long llama-experimental
llama-fast llama-medium llama-long llama-ultra llama-experimental
docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \
profile-controller router open-webui