michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add reproducible Docker and WireGuard host bootstrap

Browse Source
This commit is contained in:
Mikei386
2026-08-20 21:23:16 +02:00
parent cb07779f5a
commit e83c0e2c70
25 changed files with 1584 additions and 835 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose.yaml
+399
View File
@@ -0,0 +1,399 @@
name: mike-ai
x-llama-common: &llama-common
image: ${LLAMA_IMAGE:-mike-ai/llama.cpp:local}
restart: "no"
profiles: [inference]
gpus: all
ipc: host
read_only: true
tmpfs:
- /tmp:size=1g,mode=1777
volumes:
- "${MODEL_DIR:-/srv/mike-ai/models}:/models:ro"
- ./platform/docker/mcp-standard.json:/etc/mike-ai/mcp-standard.json:ro
environment:
SEARXNG_URL: http://searxng:8080
NVIDIA_DRIVER_CAPABILITIES: compute,utility
dns: ["${AI_DNS:-1.1.1.1}"]
networks:
inference:
aliases: [llama-upstream]
search: {}
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
healthcheck:
test: [CMD, curl, -fsS, "http://127.0.0.1:8080/health"]
interval: 10s
timeout: 5s
retries: 60
start_period: 30s
services:
llama-fast:
<<: *llama-common
container_name: mike-ai-llama-fast
labels:
com.mike-ai.llama-profile: fast
environment:
SEARXNG_URL: http://searxng:8080
NVIDIA_VISIBLE_DEVICES: ${FAST_GPU_DEVICES:-0}
NVIDIA_DRIVER_CAPABILITIES: compute,utility
command:
- --model
- "/models/${FAST_MODEL_FILE:?FAST_MODEL_FILE is required}"
- --mmproj
- "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
- --no-mmproj-offload
- --alias
- qwen-fast
- --ctx-size
- "${FAST_CONTEXT:-76800}"
- --flash-attn
- "on"
- --cache-type-k
- q4_0
- --cache-type-v
- q4_0
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
- "${LLAMA_THREADS_BATCH:-6}"
- --batch-size
- "64"
- --ubatch-size
- "32"
- --parallel
- "1"
- --jinja
- --reasoning
- auto
- --host
- 0.0.0.0
- --port
- "8080"
- --metrics
- --fit
- "off"
- --n-gpu-layers
- all
- --no-mmap
- --no-ui
- --temperature
- "0.2"
- --top-p
- "0.8"
- --top-k
- "20"
- --mcp-servers-config
- /etc/mike-ai/mcp-standard.json
- --device
- CUDA0
- --split-mode
- none
- --spec-type
- draft-mtp
- --model-draft
- "/models/${MTP_MODEL_FILE:?MTP_MODEL_FILE is required}"
- --spec-draft-n-max
- "2"
- --spec-draft-type-k
- f16
- --spec-draft-type-v
- f16
llama-medium:
<<: *llama-common
container_name: mike-ai-llama-medium
labels:
com.mike-ai.llama-profile: medium
environment:
SEARXNG_URL: http://searxng:8080
NVIDIA_VISIBLE_DEVICES: ${MEDIUM_GPU_DEVICES:-0}
NVIDIA_DRIVER_CAPABILITIES: compute,utility
command:
- --model
- "/models/${MEDIUM_MODEL_FILE:?MEDIUM_MODEL_FILE is required}"
- --mmproj
- "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
- --no-mmproj-offload
- --alias
- qwen-medium
- --ctx-size
- "${MEDIUM_CONTEXT:-94208}"
- --flash-attn
- "on"
- --cache-type-k
- q4_0
- --cache-type-v
- q4_0
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
- "${LLAMA_THREADS_BATCH:-6}"
- --batch-size
- "64"
- --ubatch-size
- "32"
- --parallel
- "1"
- --jinja
- --reasoning
- auto
- --host
- 0.0.0.0
- --port
- "8080"
- --metrics
- --fit
- "off"
- --n-gpu-layers
- all
- --no-mmap
- --no-ui
- --temperature
- "0.2"
- --top-p
- "0.8"
- --top-k
- "20"
- --mcp-servers-config
- /etc/mike-ai/mcp-standard.json
- --device
- CUDA0
- --split-mode
- none
llama-long:
<<: *llama-common
container_name: mike-ai-llama-long
labels:
com.mike-ai.llama-profile: long
environment:
SEARXNG_URL: http://searxng:8080
NVIDIA_VISIBLE_DEVICES: ${LONG_GPU_DEVICES:-0}
NVIDIA_DRIVER_CAPABILITIES: compute,utility
command:
- --model
- "/models/${LONG_MODEL_FILE:?LONG_MODEL_FILE is required}"
- --mmproj
- "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
- --no-mmproj-offload
- --alias
- qwen-long
- --ctx-size
- "${LONG_CONTEXT:-131072}"
- --flash-attn
- "on"
- --cache-type-k
- q4_0
- --cache-type-v
- q4_0
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
- "${LLAMA_THREADS_BATCH:-6}"
- --batch-size
- "64"
- --ubatch-size
- "32"
- --parallel
- "1"
- --jinja
- --reasoning
- auto
- --host
- 0.0.0.0
- --port
- "8080"
- --metrics
- --fit
- "off"
- --n-gpu-layers
- all
- --override-tensor
- blk.([0-9]|1[0-1]).ffn_.*=CPU
- --no-mmap
- --no-ui
- --temperature
- "0.2"
- --top-p
- "0.8"
- --top-k
- "20"
- --mcp-servers-config
- /etc/mike-ai/mcp-standard.json
- --device
- CUDA0
- --split-mode
- none
- --spec-type
- draft-mtp
- --model-draft
- "/models/${MTP_MODEL_FILE:?MTP_MODEL_FILE is required}"
- --spec-draft-n-max
- "2"
- --spec-draft-type-k
- q4_0
- --spec-draft-type-v
- q4_0
llama-experimental:
<<: *llama-common
container_name: mike-ai-llama-experimental
labels:
com.mike-ai.llama-profile: experimental
environment:
SEARXNG_URL: http://searxng:8080
NVIDIA_VISIBLE_DEVICES: ${EXPERIMENTAL_GPU_DEVICES:-0}
NVIDIA_DRIVER_CAPABILITIES: compute,utility
command:
- --model
- "/models/${EXPERIMENTAL_MODEL_FILE:?EXPERIMENTAL_MODEL_FILE is required}"
- --alias
- qwen-experimental
- --ctx-size
- "${EXPERIMENTAL_CONTEXT:-76800}"
- --flash-attn
- "on"
- --cache-type-k
- q4_0
- --cache-type-v
- q4_0
- --parallel
- "1"
- --jinja
- --reasoning
- auto
- --host
- 0.0.0.0
- --port
- "8080"
- --metrics
- --fit
- "off"
- --n-gpu-layers
- all
- --no-mmap
- --no-ui
- --mcp-servers-config
- /etc/mike-ai/mcp-standard.json
- --device
- CUDA0
- --split-mode
- none
profile-controller:
build: ./platform/docker/profile-controller
image: mike-ai/profile-controller:local
container_name: mike-ai-profile-controller
restart: unless-stopped
read_only: true
tmpfs: ["/tmp:size=16m"]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
CONTROLLER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}"
ALLOWED_PROFILES: fast,medium,long,experimental
networks: [control]
security_opt: ["no-new-privileges:true"]
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/health', timeout=2)"]
interval: 10s
timeout: 3s
retries: 10
router:
build:
context: .
dockerfile: platform/docker/router/Dockerfile
image: mike-ai/profile-router:local
container_name: mike-ai-router
restart: unless-stopped
read_only: true
tmpfs: ["/tmp:size=256m"]
volumes:
- ./router/router_profiles.json:/etc/mike-ai/router-profiles.json:ro
- router-state:/var/lib/mike-ai-profile-router
- router-images:/data/images
environment:
ROUTER_HOST: 0.0.0.0
ROUTER_PORT: "8081"
ROUTER_AUTH_MODE: required
ROUTER_API_KEY: "${ROUTER_API_KEY:?ROUTER_API_KEY is required}"
ROUTER_PROFILES_FILE: /etc/mike-ai/router-profiles.json
ROUTER_STATE_FILE: /var/lib/mike-ai-profile-router/state.json
ROUTER_MAX_CONCURRENT_REQUESTS: "16"
UPSTREAM_URL: http://llama-upstream:8080
PROFILE_CONTROL_URL: http://profile-controller:8090
PROFILE_CONTROL_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}"
SWITCH_TIMEOUT: "600"
REQUEST_TIMEOUT: "600"
IMAGE_DIR: /data/images
CHAT_IMAGE_ALLOW_REMOTE_URLS: "false"
ENABLE_IMAGE_GENERATION: "false"
ENABLE_TTS: "false"
ENABLE_STT: "false"
ports:
- "${AI_BIND_ADDRESS:-127.0.0.1}:8081:8081"
networks: [frontend, control, inference]
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
depends_on:
profile-controller:
condition: service_healthy
open-webui:
image: ${OPENWEBUI_IMAGE:-ghcr.io/open-webui/open-webui:v0.9.5}
container_name: mike-ai-open-webui
restart: unless-stopped
volumes:
- open-webui-data:/app/backend/data
environment:
WEBUI_SECRET_KEY: "${WEBUI_SECRET_KEY:?WEBUI_SECRET_KEY is required}"
OLLAMA_BASE_URL: ""
OPENAI_API_BASE_URLS: http://router:8081/v1
OPENAI_API_KEYS: "${ROUTER_API_KEY:?ROUTER_API_KEY is required}"
ENABLE_SIGNUP: ${OPENWEBUI_ENABLE_SIGNUP:-false}
DO_NOT_TRACK: "true"
SCARF_NO_ANALYTICS: "true"
ports:
- "${AI_BIND_ADDRESS:-127.0.0.1}:8080:8080"
dns: ["${AI_DNS:-1.1.1.1}"]
networks: [frontend]
depends_on: [router]
security_opt: ["no-new-privileges:true"]
searxng:
image: searxng/searxng@sha256:e45d5894bfaa0bf8773b9f283795ae57f1c15ddb29c8cecb70b3665b0ce9ec60
container_name: mike-ai-searxng
restart: unless-stopped
volumes:
- ./platform/web-search/searxng-settings.yml:/etc/searxng/settings.yml:ro
networks: [search]
dns: ["${AI_DNS:-1.1.1.1}"]
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
networks:
frontend:
internal: false
ipam:
config: [{subnet: 172.30.10.0/24}]
control:
internal: true
ipam:
config: [{subnet: 172.30.20.0/24}]
inference:
internal: true
ipam:
config: [{subnet: 172.30.30.0/24}]
search:
internal: false
ipam:
config: [{subnet: 172.30.40.0/24}]
volumes:
open-webui-data:
router-state:
router-images: