Add final Qwen3.8 runtime and uncensored profile
This commit is contained in:
+88
-1
@@ -76,6 +76,8 @@ services:
|
||||
- --mmproj
|
||||
- "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
|
||||
- --mmproj-offload
|
||||
- --mmproj-device
|
||||
- CUDA1
|
||||
- --alias
|
||||
- qwen-fast
|
||||
- --ctx-size
|
||||
@@ -146,6 +148,8 @@ services:
|
||||
- --mmproj
|
||||
- "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
|
||||
- --mmproj-offload
|
||||
- --mmproj-device
|
||||
- CUDA1
|
||||
- --alias
|
||||
- qwen-medium
|
||||
- --ctx-size
|
||||
@@ -202,6 +206,8 @@ services:
|
||||
- f16
|
||||
- --spec-draft-type-v
|
||||
- f16
|
||||
- --spec-draft-p-min
|
||||
- "0.05"
|
||||
|
||||
llama-large:
|
||||
<<: *llama-common
|
||||
@@ -218,6 +224,8 @@ services:
|
||||
- --mmproj
|
||||
- "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
|
||||
- --mmproj-offload
|
||||
- --mmproj-device
|
||||
- CUDA1
|
||||
- --alias
|
||||
- qwen-large
|
||||
- --ctx-size
|
||||
@@ -346,6 +354,85 @@ services:
|
||||
- --spec-draft-type-v
|
||||
- f16
|
||||
|
||||
# Deliberately less refusal-prone weight-level ablation. It remains behind
|
||||
# the same authenticated router, tool permissions and confirmation guards as
|
||||
# every other profile; "uncensored" never means unrestricted tool access.
|
||||
llama-uncensored:
|
||||
<<: *llama-common
|
||||
container_name: mike-ai-llama-uncensored
|
||||
labels:
|
||||
com.mike-ai.llama-profile: uncensored
|
||||
environment:
|
||||
NVIDIA_VISIBLE_DEVICES: ${UNCENSORED_GPU_DEVICES:-0,1}
|
||||
NVIDIA_DRIVER_CAPABILITIES: compute,utility
|
||||
MTMD_BACKEND_DEVICE: CUDA1
|
||||
command:
|
||||
- --model
|
||||
- "/models/${UNCENSORED_MODEL_FILE:?UNCENSORED_MODEL_FILE is required}"
|
||||
- --mmproj
|
||||
- "/models/${UNCENSORED_PROJECTOR_FILE:?UNCENSORED_PROJECTOR_FILE is required}"
|
||||
- --mmproj-offload
|
||||
- --mmproj-device
|
||||
- CUDA1
|
||||
- --alias
|
||||
- qwen-uncensored
|
||||
- --ctx-size
|
||||
- "${UNCENSORED_CONTEXT:-80000}"
|
||||
- --flash-attn
|
||||
- "on"
|
||||
- --cache-type-k
|
||||
- q4_0
|
||||
- --cache-type-v
|
||||
- q4_0
|
||||
- --threads
|
||||
- "${LLAMA_THREADS:-6}"
|
||||
- --threads-batch
|
||||
- "${LLAMA_THREADS_BATCH:-6}"
|
||||
- --batch-size
|
||||
- "64"
|
||||
- --ubatch-size
|
||||
- "32"
|
||||
- --parallel
|
||||
- "1"
|
||||
- --jinja
|
||||
- --reasoning
|
||||
- auto
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --metrics
|
||||
- --fit
|
||||
- "off"
|
||||
- --n-gpu-layers
|
||||
- all
|
||||
- --no-mmap
|
||||
- --no-ui
|
||||
- --temperature
|
||||
- "0.2"
|
||||
- --top-p
|
||||
- "0.8"
|
||||
- --top-k
|
||||
- "20"
|
||||
- --device
|
||||
- CUDA0,CUDA1
|
||||
- --main-gpu
|
||||
- "0"
|
||||
- --split-mode
|
||||
- layer
|
||||
- --tensor-split
|
||||
- "${UNCENSORED_TENSOR_SPLIT:-90,10}"
|
||||
- --spec-type
|
||||
- draft-mtp
|
||||
- --spec-draft-n-max
|
||||
- "${UNCENSORED_MTP_MAX:-2}"
|
||||
- --spec-draft-p-min
|
||||
- "0.10"
|
||||
- --spec-draft-type-k
|
||||
- f16
|
||||
- --spec-draft-type-v
|
||||
- f16
|
||||
|
||||
llama-experimental:
|
||||
<<: *llama-common
|
||||
container_name: mike-ai-llama-experimental
|
||||
@@ -400,7 +487,7 @@ services:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
CONTROLLER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}"
|
||||
ALLOWED_PROFILES: fast,medium,large,ultra,experimental
|
||||
ALLOWED_PROFILES: fast,medium,large,ultra,uncensored,experimental
|
||||
IMAGE_WORKER: flux
|
||||
networks: [control]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
|
||||
Reference in New Issue
Block a user