114 lines
2.2 KiB
JSON
114 lines
2.2 KiB
JSON
{
|
|
"args": [
|
|
"--model",
|
|
"/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf",
|
|
"--mmproj",
|
|
"/models/qwen/mmproj-BF16.gguf",
|
|
"--mmproj-offload",
|
|
"--mmproj-device",
|
|
"CUDA1",
|
|
"--alias",
|
|
"qwen-medium",
|
|
"--ctx-size",
|
|
"160000",
|
|
"--flash-attn",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q4_0",
|
|
"--cache-type-v",
|
|
"q4_0",
|
|
"--cache-prompt",
|
|
"--cache-ram",
|
|
"32768",
|
|
"--threads",
|
|
"6",
|
|
"--threads-batch",
|
|
"6",
|
|
"--batch-size",
|
|
"2048",
|
|
"--ubatch-size",
|
|
"256",
|
|
"--parallel",
|
|
"2",
|
|
"--kv-unified",
|
|
"--jinja",
|
|
"--reasoning",
|
|
"auto",
|
|
"--reasoning-preserve",
|
|
"--host",
|
|
"0.0.0.0",
|
|
"--port",
|
|
"8080",
|
|
"--metrics",
|
|
"--fit",
|
|
"off",
|
|
"--n-gpu-layers",
|
|
"all",
|
|
"--load-mode",
|
|
"none",
|
|
"--no-ui",
|
|
"--temperature",
|
|
"1.0",
|
|
"--top-p",
|
|
"0.95",
|
|
"--top-k",
|
|
"20",
|
|
"--device",
|
|
"CUDA0,CUDA1",
|
|
"--main-gpu",
|
|
"0",
|
|
"--split-mode",
|
|
"layer",
|
|
"--tensor-split",
|
|
"85,15",
|
|
"--spec-type",
|
|
"draft-mtp",
|
|
"--spec-draft-n-max",
|
|
"2",
|
|
"--spec-draft-type-k",
|
|
"f16",
|
|
"--spec-draft-type-v",
|
|
"f16",
|
|
"--spec-draft-p-min",
|
|
"0.05"
|
|
],
|
|
"smoke": {
|
|
"choices": [
|
|
{
|
|
"finish_reason": "stop",
|
|
"index": 0,
|
|
"message": {
|
|
"role": "assistant",
|
|
"content": "OK"
|
|
}
|
|
}
|
|
],
|
|
"created": 1789939528,
|
|
"model": "qwen-medium",
|
|
"system_fingerprint": "b10964-b29c606e2",
|
|
"object": "chat.completion",
|
|
"usage": {
|
|
"completion_tokens": 2,
|
|
"prompt_tokens": 19,
|
|
"total_tokens": 21,
|
|
"prompt_tokens_details": {
|
|
"cached_tokens": 0
|
|
}
|
|
},
|
|
"id": "chatcmpl-PAF96fKOq1RJLre26QZBbmTRrk5mcFBi",
|
|
"timings": {
|
|
"cache_n": 0,
|
|
"prompt_n": 19,
|
|
"prompt_ms": 774.188,
|
|
"prompt_per_token_ms": 40.746736842105264,
|
|
"prompt_per_second": 24.54184254987161,
|
|
"predicted_n": 2,
|
|
"predicted_ms": 287.02,
|
|
"predicted_per_token_ms": 287.02,
|
|
"predicted_per_second": 3.4840777646157064,
|
|
"draft_n": 2,
|
|
"draft_n_accepted": 2
|
|
}
|
|
},
|
|
"time": 1789939528.283845
|
|
} |