198 lines
4.5 KiB
JSON
198 lines
4.5 KiB
JSON
{
|
|
"case": {
|
|
"id": "11-latest-abliterated-80k-mtp2-p010",
|
|
"model": "abliterated-latest",
|
|
"context": 80000,
|
|
"split": [
|
|
72,
|
|
28
|
|
],
|
|
"projector": "gpu",
|
|
"mtp": true,
|
|
"mtp_max": 2,
|
|
"quality": false,
|
|
"long_fill": 0.0,
|
|
"vision": true
|
|
},
|
|
"model_path": "/models/qwen3.8-27b-abliterated/Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf",
|
|
"started": "2026-08-22T21:28:26.289184+00:00",
|
|
"command": [
|
|
"docker",
|
|
"run",
|
|
"-d",
|
|
"--name",
|
|
"mike-ai-qwen38-dualgpu-bench",
|
|
"--gpus",
|
|
"all",
|
|
"--ipc",
|
|
"host",
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp:size=1g,mode=1777",
|
|
"--security-opt",
|
|
"no-new-privileges:true",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"-e",
|
|
"NVIDIA_VISIBLE_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b",
|
|
"-v",
|
|
"/data/models:/models:ro",
|
|
"-p",
|
|
"127.0.0.1:18099:8080",
|
|
"-e",
|
|
"MTMD_BACKEND_DEVICE=CUDA1",
|
|
"mike-ai/llama.cpp:3f545bec-test",
|
|
"--model",
|
|
"/models/qwen3.8-27b-abliterated/Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf",
|
|
"--alias",
|
|
"11-latest-abliterated-80k-mtp2-p010",
|
|
"--ctx-size",
|
|
"80000",
|
|
"--flash-attn",
|
|
"on",
|
|
"--cache-type-k",
|
|
"q4_0",
|
|
"--cache-type-v",
|
|
"q4_0",
|
|
"--threads",
|
|
"6",
|
|
"--threads-batch",
|
|
"6",
|
|
"--batch-size",
|
|
"64",
|
|
"--ubatch-size",
|
|
"32",
|
|
"--parallel",
|
|
"1",
|
|
"--jinja",
|
|
"--reasoning",
|
|
"auto",
|
|
"--reasoning-budget",
|
|
"768",
|
|
"--host",
|
|
"0.0.0.0",
|
|
"--port",
|
|
"8080",
|
|
"--metrics",
|
|
"--fit",
|
|
"off",
|
|
"--n-gpu-layers",
|
|
"all",
|
|
"--no-mmap",
|
|
"--no-ui",
|
|
"--temperature",
|
|
"0.2",
|
|
"--top-p",
|
|
"0.8",
|
|
"--top-k",
|
|
"20",
|
|
"--device",
|
|
"CUDA0,CUDA1",
|
|
"--main-gpu",
|
|
"0",
|
|
"--split-mode",
|
|
"layer",
|
|
"--tensor-split",
|
|
"72,28",
|
|
"--mmproj",
|
|
"/models/qwen3.8-27b-abliterated/mmproj-Qwen3.8-27B-ABLITERATED-F16.gguf",
|
|
"--image-min-tokens",
|
|
"1024",
|
|
"--spec-type",
|
|
"draft-mtp",
|
|
"--spec-draft-n-max",
|
|
"2",
|
|
"--spec-draft-type-k",
|
|
"f16",
|
|
"--spec-draft-type-v",
|
|
"f16",
|
|
"--mmproj-device",
|
|
"CUDA1",
|
|
"--spec-draft-p-min",
|
|
"0.10"
|
|
],
|
|
"docker_launch_returncode": 0,
|
|
"docker_launch_stderr": "",
|
|
"loaded": true,
|
|
"load_seconds": 8.106,
|
|
"gpu_after_load": [
|
|
{
|
|
"index": "0",
|
|
"name": "NVIDIA GeForce RTX 3060",
|
|
"uuid": "GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b",
|
|
"total_mib": "12288",
|
|
"used_mib": "7295",
|
|
"free_mib": "4615",
|
|
"temperature_c": "50",
|
|
"power_w": "44.72"
|
|
},
|
|
{
|
|
"index": "1",
|
|
"name": "NVIDIA GeForce RTX 5080",
|
|
"uuid": "GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe",
|
|
"total_mib": "16303",
|
|
"used_mib": "12524",
|
|
"free_mib": "3358",
|
|
"temperature_c": "52",
|
|
"power_w": "57.75"
|
|
}
|
|
],
|
|
"speed_runs": [
|
|
{
|
|
"tokens": 384,
|
|
"timings": {
|
|
"cache_n": 0,
|
|
"prompt_n": 28,
|
|
"prompt_ms": 222.267,
|
|
"prompt_per_token_ms": 7.938107142857143,
|
|
"prompt_per_second": 125.97461611485285,
|
|
"predicted_n": 384,
|
|
"predicted_ms": 8564.865,
|
|
"predicted_per_token_ms": 22.362571801566578,
|
|
"predicted_per_second": 44.717575817015216,
|
|
"draft_n": 324,
|
|
"draft_n_accepted": 221
|
|
}
|
|
},
|
|
{
|
|
"tokens": 384,
|
|
"timings": {
|
|
"cache_n": 0,
|
|
"prompt_n": 28,
|
|
"prompt_ms": 203.501,
|
|
"prompt_per_token_ms": 7.267892857142857,
|
|
"prompt_per_second": 137.5914614670198,
|
|
"predicted_n": 384,
|
|
"predicted_ms": 8556.474,
|
|
"predicted_per_token_ms": 22.34066318537859,
|
|
"predicted_per_second": 44.76142859780793,
|
|
"draft_n": 324,
|
|
"draft_n_accepted": 221
|
|
}
|
|
}
|
|
],
|
|
"mean_predicted_tps": 44.739502207411576,
|
|
"vision": {
|
|
"ok": true,
|
|
"elapsed_seconds": 27.45,
|
|
"content": "",
|
|
"timings": {
|
|
"cache_n": 0,
|
|
"prompt_n": 4358,
|
|
"prompt_ms": 15624.828,
|
|
"prompt_per_token_ms": 3.585320789352914,
|
|
"prompt_per_second": 278.91507029709385,
|
|
"predicted_n": 500,
|
|
"predicted_ms": 11572.721,
|
|
"predicted_per_token_ms": 23.191825651302604,
|
|
"predicted_per_second": 43.11864081057515,
|
|
"draft_n": 429,
|
|
"draft_n_accepted": 284
|
|
}
|
|
},
|
|
"status": "complete",
|
|
"server_log": "/data/benchmarks/qwen38-final-acceptance-20260822/cases/11-latest-abliterated-80k-mtp2-p010/server.log",
|
|
"oom": false,
|
|
"finished": "2026-08-22T21:29:20.614813+00:00"
|
|
}
|