Record bounded Medium GPU split and MTP comparison
This commit is contained in:
Mikei386 committed
1 parent
3cbcf41fab
commit
635b3c4ba9
32 files changed
+5107
No files matched your search
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"label": "83-mtp3",
|
||||
"ubatch": 256,
|
||||
"split": "83,17",
|
||||
"mtp": 3,
|
||||
"prompts": [
|
||||
24576
|
||||
],
|
||||
"minimum_headroom_mib": 448
|
||||
}
|
||||
@@ -0,0 +1,623 @@
|
||||
[
|
||||
{
|
||||
"time": 1789936766.905543,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "7346",
|
||||
"total": "12288",
|
||||
"temp": "50",
|
||||
"util": "100",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "10888",
|
||||
"total": "16303",
|
||||
"temp": "56",
|
||||
"util": "0",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936768.9340215,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "9024",
|
||||
"total": "12288",
|
||||
"temp": "50",
|
||||
"util": "0",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15228",
|
||||
"total": "16303",
|
||||
"temp": "55",
|
||||
"util": "1",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936770.9613502,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11222",
|
||||
"total": "12288",
|
||||
"temp": "49",
|
||||
"util": "0",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15256",
|
||||
"total": "16303",
|
||||
"temp": "54",
|
||||
"util": "0",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936772.992449,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11222",
|
||||
"total": "12288",
|
||||
"temp": "48",
|
||||
"util": "0",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15256",
|
||||
"total": "16303",
|
||||
"temp": "53",
|
||||
"util": "0",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936775.0237627,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "54",
|
||||
"util": "48",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "59",
|
||||
"util": "44",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936777.056265,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "54",
|
||||
"util": "60",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "62",
|
||||
"util": "34",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936779.0883763,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "55",
|
||||
"util": "59",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "59",
|
||||
"util": "46",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936781.11879,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "55",
|
||||
"util": "49",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "59",
|
||||
"util": "41",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936783.1498864,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "51",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "61",
|
||||
"util": "48",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936785.180021,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "55",
|
||||
"util": "49",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "62",
|
||||
"util": "34",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936787.2098007,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "54",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "63",
|
||||
"util": "41",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936789.240345,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "52",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "63",
|
||||
"util": "39",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936791.2699392,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "59",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "64",
|
||||
"util": "36",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936793.3014712,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "56",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "60",
|
||||
"util": "46",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936795.3310409,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "61",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "62",
|
||||
"util": "50",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936797.3619807,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11228",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "55",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15272",
|
||||
"total": "16303",
|
||||
"temp": "60",
|
||||
"util": "45",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936799.3926547,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "56",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "59",
|
||||
"util": "15",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936801.422532,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "32",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "72",
|
||||
"util": "98",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936803.4543598,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "45",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "73",
|
||||
"util": "98",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936805.4877064,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "55",
|
||||
"util": "70",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "73",
|
||||
"util": "98",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936807.533202,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "68",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "72",
|
||||
"util": "12",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936809.5705192,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "70",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "75",
|
||||
"util": "46",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936811.6039462,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "55",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "64",
|
||||
"util": "40",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936813.633929,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "51",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "62",
|
||||
"util": "39",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936815.6666515,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "57",
|
||||
"util": "54",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "62",
|
||||
"util": "40",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936817.7020009,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "56",
|
||||
"util": "46",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "62",
|
||||
"util": "40",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"time": 1789936819.7318478,
|
||||
"gpus": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11230",
|
||||
"total": "12288",
|
||||
"temp": "58",
|
||||
"util": "51",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15274",
|
||||
"total": "16303",
|
||||
"temp": "61",
|
||||
"util": "40",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"case": {
|
||||
"label": "83-mtp3",
|
||||
"ubatch": 256,
|
||||
"split": "83,17",
|
||||
"mtp": 3,
|
||||
"prompts": [
|
||||
24576
|
||||
],
|
||||
"minimum_headroom_mib": 448
|
||||
},
|
||||
"started": 1789936764.8736556,
|
||||
"idle_gpu": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11222",
|
||||
"total": "12288",
|
||||
"temp": "48",
|
||||
"util": "0",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15256",
|
||||
"total": "16303",
|
||||
"temp": "53",
|
||||
"util": "0",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
],
|
||||
"props": {
|
||||
"default_generation_settings": {
|
||||
"params": {
|
||||
"seed": 4294967295,
|
||||
"temperature": 1.0,
|
||||
"dynatemp_range": 0.0,
|
||||
"dynatemp_exponent": 1.0,
|
||||
"top_k": 20,
|
||||
"top_p": 0.949999988079071,
|
||||
"min_p": 0.05000000074505806,
|
||||
"top_n_sigma": -1.0,
|
||||
"xtc_probability": 0.0,
|
||||
"xtc_threshold": 0.10000000149011612,
|
||||
"typical_p": 1.0,
|
||||
"repeat_last_n": 64,
|
||||
"repeat_penalty": 1.0,
|
||||
"presence_penalty": 0.0,
|
||||
"frequency_penalty": 0.0,
|
||||
"dry_multiplier": 0.0,
|
||||
"dry_base": 1.75,
|
||||
"dry_allowed_length": 2,
|
||||
"dry_penalty_last_n": 64,
|
||||
"mirostat": 0,
|
||||
"mirostat_tau": 5.0,
|
||||
"mirostat_eta": 0.10000000149011612,
|
||||
"adaptive_target": -1.0,
|
||||
"adaptive_decay": 0.8999999761581421,
|
||||
"max_tokens": -1,
|
||||
"n_predict": -1,
|
||||
"n_keep": 0,
|
||||
"n_discard": 0,
|
||||
"ignore_eos": false,
|
||||
"stream": false,
|
||||
"n_probs": 0,
|
||||
"min_keep": 0,
|
||||
"chat_format": "Content-only",
|
||||
"reasoning_format": "none",
|
||||
"reasoning_in_content": false,
|
||||
"generation_prompt": "",
|
||||
"samplers": [
|
||||
"penalties",
|
||||
"dry",
|
||||
"top_n_sigma",
|
||||
"top_k",
|
||||
"typ_p",
|
||||
"top_p",
|
||||
"min_p",
|
||||
"xtc",
|
||||
"temperature"
|
||||
],
|
||||
"speculative.types": "none",
|
||||
"timings_per_token": false,
|
||||
"post_sampling_probs": false,
|
||||
"backend_sampling": false,
|
||||
"lora": []
|
||||
},
|
||||
"n_ctx": 160000
|
||||
},
|
||||
"total_slots": 2,
|
||||
"model_alias": "qwen-medium",
|
||||
"model_ftype": "IQ4_XS - 4.25 bpw",
|
||||
"model_path": "/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf",
|
||||
"modalities": {
|
||||
"vision": true,
|
||||
"video": true,
|
||||
"audio": false
|
||||
},
|
||||
"media_marker": "<__media_6beqV5u0Vr11b3oODbUSmmU1ZGgPZj1J__>",
|
||||
"endpoint_slots": true,
|
||||
"endpoint_props": false,
|
||||
"endpoint_metrics": true,
|
||||
"ui": false,
|
||||
"ui_settings": {},
|
||||
"chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set sysns = namespace(count=0, text='') %}\n{%- for message in messages %}\n {%- if sysns.count == loop.index0 and (message.role == 'system' or message.role == 'developer') %}\n {%- set sys_content = render_content(message.content, false, true)|trim %}\n {%- if sys_content %}\n {%- set sysns.text = sysns.text + ('\\n' if sysns.text else '') + sys_content %}\n {%- endif %}\n {%- set sysns.count = sysns.count + 1 %}\n {%- endif %}\n{%- endfor %}\n{%- set num_sys = sysns.count %}\n{%- set merged_system = sysns.text %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort == 'high' %}\n {%- set resolved_reasoning_effort = 'xhigh' %}\n {%- endif %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calLine truncated
|
||||
"chat_template_caps": {
|
||||
"supports_object_arguments": true,
|
||||
"supports_parallel_tool_calls": true,
|
||||
"supports_preserve_reasoning": true,
|
||||
"supports_reasoning_effort": true,
|
||||
"supports_string_content": true,
|
||||
"supports_system_role": true,
|
||||
"supports_tool_calls": true,
|
||||
"supports_tools": true,
|
||||
"supports_typed_content": true
|
||||
},
|
||||
"bos_token": "<|endoftext|>",
|
||||
"eos_token": "<|im_end|>",
|
||||
"build_info": "b10964-b29c606e2",
|
||||
"is_sleeping": false,
|
||||
"cors_proxy_enabled": false
|
||||
},
|
||||
"slots": [
|
||||
{
|
||||
"id": 0,
|
||||
"n_ctx": 160000,
|
||||
"speculative": true,
|
||||
"is_processing": false
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"n_ctx": 160000,
|
||||
"speculative": true,
|
||||
"is_processing": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
{
|
||||
"case": {
|
||||
"label": "83-mtp3",
|
||||
"ubatch": 256,
|
||||
"split": "83,17",
|
||||
"mtp": 3,
|
||||
"prompts": [
|
||||
24576
|
||||
],
|
||||
"minimum_headroom_mib": 448
|
||||
},
|
||||
"started": 1789936764.8736556,
|
||||
"idle_gpu": [
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 3060",
|
||||
"used": "11222",
|
||||
"total": "12288",
|
||||
"temp": "48",
|
||||
"util": "0",
|
||||
"pcie_gen": "3",
|
||||
"pcie_width": "4"
|
||||
},
|
||||
{
|
||||
"name": "NVIDIA GeForce RTX 5080",
|
||||
"used": "15256",
|
||||
"total": "16303",
|
||||
"temp": "53",
|
||||
"util": "0",
|
||||
"pcie_gen": "4",
|
||||
"pcie_width": "16"
|
||||
}
|
||||
],
|
||||
"props": {
|
||||
"default_generation_settings": {
|
||||
"params": {
|
||||
"seed": 4294967295,
|
||||
"temperature": 1.0,
|
||||
"dynatemp_range": 0.0,
|
||||
"dynatemp_exponent": 1.0,
|
||||
"top_k": 20,
|
||||
"top_p": 0.949999988079071,
|
||||
"min_p": 0.05000000074505806,
|
||||
"top_n_sigma": -1.0,
|
||||
"xtc_probability": 0.0,
|
||||
"xtc_threshold": 0.10000000149011612,
|
||||
"typical_p": 1.0,
|
||||
"repeat_last_n": 64,
|
||||
"repeat_penalty": 1.0,
|
||||
"presence_penalty": 0.0,
|
||||
"frequency_penalty": 0.0,
|
||||
"dry_multiplier": 0.0,
|
||||
"dry_base": 1.75,
|
||||
"dry_allowed_length": 2,
|
||||
"dry_penalty_last_n": 64,
|
||||
"mirostat": 0,
|
||||
"mirostat_tau": 5.0,
|
||||
"mirostat_eta": 0.10000000149011612,
|
||||
"adaptive_target": -1.0,
|
||||
"adaptive_decay": 0.8999999761581421,
|
||||
"max_tokens": -1,
|
||||
"n_predict": -1,
|
||||
"n_keep": 0,
|
||||
"n_discard": 0,
|
||||
"ignore_eos": false,
|
||||
"stream": false,
|
||||
"n_probs": 0,
|
||||
"min_keep": 0,
|
||||
"chat_format": "Content-only",
|
||||
"reasoning_format": "none",
|
||||
"reasoning_in_content": false,
|
||||
"generation_prompt": "",
|
||||
"samplers": [
|
||||
"penalties",
|
||||
"dry",
|
||||
"top_n_sigma",
|
||||
"top_k",
|
||||
"typ_p",
|
||||
"top_p",
|
||||
"min_p",
|
||||
"xtc",
|
||||
"temperature"
|
||||
],
|
||||
"speculative.types": "none",
|
||||
"timings_per_token": false,
|
||||
"post_sampling_probs": false,
|
||||
"backend_sampling": false,
|
||||
"lora": []
|
||||
},
|
||||
"n_ctx": 160000
|
||||
},
|
||||
"total_slots": 2,
|
||||
"model_alias": "qwen-medium",
|
||||
"model_ftype": "IQ4_XS - 4.25 bpw",
|
||||
"model_path": "/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf",
|
||||
"modalities": {
|
||||
"vision": true,
|
||||
"video": true,
|
||||
"audio": false
|
||||
},
|
||||
"media_marker": "<__media_6beqV5u0Vr11b3oODbUSmmU1ZGgPZj1J__>",
|
||||
"endpoint_slots": true,
|
||||
"endpoint_props": false,
|
||||
"endpoint_metrics": true,
|
||||
"ui": false,
|
||||
"ui_settings": {},
|
||||
"chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set sysns = namespace(count=0, text='') %}\n{%- for message in messages %}\n {%- if sysns.count == loop.index0 and (message.role == 'system' or message.role == 'developer') %}\n {%- set sys_content = render_content(message.content, false, true)|trim %}\n {%- if sys_content %}\n {%- set sysns.text = sysns.text + ('\\n' if sysns.text else '') + sys_content %}\n {%- endif %}\n {%- set sysns.count = sysns.count + 1 %}\n {%- endif %}\n{%- endfor %}\n{%- set num_sys = sysns.count %}\n{%- set merged_system = sysns.text %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort == 'high' %}\n {%- set resolved_reasoning_effort = 'xhigh' %}\n {%- endif %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calLine truncated
|
||||
"chat_template_caps": {
|
||||
"supports_object_arguments": true,
|
||||
"supports_parallel_tool_calls": true,
|
||||
"supports_preserve_reasoning": true,
|
||||
"supports_reasoning_effort": true,
|
||||
"supports_string_content": true,
|
||||
"supports_system_role": true,
|
||||
"supports_tool_calls": true,
|
||||
"supports_tools": true,
|
||||
"supports_typed_content": true
|
||||
},
|
||||
"bos_token": "<|endoftext|>",
|
||||
"eos_token": "<|im_end|>",
|
||||
"build_info": "b10964-b29c606e2",
|
||||
"is_sleeping": false,
|
||||
"cors_proxy_enabled": false
|
||||
},
|
||||
"slots": [
|
||||
{
|
||||
"id": 0,
|
||||
"n_ctx": 160000,
|
||||
"speculative": true,
|
||||
"is_processing": false
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"n_ctx": 160000,
|
||||
"speculative": true,
|
||||
"is_processing": false
|
||||
}
|
||||
],
|
||||
"smoke": {
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "OK"
|
||||
}
|
||||
}
|
||||
],
|
||||
"created": 1789936774,
|
||||
"model": "qwen-medium",
|
||||
"system_fingerprint": "b10964-b29c606e2",
|
||||
"object": "chat.completion",
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 19,
|
||||
"total_tokens": 21,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
},
|
||||
"id": "chatcmpl-crkNjSblJZQlUtkCBAwAiGOYZ2MavxPK",
|
||||
"timings": {
|
||||
"cache_n": 0,
|
||||
"prompt_n": 19,
|
||||
"prompt_ms": 782.694,
|
||||
"prompt_per_token_ms": 41.194421052631576,
|
||||
"prompt_per_second": 24.275131788412843,
|
||||
"predicted_n": 2,
|
||||
"predicted_ms": 293.483,
|
||||
"predicted_per_token_ms": 293.483,
|
||||
"predicted_per_second": 3.407352384976302,
|
||||
"draft_n": 3,
|
||||
"draft_n_accepted": 3
|
||||
},
|
||||
"wall_seconds": 1.0789340379997157
|
||||
},
|
||||
"decode": [
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Ein Reverse Proxy ist ein zentrales Element in modernen IT-Infrastrukturen, das als Vermittler zwischen Clienten und Backend-Servern agiert. Im Gegensatz zu einem Forward Proxy, der Anfragen von internen Clients nach außen vertritt, stellt der Reverse Proxy eine Eindepunkt-Schnittstelle (Single Point of Entry) für externe Anfragen dar. Typische Vertreter sind nginx, HAProxy oder Envoy. Der primäre Vorteil dieser Architektur liegt in der Entkopplung des Clients vom eigentlichen Server: Der Client muss die konkrete IP-Adresse oder den Hostnamen des Backend-Servers nicht kennen. Der Proxy übernimmt die Lastverteilung (Load Balancing), Terminiert TLS-Verbindungen, bündelt Traffic und kann zusätzliche Sicherheitsmechanismen wie Rate-Limiting oder Firewall-Regeln implementieren.\n\nDie Funktionsweise basiert im Kern auf der Weiterleitung von Anfragen. Wenn ein HTTP-Request auf dem Reverse Proxy eingeht, wird dieser inspiziert, bevor er an den internen Server weitergeleitet wird. Der Proxy ersetzt dabei oft den eigenen Hostnamen durch den internen Zielsystem-Hostnamen und passt Header-Informationen an. Eine entscheidende Rolle spielt dabei die Kommunikation über TCP/IP. Der Proxy öffnet eine Verbindung zum Backend-Server, sendet die Anfrage und wartet auf die Antwort, die er anschließend an den Client zurückgibt. In dynamischen Umgebungen, insbesondere bei der Nutzung von Containern (z. B. via Docker oder Kubernetes), ist diese Architektur besonders anfällig für eine spezifische Klasse von Fehlern: die IP-Adressen-Änderung.\n\nContainer sind durch ihre Effizienz und Skalierbarkeit zwar ideal für moderne Anwendungen, sie haben aber die Eigenschaft, dass ihre IP-Adressen nicht statisch sind. Wenn ein Container neu gestartet, migriert oder wegen Ressourcenengpässen evakuiert wird, wird ihm häufig eine neue IP-Adresse zugewiesen. Wenn ein Reverse Proxy die Ziel-IP-Adresse hartkodiert oder nicht dynamisch auf Veränderungen reagiert, kommt es zu schwerwiegenden Kommunikationsausfällen. Die häufigsten Symptome hierfür sind „Connection Refused“-Fehler, wenn der Proxy versucht, auf eine alte, nicht mehr existierende IP zuzugreifen, oder Timeouts, wenn Pakete an eine Adresse gerichtet werden, auf der kein Prozess mehr lauscht. Ein weiteres subtiles Problem ist das „Stale Connection“-Phänomen: Lange bestehende TCP-Verbindungen können hängen bleiben, obwohl das Backend bereits neu gestartet wurde.\n\nUm solche Fehler präzise einzugrenzen, ist eine systematische Analyse der Logs essenziell, da man zwischen Client-, Proxy- und Server-Seite unterscheiden muss. Der erste Schritt ist die Prüfung der Access Logs des Reverse Proxys. Hier sucht man nach HTTP-Statuscodes, die auf Backend-Fehler hindeuten, insbesondere 502 (Bad Gateway), 503 (Service Unavailable) oder 504 (Gateway Timeout). Ein 502-Fehler bedeutet oft, dass der Proxy die Anfrage an das Backend geschickt hat, aber keine gültige HTTP-Antwort erhalten hat – typischerweise, weil die TCP-Verbindung zum Backend abgelehnt wurde (Connection Reset oder Refused).\n\nIn den Error-Logs des Proxys (z. B. bei nginx in der Datei `error.log`) finden sich die spezifischen technischen Ursachen. Man sieht Meldungen wie „connect() failed (111: Connection refused) to [IP]:[PORT]“. Diese IP-Adresse ist der entscheidende Hinweis. Vergleicht man diese IP mit der aktuell aktiven IP des Containers (z."
|
||||
}
|
||||
}
|
||||
],
|
||||
"created": 1789936787,
|
||||
"model": "qwen-medium",
|
||||
"system_fingerprint": "b10964-b29c606e2",
|
||||
"object": "chat.completion",
|
||||
"usage": {
|
||||
"completion_tokens": 768,
|
||||
"prompt_tokens": 59,
|
||||
"total_tokens": 827,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
},
|
||||
"id": "chatcmpl-MInUDwt7oMjqbagxGKpU2FNOn00er50H",
|
||||
"timings": {
|
||||
"cache_n": 0,
|
||||
"prompt_n": 59,
|
||||
"prompt_ms": 124.032,
|
||||
"prompt_per_token_ms": 2.1022372881355933,
|
||||
"prompt_per_second": 475.68369453044374,
|
||||
"predicted_n": 768,
|
||||
"predicted_ms": 13329.771,
|
||||
"predicted_per_token_ms": 17.379101694915256,
|
||||
"predicted_per_second": 57.540373349249585,
|
||||
"draft_n": 1024,
|
||||
"draft_n_accepted": 425
|
||||
},
|
||||
"wall_seconds": 13.5130456770421
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "```python\nimport asyncio\nfrom typing import Any, Awaitable, Callable, List, Optional, Union\n\n\nasync def first_success(\n awaitables: List[Awaitable],\n return_exceptions: bool = True\n) -> Any:\n \"\"\"\n Start all awaitables concurrently, return the first successful result,\n cancel and await remaining tasks, and collect exceptions if all fail.\n\n Args:\n awaitables: A list of awaitables (coroutines, tasks, or other awaitables).\n return_exceptions: If True, raise an ExceptionGroup containing all exceptions\n when all awaitables fail. If False, raise the first exception.\n\n Returns:\n The result of the first successfully completed awaitable.\n\n Raises:\n ExceptionGroup: If all awaitables fail and return_exceptions is True.\n Exception: If all awaitables fail and return_exceptions is False (first exception).\n \"\"\"\n if not awaitables:\n raise ValueError(\"At least one awaitable is required\")\n\n # Convert all awaitables to tasks\n tasks = [asyncio.ensure_future(av) if not isinstance(av, asyncio.Task) else av\n for av in awaitables]\n\n # Use asyncio.wait with FIRST_COMPLETED to get results as they complete\n pending = set(tasks)\n results = {} # task -> result or exception\n exceptions = []\n\n while pending:\n done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)\n\n for task in done:\n try:\n result = task.result()\n # Success: cancel all other pending tasks and return\n for t in pending:\n t.cancel()\n # Await all cancelled tasks to ensure they finish cancelling\n if pending:\n await asyncio.wait(pending)\n return result\n except asyncio.CancelledError:\n # This task was cancelled (unlikely in this context since we\n # only cancel pending ones after a success, but handle for safety)\n pass\n except Exception as e:\n # Record the exception for this task\n exceptions.append(e)\n\n # All tasks have failed\n if not return_exceptions:\n if exceptions:\n raise exceptions[0]\n else:\n raise RuntimeError(\"All awaitables failed with no exception recorded\")\n\n # Return an ExceptionGroup with all collected exceptions\n if len(exceptions) == 1:\n raise exceptions[0]\n else:\n # Create an ExceptionGroup-like behavior. In Python 3.11+, we can use ExceptionGroup.\n # For broader compatibility, we'll create a custom one or use the built-in.\n if hasattr(asyncio, 'ExceptionGroup') or hasattr(Builtins, 'ExceptionGroup'):\n try:\n # Python 3.11+ has ExceptionGroup in builtins\n raise ExceptionGroup(\"All awaitables failed\", exceptions)\n except NameError:\n # Fallback: raise the last exception\n raise exceptions[-1]\n else:\n raise exceptions[-1]\n\n\n# For Python 3.10 and below compatibility, provide a simple ExceptionGroup alternative\nclass CustomExceptionGroup(Exception):\n def __init__(self, message: str, exceptions: List[Exception]):\n self.exceptions = exceptions\n super().__init__(message)\n\n def __str__(self):\n msgs = [str(e) for e in self.exceptions]\n return f\"{self.args[0]}:\\n\" + \"\\n\".join(f\" {m}\" for m in msgs"
|
||||
}
|
||||
}
|
||||
],
|
||||
"created": 1789936798,
|
||||
"model": "qwen-medium",
|
||||
"system_fingerprint": "b10964-b29c606e2",
|
||||
"object": "chat.completion",
|
||||
"usage": {
|
||||
"completion_tokens": 768,
|
||||
"prompt_tokens": 54,
|
||||
"total_tokens": 822,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
},
|
||||
"id": "chatcmpl-QpTMAiaFXqAAp22cfxOcjJb7A0v6toBp",
|
||||
"timings": {
|
||||
"cache_n": 0,
|
||||
"prompt_n": 54,
|
||||
"prompt_ms": 122.928,
|
||||
"prompt_per_token_ms": 2.2764444444444445,
|
||||
"prompt_per_second": 439.2815306520891,
|
||||
"predicted_n": 768,
|
||||
"predicted_ms": 10366.471,
|
||||
"predicted_per_token_ms": 13.515607561929595,
|
||||
"predicted_per_second": 73.98853476752117,
|
||||
"draft_n": 789,
|
||||
"draft_n_accepted": 503
|
||||
},
|
||||
"wall_seconds": 10.554032287036534
|
||||
}
|
||||
],
|
||||
"prefill": [
|
||||
{
|
||||
"target": 24576,
|
||||
"response": {
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "```json\n{\n \"alpha\": \"RAVEN-417\",\n \"beta\": \"CEDAR-928\",\n \"gamma\": \"ORBIT-563\"\n}\n```\n\nDie Verifizierung der identifizierten Datensätze erfordert einen strengen, methodischen Ansatz, der auf der genauen Analyse der bereitgestellten Textdaten basiert, ohne auf externe Annahmen oder erfundene Beweise zurückzugreifen. Der erste Schritt besteht darin, die Struktur des vorliegenden Datensatzes zu verstehen. Die meisten Einträge folgen einem einheitlichen Muster: \"Record [ID]: cobalt lantern maple orbit quartz river silver tango.\" Diese Wiederholung dient als Baseline, von der jede Abweichung abweicht.\n\nUm die Nadelwerte (NEEDLE values) zu verifizieren, muss man gezielt nach Stellen im Text suchen, an denen dieses Muster durchbrochen wird. Bei \"NEEDLE_ALPHA=RAVEN-417\" sieht man, dass zwischen Record 000146 und Record 000147 eine unterbrochene Zeile steht: \"Record 000146: cob\" gefolgt von \"NEEDLE_ALPHA=RAVEN-417\" und dann \"alt lantern maple orbit quartz river silver tango.\" Hier ist der Wert klar als separater String definiert, der das Wort \"cobalt\" teilt und ersetzt. Die Verifizierung erfolgt, indem man prüft, ob der String \"RAVEN-417\" exakt an dieser Stelle zwischen den Fragmenten \"cob\" und \"alt\" steht, was die Integrität der Einbettung bestätigt.\n\nÄhnlich verhält es sich mit \"NEEDLE_BETA=CEDAR-928\". Hier wird Record 000584 in zwei Teile geteilt: \"Recor\" und \"d 000585...\". Dazwischen ist der Wert \"CEDAR-928\" positioniert. Die Überprüfung erfordert das Abgleichen des Anfangs \"Recor\" mit dem Fortsetzungsanfang \"d 000585\", um sicherzustellen, dass der Datensatz nur durch die Einspeisung des NEEDLE-Werts unterbrochen wurde. Es ist wichtig, nicht zu versuchen, den fehlenden Teil des Wortes \"Record\" zu rekonstruieren oder zu ändern, sondern"
|
||||
}
|
||||
}
|
||||
],
|
||||
"created": 1789936820,
|
||||
"model": "qwen-medium",
|
||||
"system_fingerprint": "b10964-b29c606e2",
|
||||
"object": "chat.completion",
|
||||
"usage": {
|
||||
"completion_tokens": 512,
|
||||
"prompt_tokens": 24674,
|
||||
"total_tokens": 25186,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
},
|
||||
"id": "chatcmpl-FcH5bKbslbNLAiLQGhilCzDIBppAAnN0",
|
||||
"timings": {
|
||||
"cache_n": 0,
|
||||
"prompt_n": 24674,
|
||||
"prompt_ms": 12783.075,
|
||||
"prompt_per_token_ms": 0.518078746859042,
|
||||
"prompt_per_second": 1930.208498346446,
|
||||
"predicted_n": 512,
|
||||
"predicted_ms": 9104.903,
|
||||
"predicted_per_token_ms": 17.81781409001957,
|
||||
"predicted_per_second": 56.123607247655464,
|
||||
"draft_n": 598,
|
||||
"draft_n_accepted": 311
|
||||
},
|
||||
"wall_seconds": 21.976672343036626,
|
||||
"recall": {
|
||||
"RAVEN-417": true,
|
||||
"CEDAR-928": true,
|
||||
"ORBIT-563": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"finished": 1789936820.3087893
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
[
|
||||
"--model",
|
||||
"/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf",
|
||||
"--mmproj",
|
||||
"/models/qwen/mmproj-BF16.gguf",
|
||||
"--mmproj-offload",
|
||||
"--mmproj-device",
|
||||
"CUDA1",
|
||||
"--alias",
|
||||
"qwen-medium",
|
||||
"--ctx-size",
|
||||
"160000",
|
||||
"--flash-attn",
|
||||
"on",
|
||||
"--cache-type-k",
|
||||
"q4_0",
|
||||
"--cache-type-v",
|
||||
"q4_0",
|
||||
"--cache-prompt",
|
||||
"--cache-ram",
|
||||
"32768",
|
||||
"--threads",
|
||||
"6",
|
||||
"--threads-batch",
|
||||
"6",
|
||||
"--batch-size",
|
||||
"2048",
|
||||
"--ubatch-size",
|
||||
"256",
|
||||
"--parallel",
|
||||
"2",
|
||||
"--kv-unified",
|
||||
"--jinja",
|
||||
"--reasoning",
|
||||
"auto",
|
||||
"--reasoning-preserve",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"5005",
|
||||
"--metrics",
|
||||
"--fit",
|
||||
"off",
|
||||
"--n-gpu-layers",
|
||||
"all",
|
||||
"--load-mode",
|
||||
"none",
|
||||
"--no-ui",
|
||||
"--temperature",
|
||||
"1.0",
|
||||
"--top-p",
|
||||
"0.95",
|
||||
"--top-k",
|
||||
"20",
|
||||
"--device",
|
||||
"CUDA0,CUDA1",
|
||||
"--main-gpu",
|
||||
"0",
|
||||
"--split-mode",
|
||||
"layer",
|
||||
"--tensor-split",
|
||||
"83,17",
|
||||
"--spec-type",
|
||||
"draft-mtp",
|
||||
"--spec-draft-n-max",
|
||||
"3",
|
||||
"--spec-draft-type-k",
|
||||
"f16",
|
||||
"--spec-draft-type-v",
|
||||
"f16",
|
||||
"--spec-draft-p-min",
|
||||
"0.05",
|
||||
"--verbosity",
|
||||
"3"
|
||||
]
|
||||
Binary file not shown.
Reference in new issue
Block a user