From 2ae61baec7a893d5f4d0049311c5d9552b1d5aa4 Mon Sep 17 00:00:00 2001
From: Mikei386 <44135113+Mikei386@users.noreply.github.com>
Date: Mon, 7 Sep 2026 16:22:51 +0200
Subject: [PATCH] Prepare isolated Qwen3.8 IQ4_XS A/B tests

---
 experiments/qwen38-20260907-ab/README.md      |  82 ++++++++++++++
 experiments/qwen38-20260907-ab/candidates.sh  |  32 ++++++
 experiments/qwen38-20260907-ab/cleanup.sh     |  42 +++++++
 .../qwen38-20260907-ab/download-candidate.sh  |  32 ++++++
 experiments/qwen38-20260907-ab/inventory.sh   |  32 ++++++
 .../qwen38-20260907-ab/run-benchmark.sh       |  24 ++++
 experiments/qwen38-20260907-ab/run-case.sh    | 103 ++++++++++++++++++
 experiments/qwen38-20260907-ab/stop-case.sh   |   5 +
 experiments/qwen38-20260907-ab/wait-ready.sh  |  19 ++++
 9 files changed, 371 insertions(+)
 create mode 100644 experiments/qwen38-20260907-ab/README.md
 create mode 100755 experiments/qwen38-20260907-ab/candidates.sh
 create mode 100755 experiments/qwen38-20260907-ab/cleanup.sh
 create mode 100755 experiments/qwen38-20260907-ab/download-candidate.sh
 create mode 100755 experiments/qwen38-20260907-ab/inventory.sh
 create mode 100755 experiments/qwen38-20260907-ab/run-benchmark.sh
 create mode 100755 experiments/qwen38-20260907-ab/run-case.sh
 create mode 100755 experiments/qwen38-20260907-ab/stop-case.sh
 create mode 100755 experiments/qwen38-20260907-ab/wait-ready.sh

diff --git a/experiments/qwen38-20260907-ab/README.md b/experiments/qwen38-20260907-ab/README.md
new file mode 100644
index 0000000..92a285a
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/README.md
@@ -0,0 +1,82 @@
+# Qwen3.8 September 2026 A/B preparation
+
+This directory prepares an isolated comparison of two Qwen3.8-27B IQ4_XS
+artifacts without adding router profiles or changing the running Athena stack.
+
+## Candidates
+
+| ID | Artifact | Purpose | Pinned revision | Size |
+| --- | --- | --- | --- | ---: |
+| `qwopus` | `Jackrong/Qwopus3.8-27B-Flash-GGUF` / `Qwopus3.8-27B-Flash-MTP-IQ4_XS.gguf` | Efficiency fine-tune | `e146d61e88782677805b3b68ad3adf8674dde80d` | 15,420,445,792 B |
+| `bartowski` | `bartowski/Qwen3.8-27B-GGUF` / `Qwen3.8-27B-IQ4_XS.gguf` | Standard Qwen, alternative IQ4_XS quant | `f0eec4a4bb4975114a030d048952d83c0a53c034` | 15,567,824,480 B |
+
+The production reference remains
+`jpetrina/Qwen3.8-27B-IQ4_XS-pure-GGUF`. The candidates intentionally use the
+same IQ4_XS quantization class so that the first comparison does not mix a
+fine-tune difference with a quantization-class difference.
+
+## Safety boundary
+
+- Nothing in this directory is called by installation, Compose or the router.
+- No production profile is added.
+- Downloads happen only after explicitly running `download-candidate.sh`.
+- `run-case.sh` refuses to start while a production `mike-ai-llama-*` model is
+  running. It never stops production itself.
+- The test server binds to `127.0.0.1:5005` and is not exposed to the LAN.
+- Cleanup is a dry run unless an explicit deletion flag is supplied. It only
+  addresses the exact test container, result directory and two pinned files.
+
+## Later test sequence
+
+Run these commands on Athena only after the active coding task has finished and
+the GPUs have deliberately been released:
+
+```sh
+cd /opt/mike-ai/stack/experiments/qwen38-20260907-ab
+./inventory.sh
+./download-candidate.sh qwopus
+./download-candidate.sh bartowski
+
+./run-case.sh qwopus 160000 85,15
+./wait-ready.sh
+./run-benchmark.sh qwopus-160k
+./stop-case.sh
+
+./run-case.sh bartowski 160000 85,15
+./wait-ready.sh
+./run-benchmark.sh bartowski-160k
+./stop-case.sh
+```
+
+Only after both candidates pass the 160K quality and tool-call tests should
+192K and 262144 be attempted. Promotion into the router is a separate decision
+and is deliberately not implemented here.
+
+## Cleanup
+
+Preview everything owned by this experiment:
+
+```sh
+./cleanup.sh
+```
+
+Remove only the test container and benchmark results:
+
+```sh
+./cleanup.sh --results
+```
+
+Remove only the two downloaded candidate files and their now-empty directory:
+
+```sh
+./cleanup.sh --models
+```
+
+Remove both:
+
+```sh
+./cleanup.sh --all
+```
+
+The script never touches the production Pure, Mix, Beta 1 or uncensored model
+directories.
diff --git a/experiments/qwen38-20260907-ab/candidates.sh b/experiments/qwen38-20260907-ab/candidates.sh
new file mode 100755
index 0000000..22806b2
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/candidates.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+candidate_config() {
+  case "${1:-}" in
+    qwopus)
+      CANDIDATE_ID=qwopus
+      REPOSITORY=Jackrong/Qwopus3.8-27B-Flash-GGUF
+      REVISION=e146d61e88782677805b3b68ad3adf8674dde80d
+      MODEL_FILE=Qwopus3.8-27B-Flash-MTP-IQ4_XS.gguf
+      MODEL_SHA256=88848920fd069ecfe509afd60d4b1f2192327f1edde5e10491a48f7f80c16fb8
+      MODEL_SIZE=15420445792
+      MODEL_ALIAS=qwen-qwopus-flash-ab
+      ;;
+    bartowski)
+      CANDIDATE_ID=bartowski
+      REPOSITORY=bartowski/Qwen3.8-27B-GGUF
+      REVISION=f0eec4a4bb4975114a030d048952d83c0a53c034
+      MODEL_FILE=Qwen3.8-27B-IQ4_XS.gguf
+      MODEL_SHA256=c2ae2b018f967370087c196c86d6811b2340ec19138a3752252ade5fbd1f4786
+      MODEL_SIZE=15567824480
+      MODEL_ALIAS=qwen-bartowski-iq4-xs-ab
+      ;;
+    *)
+      echo "Candidate must be qwopus or bartowski" >&2
+      return 2
+      ;;
+  esac
+
+  MODEL_ROOT=${MODEL_ROOT:-/data/models/experiments/qwen38-ab-20260907}
+  MODEL_PATH="$MODEL_ROOT/$CANDIDATE_ID/$MODEL_FILE"
+  MODEL_URL="https://huggingface.co/$REPOSITORY/resolve/$REVISION/$MODEL_FILE"
+}
diff --git a/experiments/qwen38-20260907-ab/cleanup.sh b/experiments/qwen38-20260907-ab/cleanup.sh
new file mode 100755
index 0000000..6c9327e
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/cleanup.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+MODE=${1:-preview}
+MODEL_ROOT=${MODEL_ROOT:-/data/models/experiments/qwen38-ab-20260907}
+RESULT_DIR=${RESULT_DIR:-/data/benchmarks/qwen38-ab-20260907}
+NAME=mike-ai-llama-qwen38-ab
+
+QWOPUS="$MODEL_ROOT/qwopus/Qwopus3.8-27B-Flash-MTP-IQ4_XS.gguf"
+BARTOWSKI="$MODEL_ROOT/bartowski/Qwen3.8-27B-IQ4_XS.gguf"
+
+case "$MODE" in
+  preview)
+    echo "Dry run only. Exact owned targets:"
+    printf '  container: %s\n  results:   %s\n  model:     %s\n  model:     %s\n' \
+      "$NAME" "$RESULT_DIR" "$QWOPUS" "$BARTOWSKI"
+    echo "Use --results, --models or --all to delete these exact targets."
+    exit 0
+    ;;
+  --results|--models|--all) ;;
+  *)
+    echo "Usage: $0 [--results|--models|--all]" >&2
+    exit 2
+    ;;
+esac
+
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+
+if [[ $MODE == --results || $MODE == --all ]]; then
+  if [[ -d $RESULT_DIR ]]; then
+    find "$RESULT_DIR" -maxdepth 1 -type f -name '*.json' -delete
+    rmdir "$RESULT_DIR" 2>/dev/null || true
+  fi
+fi
+
+if [[ $MODE == --models || $MODE == --all ]]; then
+  rm -f -- "$QWOPUS" "$QWOPUS.part" "$BARTOWSKI" "$BARTOWSKI.part"
+  rmdir "$MODEL_ROOT/qwopus" "$MODEL_ROOT/bartowski" 2>/dev/null || true
+  rmdir "$MODEL_ROOT" 2>/dev/null || true
+fi
+
+echo "Cleanup complete for mode $MODE. No production model path was addressed."
diff --git a/experiments/qwen38-20260907-ab/download-candidate.sh b/experiments/qwen38-20260907-ab/download-candidate.sh
new file mode 100755
index 0000000..46321c3
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/download-candidate.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=candidates.sh
+source "$SCRIPT_DIR/candidates.sh"
+candidate_config "${1:-}"
+
+target_dir="$MODEL_ROOT/$CANDIDATE_ID"
+target="$target_dir/$MODEL_FILE"
+partial="$target.part"
+
+install -d -m 0755 "$target_dir"
+
+if [[ -s $target ]]; then
+  printf '%s  %s\n' "$MODEL_SHA256" "$target" | sha256sum -c -
+  echo "Already prepared: $target"
+  exit 0
+fi
+
+echo "Downloading pinned $CANDIDATE_ID artifact ($MODEL_SIZE bytes)"
+curl --fail --location --retry 8 --retry-delay 5 --continue-at - \
+  --output "$partial" "$MODEL_URL"
+
+actual_size=$(stat -c %s "$partial")
+if [[ $actual_size != "$MODEL_SIZE" ]]; then
+  echo "Size mismatch: expected $MODEL_SIZE, got $actual_size; keeping $partial for inspection" >&2
+  exit 1
+fi
+printf '%s  %s\n' "$MODEL_SHA256" "$partial" | sha256sum -c -
+mv -f "$partial" "$target"
+echo "Prepared and verified: $target"
diff --git a/experiments/qwen38-20260907-ab/inventory.sh b/experiments/qwen38-20260907-ab/inventory.sh
new file mode 100755
index 0000000..863ecda
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/inventory.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+MODEL_ROOT=${MODEL_ROOT:-/data/models/experiments/qwen38-ab-20260907}
+RESULT_DIR=${RESULT_DIR:-/data/benchmarks/qwen38-ab-20260907}
+
+echo "== Containers owned by this experiment =="
+docker ps -a --filter label=mike-ai.experiment=qwen38-ab-20260907 \
+  --format 'table {{.Names}}\t{{.Status}}\t{{.Label "mike-ai.candidate"}}' || true
+
+echo "== Candidate model files and partial downloads =="
+if [[ -d $MODEL_ROOT ]]; then
+  find "$MODEL_ROOT" -maxdepth 2 -type f -exec ls -lh {} +
+else
+  echo "Not present: $MODEL_ROOT"
+fi
+
+echo "== Benchmark results =="
+if [[ -d $RESULT_DIR ]]; then
+  find "$RESULT_DIR" -maxdepth 1 -type f -exec ls -lh {} +
+else
+  echo "Not present: $RESULT_DIR"
+fi
+
+echo "== Known older experimental model directories (read-only report) =="
+for path in \
+  /data/models/qwen3.8-27b-dirk \
+  /data/models/qwen3.8-27b-gsq-rco-test \
+  /data/models/qwen3.8-27b-iq4-mix \
+  /data/models/qwen3.8-27b-iq4-xs-pure; do
+  [[ -d $path ]] && du -sh "$path"
+done
diff --git a/experiments/qwen38-20260907-ab/run-benchmark.sh b/experiments/qwen38-20260907-ab/run-benchmark.sh
new file mode 100755
index 0000000..eaf3127
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/run-benchmark.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+LABEL=${1:-}
+CONTEXT=${2:-160000}
+[[ -n $LABEL && $LABEL =~ ^[a-zA-Z0-9._-]+$ ]] || {
+  echo "Usage: $0 SAFE_LABEL [CONTEXT]" >&2
+  exit 2
+}
+[[ $CONTEXT =~ ^[0-9]+$ ]] || {
+  echo "CONTEXT must be an integer" >&2
+  exit 2
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REFERENCE_DIR="$SCRIPT_DIR/../dirk-qwen38"
+RESULT_DIR=${RESULT_DIR:-/data/benchmarks/qwen38-ab-20260907}
+TASKS=${TASKS:-/opt/mike-ai/stack/dev/QWEN38-FINAL-ACCEPTANCE-v1.json}
+
+curl -fsS --max-time 3 http://127.0.0.1:5005/health >/dev/null
+python3 "$REFERENCE_DIR/bench-case.py" "$LABEL" "$CONTEXT" \
+  --base http://127.0.0.1:5005 --output "$RESULT_DIR"
+python3 "$REFERENCE_DIR/quality-ab.py" "$LABEL" \
+  --base http://127.0.0.1:5005 --tasks "$TASKS" --output "$RESULT_DIR"
diff --git a/experiments/qwen38-20260907-ab/run-case.sh b/experiments/qwen38-20260907-ab/run-case.sh
new file mode 100755
index 0000000..1b624b6
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/run-case.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+CANDIDATE=${1:-}
+CONTEXT=${2:-}
+SPLIT=${3:-}
+
+if [[ -z $CANDIDATE || -z $CONTEXT || -z $SPLIT || ! $CONTEXT =~ ^[0-9]+$ || ! $SPLIT =~ ^[0-9]+,[0-9]+$ ]]; then
+  echo "Usage: $0 {qwopus|bartowski} CONTEXT TENSOR_SPLIT" >&2
+  exit 2
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=candidates.sh
+source "$SCRIPT_DIR/candidates.sh"
+candidate_config "$CANDIDATE"
+
+HOST_MODEL_FILE=$MODEL_PATH
+CONTAINER_MODEL="/models/$CANDIDATE_ID/$MODEL_FILE"
+IMAGE=${LLAMA_IMAGE:-mike-ai/llama.cpp:local}
+NAME=mike-ai-llama-qwen38-ab
+RESULT_DIR=${RESULT_DIR:-/data/benchmarks/qwen38-ab-20260907}
+
+[[ -s $HOST_MODEL_FILE ]] || {
+  echo "Candidate is not downloaded: $HOST_MODEL_FILE" >&2
+  exit 1
+}
+printf '%s  %s\n' "$MODEL_SHA256" "$HOST_MODEL_FILE" | sha256sum -c -
+
+mapfile -t blockers < <(
+  docker ps --format '{{.Names}}' |
+    grep -E '^mike-ai-llama-' |
+    grep -vE '^mike-ai-llama-dashboard$|^mike-ai-llama-qwen38-ab$' || true
+)
+if ((${#blockers[@]})); then
+  printf 'Refusing to start: production model container(s) still running: %s\n' "${blockers[*]}" >&2
+  exit 1
+fi
+
+if docker ps --format '{{.Names}}' | grep -qx "$NAME"; then
+  echo "Refusing to replace a running A/B container; run ./stop-case.sh first" >&2
+  exit 1
+fi
+
+install -d -m 0755 "$RESULT_DIR"
+
+args=(
+  --model "$CONTAINER_MODEL"
+  --alias "$MODEL_ALIAS"
+  --ctx-size "$CONTEXT"
+  --flash-attn on
+  --cache-type-k q4_0
+  --cache-type-v q4_0
+  --cache-prompt
+  --cache-ram 8192
+  --threads 6
+  --threads-batch 6
+  --batch-size 64
+  --ubatch-size 32
+  --parallel 1
+  --jinja
+  --reasoning auto
+  --reasoning-budget 8192
+  --reasoning-preserve
+  --host 127.0.0.1
+  --port 5005
+  --metrics
+  --fit off
+  --n-gpu-layers all
+  --no-mmap
+  --temperature 1.0
+  --top-p 0.95
+  --top-k 20
+  --device CUDA0,CUDA1
+  --main-gpu 0
+  --split-mode layer
+  --tensor-split "$SPLIT"
+  --spec-type draft-mtp
+  --spec-draft-n-max 3
+  --spec-draft-type-k f16
+  --spec-draft-type-v f16
+)
+
+label="$CANDIDATE-ctx${CONTEXT}-split${SPLIT//,/-}"
+docker run -d \
+  --name "$NAME" \
+  --gpus all \
+  --network host \
+  --read-only \
+  --tmpfs /tmp:rw,noexec,nosuid,nodev,size=256m \
+  --security-opt no-new-privileges:true \
+  --cap-drop ALL \
+  --pids-limit 1024 \
+  --log-opt max-size=20m \
+  --log-opt max-file=2 \
+  -v "$MODEL_ROOT":/models:ro \
+  -v "$RESULT_DIR":/results \
+  --label mike-ai.experiment=qwen38-ab-20260907 \
+  --label mike-ai.candidate="$CANDIDATE" \
+  --label mike-ai.case="$label" \
+  "$IMAGE" "${args[@]}"
+
+printf 'Started isolated case %s on http://127.0.0.1:5005\n' "$label"
diff --git a/experiments/qwen38-20260907-ab/stop-case.sh b/experiments/qwen38-20260907-ab/stop-case.sh
new file mode 100755
index 0000000..d839dd4
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/stop-case.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+docker rm -f mike-ai-llama-qwen38-ab >/dev/null 2>&1 || true
+echo "A/B test container removed. Production was not changed."
diff --git a/experiments/qwen38-20260907-ab/wait-ready.sh b/experiments/qwen38-20260907-ab/wait-ready.sh
new file mode 100755
index 0000000..adeb510
--- /dev/null
+++ b/experiments/qwen38-20260907-ab/wait-ready.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+NAME=mike-ai-llama-qwen38-ab
+deadline=$((SECONDS + ${READY_TIMEOUT:-900}))
+until curl -fsS --max-time 3 http://127.0.0.1:5005/health >/dev/null; do
+  state=$(docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null || true)
+  if [[ $state != true ]]; then
+    docker logs --tail 100 "$NAME" >&2 || true
+    exit 1
+  fi
+  if ((SECONDS >= deadline)); then
+    docker logs --tail 100 "$NAME" >&2 || true
+    exit 1
+  fi
+  sleep 2
+done
+curl -fsS http://127.0.0.1:5005/props
+printf '\nA/B test server is ready.\n'