Document Ornith 1.5 A/B benchmark

experiments/ornith15-ab/vision_probe.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Small, fully synthetic vision smoke test for the isolated A/B servers."""
+import argparse
+import base64
+import json
+import pathlib
+import time
+
+from measure import request
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--base", required=True)
+    p.add_argument("--model", required=True)
+    p.add_argument("--output", type=pathlib.Path, required=True)
+    p.add_argument("--go", action="store_true")
+    args = p.parse_args()
+    if not args.go:
+        p.error("Inference requires explicit --go")
+    image = pathlib.Path(__file__).with_name("assets") / "vision-fixture.png"
+    url = "data:image/png;base64," + base64.b64encode(image.read_bytes()).decode("ascii")
+    prompt = "Was ist links und was ist rechts im Bild? Nenne Form und Farbe beider Objekte."
+    payload = {"model": args.model, "stream": False, "temperature": 0,
+               "reasoning_effort": "none", "max_tokens": 512,
+               "messages": [{"role": "user", "content": [
+                   {"type": "text", "text": prompt},
+                   {"type": "image_url", "image_url": {"url": url}}]}]}
+    result = {"model": args.model, "prompt": prompt, "expected":
+              "Links ein roter Kreis, rechts ein blaues Quadrat.",
+              "started": time.time(), "response": request(args.base, None, payload)}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
+    print(result["response"]["content"])
+
+
+if __name__ == "__main__":
+    main()