Review prompt fidelity and bound correction to one rechecked attempt

tests/test_review.py

@@ -0,0 +1,71 @@
+import asyncio
+import json
+
+import httpx
+import pytest
+
+from atelier.provider import Provider
+
+ORIGINAL = 'Create four games on port 8099. There is no time limit. You have about 5 hours. If needed, use web/image search.'
+BAD = 'Create four games on port 8099. Time limit: 5 hours. You must use web/image search.'
+ISSUES = [{'description': 'Zeitlimit erfunden', 'original_quote': 'There is no time limit.', 'draft_quote': 'Time limit: 5 hours.'}, {'description': 'Optionale Recherche verpflichtend gemacht', 'original_quote': 'If needed, use web/image search.', 'draft_quote': 'You must use web/image search.'}]
+
+
+def run(responses):
+    calls = []
+    def transport(request):
+        calls.append(json.loads(request.content))
+        response = responses[len(calls)-1]
+        if isinstance(response, int):
+            return httpx.Response(response)
+        return httpx.Response(200, json={'choices': [{'message': {'content': response}}]})
+    provider = Provider({'base_url': 'http://model/v1', 'chat_model': 'test'}, httpx.MockTransport(transport))
+    result = asyncio.run(provider.improve_checked(ORIGINAL, 'Formuliere klarer.'))
+    return result, calls
+
+
+def test_clean_two_calls():
+    r, calls = run([ORIGINAL, '{"issues": []}'])
+    assert len(calls) == 2
+    assert r['review']['status'] == 'passed'
+    payload = json.loads(calls[1]['messages'][1]['content'])
+    assert payload['original_template'] == ORIGINAL
+    assert payload['revision_request'] == 'Formuliere klarer.'
+
+
+def test_arcade_correction_and_recheck():
+    r, calls = run([BAD, json.dumps({'issues': ISSUES}), ORIGINAL, '{"issues": []}'])
+    assert len(calls) == 4
+    assert r['body'] == ORIGINAL
+    assert r['review']['status'] == 'corrected'
+    assert len(r['review']['initial_issues']) == 2
+    correction = json.loads(calls[2]['messages'][1]['content'])
+    assert correction['original_template'] == ORIGINAL and correction['issues'] == ISSUES
+
+
+def test_remaining_issues_stop_after_four_calls():
+    r, calls = run([BAD, json.dumps({'issues': ISSUES}), BAD, json.dumps({'issues': ISSUES})])
+    assert len(calls) == 4
+    assert r['review']['status'] == 'issues'
+    assert len(r['review']['issues']) == 2
+
+
+@pytest.mark.parametrize('failure', [503, 'not json', '{"issues": "none"}', '{"issues": [{"description":"Wrong", "original_quote":"invented", "draft_quote":""}]}'])
+def test_failed_review_preserves_draft(failure):
+    r, calls = run([BAD, failure])
+    assert len(calls) == 2
+    assert r['body'] == BAD
+    assert r['review']['status'] == 'unchecked'
+    assert r['review']['warning']
+
+
+def test_failed_correction_preserves_initial_draft():
+    r, calls = run([BAD, json.dumps({'issues': ISSUES}), 500])
+    assert len(calls) == 3
+    assert r['body'] == BAD and r['review']['status'] == 'unchecked'
+
+
+def test_failed_final_review_does_not_claim_success():
+    r, calls = run([BAD, json.dumps({'issues': ISSUES}), ORIGINAL, 500])
+    assert len(calls) == 4
+    assert r['body'] == ORIGINAL and r['review']['status'] == 'unchecked'