Add safe model diagnostics, explicit self-review stages and recheck action

This commit is contained in:
Mikei386
2026-09-24 21:59:42 +02:00
parent d704b01897
commit 4f45015b1c
7 changed files with 174 additions and 14 deletions
+12
View File
@@ -158,3 +158,15 @@ def test_stale_vectors_do_not_overwrite_new_version(client):
assert store.vectors('test') == {}
store.put_vector(p['id'], updated['version'], 'test', [1, 0])
assert p['id'] in store.vectors('test')
def test_recheck_endpoint_keeps_draft_and_requires_auth(client):
configure(client)
response=client.post('/api/recheck',json={'body':'Original {{name}}','draft':'Draft {{user}}'})
assert response.status_code==200
assert response.json()['body']=='Draft {{user}}'
assert len(response.json()['review']['literal_issues'])==2
assert response.json()['review']['status']=='unchecked'
assert client.get('/api/prompts').json()['items']==[]
client.post('/api/logout')
assert client.post('/api/recheck',json={'body':'original','draft':'draft'}).status_code==401
+43
View File
@@ -69,3 +69,46 @@ def test_failed_final_review_does_not_claim_success():
r, calls = run([BAD, json.dumps({'issues': ISSUES}), ORIGINAL, 500])
assert len(calls) == 4
assert r['body'] == ORIGINAL and r['review']['status'] == 'unchecked'
def test_recheck_is_one_call_and_keeps_text():
calls=[]
def handler(request):
calls.append(json.loads(request.content))
return httpx.Response(200,json={'choices':[{'message':{'content':'{"issues": []}'}}]})
p=Provider({'base_url':'http://model/v1','chat_model':'test'},httpx.MockTransport(handler))
r=asyncio.run(p.recheck(ORIGINAL,'Clarify',BAD))
assert len(calls)==1 and r['body']==BAD
assert r['review']['draft_kind']=='current'
assert r['review']['status']=='passed'
def test_literal_checks():
from atelier.provider import literal_checks
assert literal_checks('Use {{name}}, port 8099 and /app/data.', 'Use /app/data. Listen on port 8099 with {{name}}.')==[]
issues=literal_checks('Use {{name}} on port 8099 at /app/data', 'Use {{user}} on port 8080 at /app/new')
assert len(issues)==6
@pytest.mark.parametrize('kind', ['timeout','connect','json','http'])
def test_diagnostic_logs_redact_sensitive_data(caplog,kind):
def handler(request):
if kind=='timeout':raise httpx.ReadTimeout('SECRET_RESPONSE',request=request)
if kind=='connect':raise httpx.ConnectError('SECRET_RESPONSE',request=request)
if kind=='json':return httpx.Response(200,text='SECRET_RESPONSE')
return httpx.Response(500,text='SECRET_RESPONSE')
p=Provider({'base_url':'http://secret-host/v1','api_key':'SECRET_KEY','chat_model':'test'},httpx.MockTransport(handler))
r=asyncio.run(p.recheck('SECRET_PROMPT','SECRET_INSTRUCTION','SECRET_DRAFT'))
assert r['review']['status']=='unchecked'
assert r['review']['failed_stage']=='recheck'
assert 'error_type=' in caplog.text and 'duration_ms=' in caplog.text
assert not any(v in caplog.text for v in ['SECRET','secret-host'])
assert 'SECRET' not in r['review']['warning']
if kind=='json': assert 'HTTP 200' in r['review']['warning']
if kind=='timeout': assert 'Zeitüberschreitung' in r['review']['warning']
def test_failed_final_review_labels_corrected_draft():
r,_=run([BAD,json.dumps({'issues':ISSUES}),ORIGINAL,503])
assert r['review']['draft_kind']=='corrected'
assert r['review']['failed_stage']=='final_review'