fix(projekt-matching): use response_format json_schema (vLLM 0.22 ignores guided_json)
This commit is contained in:
@@ -123,24 +123,23 @@ z. B. vLLM, TGI, Triton" ab).
|
||||
- reason: EIN kurzer deutscher Satz mit der Begründung."""
|
||||
|
||||
|
||||
def _post(base, model, messages, body_extra):
|
||||
body = {"model": model, "messages": messages, "temperature": 0.1}
|
||||
body.update(body_extra)
|
||||
def _post(base, model, messages, schema):
|
||||
body = {"model": model, "messages": messages, "temperature": 0.1,
|
||||
"response_format": {"type": "json_schema", "json_schema": {
|
||||
"name": "out", "schema": schema}}}
|
||||
return requests.post(f"{base.rstrip('/')}/chat/completions", json=body,
|
||||
timeout=1500)
|
||||
|
||||
|
||||
def chat_json(base, model, messages, schema):
|
||||
resp = _post(base, model, messages, {"guided_json": schema})
|
||||
if resp.status_code == 400 and "guided_json" in getattr(resp, "text", ""):
|
||||
resp = _post(base, model, messages, {"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "out", "schema": schema}}})
|
||||
resp = _post(base, model, messages, schema)
|
||||
if resp.status_code != 200:
|
||||
raise LlmError(f"vLLM HTTP {resp.status_code}: "
|
||||
f"{getattr(resp, 'text', '')[:300]}")
|
||||
content = resp.json()["choices"][0]["message"]["content"] or ""
|
||||
content = THINK_RE.sub("", content).strip()
|
||||
if content.startswith("```"):
|
||||
content = re.sub(r"^```[a-z]*\s*|\s*```$", "", content).strip()
|
||||
try:
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError as exc:
|
||||
|
||||
Reference in New Issue
Block a user