simcop2387/text-generation-webui-mirror
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2025-06-07 06:06:20 -04:00
Code Issues Projects Releases Packages Wiki Activity

llama.cpp: Add a retry mechanism when getting the logits (sometimes it fails)

Browse source
This commit is contained in:
oobabooga 2025-05-03 06:51:20 -07:00
parent ea60f14674
commit 4c2e3b168b
1 changed files with 8 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
modules/llama_cpp_server.py
View file

@ -210,14 +210,15 @@ class LlamaServer:
pprint.PrettyPrinter(indent=4, sort_dicts=False).pprint(printable_payload)
print()
response = self.session.post(url, json=payload)
result = response.json()
for retry in range(5):
response = self.session.post(url, json=payload)
result = response.json()
if "completion_probabilities" in result:
if use_samplers:
return result["completion_probabilities"][0]["top_probs"]
else:
return result["completion_probabilities"][0]["top_logprobs"]
if "completion_probabilities" in result:
if use_samplers:
return result["completion_probabilities"][0]["top_probs"]
else:
return result["completion_probabilities"][0]["top_logprobs"]
else:
raise Exception(f"Unexpected response format: 'completion_probabilities' not found in {result}")