community[patch]: allows using `text-generation-inference` /generate route with `HuggingFaceEndpoint` (#20100)

- **Description:** allows to use the /generate route of
`text-generation-inference` with the `HuggingFaceEndpoint`
pull/20134/head^2
Alexander Dicke 2 months ago committed by GitHub
parent ea43c669f2
commit d7e12750df
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -258,7 +258,10 @@ class HuggingFaceEndpoint(LLM):
stream=False,
task=self.task,
)
response_text = json.loads(response.decode())[0]["generated_text"]
try:
response_text = json.loads(response.decode())[0]["generated_text"]
except KeyError:
response_text = json.loads(response.decode())["generated_text"]
# Maybe the generation has stopped at one of the stop sequences:
# then we remove this stop sequence from the end of the generated text
@ -289,7 +292,10 @@ class HuggingFaceEndpoint(LLM):
stream=False,
task=self.task,
)
response_text = json.loads(response.decode())[0]["generated_text"]
try:
response_text = json.loads(response.decode())[0]["generated_text"]
except KeyError:
response_text = json.loads(response.decode())["generated_text"]
# Maybe the generation has stopped at one of the stop sequences:
# then remove this stop sequence from the end of the generated text

Loading…
Cancel
Save