yang/CodeGeeX4
1
0
Fork 0
mirror of https://github.com/JasonYANG170/CodeGeeX4.git synced 2024-10-19 05:51:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix errors when using auto-completion

Browse Source
This commit is contained in:
Shaobo 2024-07-15 21:15:33 +08:00 committed by GitHub
parent d624f752c2
commit 38342ab02b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
local_mode/models/codegeex.py
View File

@ -29,16 +29,21 @@ class CodegeexChatModel:
def stream_chat(self, request: ChatCompletionRequest):
try:
inputs = self.tokenizer.apply_chat_template(
conversation=[msg.model_dump() for msg in request.messages],
add_generation_prompt=True,
return_tensors="pt",
return_dict=True
).to(self.model.device)
gen_configs = {
"max_new_tokens": request.max_tokens,
"temperature": request.temperature,
"top_p": request.top_p,
"repetition_penalty": request.presence_penalty
}
length = 0
for i, (response, _) in enumerate(self.model.stream_chat(
self.tokenizer,
query=request.messages[-1].content,
history=[msg.model_dump() for msg in request.messages[:-1]],
max_new_tokens=request.max_tokens,
temperature=request.temperature,
top_p=request.top_p,
repetition_penalty=request.presence_penalty
)):
for i, outputs in enumerate(self.model.stream_generate(**inputs, **gen_configs)):
response = self.tokenizer.decode(outputs.tolist()[0][len(inputs["input_ids"][0]):-1])
resp = ChatCompletionStreamResponse()
resp.choices[0].index = i
resp.choices[0].delta.content = response[length:]