mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix issue with generating on multiple GPUs

Browse source
This commit is contained in:
81300 2023-02-01 19:02:07 +02:00
parent a97afa6965
commit 0a0d289537
No known key found for this signature in database
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
server.py
View file

@ -314,7 +314,7 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
t0 = time.time()
if args.deepspeed:
with torch.no_grad():
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
output = eval(f"model.generate(input_ids, synced_gpus=True, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
else:
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}")
reply = decode(output[0])
@ -331,7 +331,7 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
for i in tqdm(range(tokens//8+1)):
if args.deepspeed:
with torch.no_grad():
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
output = eval(f"model.generate(input_ids, synced_gpus=True, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
else:
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}")
reply = decode(output[0])