mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

LoRA: Fix error "Attempting to unscale FP16 gradients" when training (#5268)

Browse source
This commit is contained in:
ilya sheprut 2024-01-17 23:11:49 +03:00 • committed by GitHub
parent 535ea9928a
commit 4d14eb8b82
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
modules/training.py
View file

@ -605,6 +605,11 @@ def do_train(lora_name: str, always_override: bool, q_proj_en: bool, v_proj_en:
control.should_training_stop = True control.should_training_stop = True
print(f"\033[1;31;1mStop Loss {stop_at_loss} reached.\033[0;37;0m") print(f"\033[1;31;1mStop Loss {stop_at_loss} reached.\033[0;37;0m")
# Fix training for mixed precision models
for param in shared.model.parameters():
if param.requires_grad:
param.data = param.data.float()
trainer = transformers.Trainer( trainer = transformers.Trainer(
model=lora_model, model=lora_model,
train_dataset=train_data, train_dataset=train_data,