From d7e14e1f78865ba6d702fa5b567ae48beeef9680 Mon Sep 17 00:00:00 2001 From: Fernando Tarin Morales Date: Fri, 7 Jul 2023 14:24:07 +0900 Subject: [PATCH] Fixed the param name when loading a LoRA using a model loaded in 4 or 8 bits (#3036) --- modules/LoRA.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/LoRA.py b/modules/LoRA.py index e54d533..0eb56b5 100644 --- a/modules/LoRA.py +++ b/modules/LoRA.py @@ -114,11 +114,12 @@ def add_lora_transformers(lora_names): if len(lora_names) > 0: params = {} if not shared.args.cpu: - params['dtype'] = shared.model.dtype - if hasattr(shared.model, "hf_device_map"): - params['device_map'] = {"base_model.model." + k: v for k, v in shared.model.hf_device_map.items()} - elif shared.args.load_in_8bit: - params['device_map'] = {'': 0} + if shared.args.load_in_4bit or shared.args.load_in_8bit: + params['peft_type'] = shared.model.dtype + else: + params['dtype'] = shared.model.dtype + if hasattr(shared.model, "hf_device_map"): + params['device_map'] = {"base_model.model." + k: v for k, v in shared.model.hf_device_map.items()} logger.info("Applying the following LoRAs to {}: {}".format(shared.model_name, ', '.join(lora_names))) shared.model = PeftModel.from_pretrained(shared.model, Path(f"{shared.args.lora_dir}/{lora_names[0]}"), adapter_name=lora_names[0], **params)