diff --git a/.gitignore b/.gitignore index 1b7f0fb..d98b81d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ torch-dumps/* *pycache* */*pycache* */*/pycache* +venv/ +.venv/ settings.json img_bot* diff --git a/modules/models.py b/modules/models.py index 63060d4..f07e738 100644 --- a/modules/models.py +++ b/modules/models.py @@ -47,7 +47,12 @@ def load_model(model_name): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: - model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), low_cpu_mem_usage=True, torch_dtype=torch.bfloat16 if shared.args.bf16 else torch.float16).cuda() + model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), low_cpu_mem_usage=True, torch_dtype=torch.bfloat16 if shared.args.bf16 else torch.float16) + if torch.has_mps: + device = torch.device('mps') + model = model.to(device) + else: + model = model.cuda() # FlexGen elif shared.args.flexgen: @@ -97,7 +102,7 @@ def load_model(model_name): # Custom else: params = {"low_cpu_mem_usage": True} - if not shared.args.cpu and not torch.cuda.is_available(): + if not any((shared.args.cpu, torch.cuda.is_available(), torch.has_mps)): print("Warning: torch.cuda.is_available() returned False.\nThis means that no GPU has been detected.\nFalling back to CPU mode.\n") shared.args.cpu = True diff --git a/modules/text_generation.py b/modules/text_generation.py index e5b4ad9..1d11de1 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -33,6 +33,9 @@ def encode(prompt, tokens_to_generate=0, add_special_tokens=True): return input_ids.numpy() elif shared.args.deepspeed: return input_ids.to(device=local_rank) + elif torch.has_mps: + device = torch.device('mps') + return input_ids.to(device) else: return input_ids.cuda()