mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Better recognize the model sizes

Browse source
This commit is contained in:
oobabooga 2023-01-07 12:21:04 -03:00
parent 62c4d9880b
commit 342e756878
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
server.py
View file

@ -30,7 +30,7 @@ def load_model(model_name):
print("Loading in .pt format...") print("Loading in .pt format...")
model = torch.load(f"torch-dumps/{model_name}.pt").cuda() model = torch.load(f"torch-dumps/{model_name}.pt").cuda()
elif model_name.lower().startswith(('gpt-neo', 'opt-', 'galactica')): elif model_name.lower().startswith(('gpt-neo', 'opt-', 'galactica')):
if any(size in model_name for size in ('13b', '20b', '30b')): if any(size in model_name.lower() for size in ('13b', '20b', '30b')):
model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", device_map='auto', load_in_8bit=True) model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", device_map='auto', load_in_8bit=True)
else: else:
model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", low_cpu_mem_usage=True, torch_dtype=torch.float16).cuda() model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", low_cpu_mem_usage=True, torch_dtype=torch.float16).cuda()