mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make llava/minigpt-4 work with AutoGPTQ

Browse source
This commit is contained in:
oobabooga 2023-06-11 17:52:23 -03:00
parent f4defde752
commit e471919e6d
1 changed files with 11 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

11
modules/AutoGPTQ_loader.py
View file

@ -51,4 +51,15 @@ def load_quantized(model_name):
logger.info(f"The AutoGPTQ params are: {params}") logger.info(f"The AutoGPTQ params are: {params}")
model = AutoGPTQForCausalLM.from_quantized(path_to_model, **params) model = AutoGPTQForCausalLM.from_quantized(path_to_model, **params)
# These lines fix the multimodal extension when used with AutoGPTQ
if not hasattr(model, 'dtype'):
model.dtype = model.model.dtype
if not hasattr(model, 'embed_tokens'):
model.embed_tokens = model.model.model.embed_tokens
if not hasattr(model.model, 'embed_tokens'):
model.model.embed_tokens = model.model.model.embed_tokens
return model return model