mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add --no_use_cuda_fp16 param for AutoGPTQ

Browse source
This commit is contained in:
oobabooga 2023-06-23 12:22:56 -03:00
parent 5646690769
commit 3ae9af01aa
5 changed files with 6 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
modules/AutoGPTQ_loader.py
View file

@ -48,7 +48,8 @@ def load_quantized(model_name):
'use_safetensors': use_safetensors,
'trust_remote_code': shared.args.trust_remote_code,
'max_memory': get_max_memory_dict(),
'quantize_config': quantize_config
'quantize_config': quantize_config,
'use_cuda_fp16': not shared.args.no_use_cuda_fp16,
}
logger.info(f"The AutoGPTQ params are: {params}")