Add the --disable_exllama option for AutoGPTQ

2023-08-12 02:26:58 -04:00 · 2023-08-12 02:26:58 -04:00 · 0230fa4e9c
commit 0230fa4e9c
parent 0e05818266
6 changed files with 6 additions and 0 deletions
--- a/modules/AutoGPTQ_loader.py
+++ b/modules/AutoGPTQ_loader.py
@ -50,6 +50,7 @@ def load_quantized(model_name):
        'max_memory': get_max_memory_dict(),
        'quantize_config': quantize_config,
        'use_cuda_fp16': not shared.args.no_use_cuda_fp16,
+        'disable_exllama': shared.args.disable_exllama,
    }

    logger.info(f"The AutoGPTQ params are: {params}")