From b284f2407d991c88d22bae48b0779f7eafb584c4 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:03:56 -0700 Subject: [PATCH] Make ExLlama_HF the new default for GPTQ --- modules/models_settings.py | 2 +- server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index 3f37e48..c6803d1 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -23,7 +23,7 @@ def infer_loader(model_name): if not path_to_model.exists(): loader = None elif Path(f'{shared.args.model_dir}/{model_name}/quantize_config.json').exists() or ('wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0): - loader = 'AutoGPTQ' + loader = 'ExLlama_HF' elif len(list(path_to_model.glob('*ggml*.bin'))) > 0: loader = 'llama.cpp' elif re.match('.*ggml.*\.bin', model_name.lower()): diff --git a/server.py b/server.py index 02a073d..4612cb0 100644 --- a/server.py +++ b/server.py @@ -204,7 +204,7 @@ def create_model_menus(): with gr.Row(): with gr.Column(): - shared.gradio['loader'] = gr.Dropdown(label="Model loader", choices=["Transformers", "AutoGPTQ", "GPTQ-for-LLaMa", "ExLlama", "ExLlama_HF", "llama.cpp"], value=None) + shared.gradio['loader'] = gr.Dropdown(label="Model loader", choices=["Transformers", "ExLlama_HF", "AutoGPTQ", "llama.cpp", "ExLlama", "GPTQ-for-LLaMa"], value=None) with gr.Box(): with gr.Row(): with gr.Column():