Add HQQ quant loader (#4888)

--------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com>
2023-12-18 19:23:16 -05:00 · 2023-12-18 19:23:16 -05:00 · 674be9a09a
commit 674be9a09a
parent 64a57d9dc2
16 changed files with 79 additions and 0 deletions
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@ -163,6 +163,8 @@ def infer_loader(model_name, model_settings):
        loader = 'RWKV'
    elif re.match(r'.*exl2', model_name.lower()):
        loader = 'ExLlamav2_HF'
+    elif re.match(r'.*-hqq', model_name.lower()):
+        return 'HQQ'
    else:
        loader = 'Transformers'