Add Support for Static NTK RoPE scaling for exllama/exllama_hf (#2955)

2023-07-04 00:13:16 -04:00 · 2023-07-04 00:13:16 -04:00 · 10c8c197bf
commit 10c8c197bf
parent 1610d5ffb2
7 changed files with 18 additions and 2 deletions
--- a/modules/exllama_hf.py
+++ b/modules/exllama_hf.py
@ -97,6 +97,11 @@ class ExllamaHF(PreTrainedModel):
        if shared.args.gpu_split:
            config.set_auto_map(shared.args.gpu_split)
            config.gpu_peer_fix = True
+
+        if shared.args.alpha_value:
+            config.alpha_value = shared.args.alpha_value
+            config.calculate_rotary_embedding_base()
+
        if torch.version.hip:
            config.rmsnorm_no_half2 = True
            config.rope_no_half2 = True