Add alpha_value/compress_pos_emb to ExLlama-v2

2023-09-12 15:02:47 -07:00 · 2023-09-12 15:02:47 -07:00 · 18e6b275f3
commit 18e6b275f3
parent 90fca6a77d
3 changed files with 10 additions and 0 deletions
--- a/modules/exllamav2.py
+++ b/modules/exllamav2.py
@ -28,6 +28,9 @@ class Exllamav2Model:
        config.prepare()

        config.max_seq_len = shared.args.max_seq_len
+        config.rope_scale = shared.args.compress_pos_emb
+        config.rope_alpha = shared.args.alpha_value
+
        model = ExLlamaV2(config)

        split = None