Bump llama-cpp-python to 0.2.24 (#5001)

2023-12-19 15:22:21 -03:00 · 2023-12-19 15:22:21 -03:00 · 0a299d5959
commit 0a299d5959
parent 83cf1a6b67
15 changed files with 104 additions and 96 deletions
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@ -86,6 +86,7 @@ class LlamaCppModel:
            'rope_freq_base': RoPE.get_rope_freq_base(shared.args.alpha_value, shared.args.rope_freq_base),
            'tensor_split': tensor_split_list,
            'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
+            'offload_kqv': not shared.args.no_offload_kqv
        }

        result.model = Llama(**params)