Bump llama-cpp-python to 0.2.24 (#5001)

2023-12-19 15:22:21 -03:00 · 2023-12-19 15:22:21 -03:00 · 0a299d5959
commit 0a299d5959
parent 83cf1a6b67
15 changed files with 104 additions and 96 deletions
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@ -204,6 +204,7 @@ class LlamacppHF(PreTrainedModel):
            'tensor_split': tensor_split_list,
            'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
            'logits_all': shared.args.logits_all,
+            'offload_kqv': not shared.args.no_offload_kqv
        }

        Llama = llama_cpp_lib().Llama