From 7743b5e9de945e69c939dff54c609a3c5b113a62 Mon Sep 17 00:00:00 2001 From: Brian Dashore Date: Sat, 7 Oct 2023 18:38:29 -0400 Subject: [PATCH] Llamacpp_HF: Fix CFG cache init (#4219) Documentation says that model.context_params should be sent when a new context is created. The current code uses model.params which doesn't exist. Signed-off-by: kingbri --- modules/llamacpp_hf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py index c5abdef..37f86e0 100644 --- a/modules/llamacpp_hf.py +++ b/modules/llamacpp_hf.py @@ -48,7 +48,7 @@ class LlamacppHF(PreTrainedModel): 'n_tokens': self.model.n_tokens, 'input_ids': self.model.input_ids.copy(), 'scores': self.model.scores.copy(), - 'ctx': llama_cpp_lib().llama_new_context_with_model(model.model, model.params) + 'ctx': llama_cpp_lib().llama_new_context_with_model(model.model, model.context_params) } def _validate_model_class(self):