Enable NUMA feature for llama_cpp_python (#4040)

2023-09-27 03:05:00 +02:00 · 2023-09-27 03:05:00 +02:00 · 7e6ff8d1f0
commit 7e6ff8d1f0
parent 87ea2d96fd
7 changed files with 21 additions and 12 deletions
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@ -81,6 +81,7 @@ class LlamaCppModel:
            'use_mlock': shared.args.mlock,
            'mul_mat_q': shared.args.mul_mat_q,
            'low_vram': shared.args.low_vram,
+            'numa': shared.args.numa,
            'n_gpu_layers': shared.args.n_gpu_layers,
            'rope_freq_base': RoPE.get_rope_freq_base(shared.args.alpha_value, shared.args.rope_freq_base),
            'tensor_split': tensor_split_list,