Add threads_batch parameter

2023-10-01 21:27:04 -07:00 · 2023-10-01 21:27:04 -07:00 · b6fe6acf88
commit b6fe6acf88
parent 41a2de96e5
7 changed files with 8 additions and 0 deletions
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@ -76,6 +76,7 @@ class LlamaCppModel:
            'n_ctx': shared.args.n_ctx,
            'seed': int(shared.args.llama_cpp_seed),
            'n_threads': shared.args.threads or None,
+            'n_threads_batch': shared.args.threads_batch or None,
            'n_batch': shared.args.n_batch,
            'use_mmap': not shared.args.no_mmap,
            'use_mlock': shared.args.mlock,