mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add threads_batch parameter

Browse source
This commit is contained in:
oobabooga 2023-10-01 21:27:04 -07:00
parent 41a2de96e5
commit b6fe6acf88
7 changed files with 8 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/shared.py
View file

@ -115,6 +115,7 @@ parser.add_argument('--use_double_quant', action='store_true', help='use_double_
# llama.cpp
parser.add_argument('--threads', type=int, default=0, help='Number of threads to use.')
parser.add_argument('--threads-batch', type=int, default=0, help='Number of threads to use for batches/prompt processing.')
parser.add_argument('--n_batch', type=int, default=512, help='Maximum number of prompt tokens to batch together when calling llama_eval.')
parser.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.')
parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.')