mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add the --disable_exllama option for AutoGPTQ (#3545 from clefever/disable-exllama)

Browse source
This commit is contained in:
oobabooga 2023-08-14 15:15:55 -03:00 • committed by GitHub
commit ccfc02a28d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/shared.py
View file

@ -139,6 +139,7 @@ parser.add_argument('--no_inject_fused_attention', action='store_true', help='Do
parser.add_argument('--no_inject_fused_mlp', action='store_true', help='Triton mode only: Do not use fused MLP (lowers VRAM requirements).')
parser.add_argument('--no_use_cuda_fp16', action='store_true', help='This can make models faster on some systems.')
parser.add_argument('--desc_act', action='store_true', help='For models that don\'t have a quantize_config.json, this parameter is used to define whether to set desc_act or not in BaseQuantizeConfig.')
parser.add_argument('--disable_exllama', action='store_true', help='Disable ExLlama kernel, which can improve inference speed on some systems.')
# ExLlama
parser.add_argument('--gpu-split', type=str, help="Comma-separated list of VRAM (in GB) to use per GPU device for model layers, e.g. 20,7,7")