mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add cache_8bit option

Browse source
This commit is contained in:
oobabooga 2023-11-02 11:23:04 -07:00
parent 42f816312d
commit c0655475ae
7 changed files with 32 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
modules/loaders.py
View file

@ -41,6 +41,8 @@ loaders_and_params = OrderedDict({
'gpu_split',
'max_seq_len',
'cfg_cache',
'no_flash_attn',
'cache_8bit',
'alpha_value',
'compress_pos_emb',
'use_fast',
@ -56,6 +58,8 @@ loaders_and_params = OrderedDict({
'ExLlamav2': [
'gpu_split',
'max_seq_len',
'no_flash_attn',
'cache_8bit',
'alpha_value',
'compress_pos_emb',
],