mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Read more GGUF metadata (scale_linear and freq_base) (#3877)

Browse source
This commit is contained in:
Gennadij 2023-09-12 23:02:42 +03:00 • committed by GitHub
parent 90fca6a77d
commit 460c40d8ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
modules/models_settings.py
View file

@ -17,6 +17,7 @@ def get_fallback_settings():
'truncation_length': shared.settings['truncation_length'],
'n_ctx': 2048,
'rope_freq_base': 0,
'compress_pos_emb': 1,
}
@ -48,6 +49,10 @@ def get_model_metadata(model):
metadata = metadata_gguf.load_metadata(model_file)
if 'llama.context_length' in metadata:
model_settings['n_ctx'] = metadata['llama.context_length']
if 'llama.rope.scale_linear' in metadata:
model_settings['compress_pos_emb'] = metadata['llama.rope.scale_linear']
if 'llama.rope.freq_base' in metadata:
model_settings['rope_freq_base'] = metadata['llama.rope.freq_base']
return model_settings