mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Bump exllamav2 to 0.0.12 (#5352)

Browse source
This commit is contained in:
oobabooga 2024-01-22 22:40:12 -03:00 • committed by GitHub
parent aa575119e6
commit 87dc421ee8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 45 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

14
modules/exllamav2.py
View file

@ -93,17 +93,27 @@ class Exllamav2Model:
def generate_with_streaming(self, prompt, state):
settings = ExLlamaV2Sampler.Settings()
settings.token_repetition_penalty = state['repetition_penalty']
settings.token_repetition_range = -1 if state['repetition_penalty_range'] <= 0 else state['repetition_penalty_range']
settings.token_frequency_penalty = state['frequency_penalty']
settings.token_presence_penalty = state['presence_penalty']
settings.temperature = state['temperature']
settings.top_k = state['top_k']
settings.top_p = state['top_p']
settings.top_a = state['top_a']
settings.min_p = state['min_p']
settings.tfs = state['tfs']
settings.typical = state['typical_p']
settings.temperature_last = state['temperature_last']
settings.mirostat = state['mirostat_mode'] == 2
settings.mirostat_tau = state['mirostat_tau']
settings.mirostat_eta = state['mirostat_eta']
settings.token_repetition_penalty = state['repetition_penalty']
settings.token_repetition_range = -1 if state['repetition_penalty_range'] <= 0 else state['repetition_penalty_range']
if state['ban_eos_token']:
settings.disallow_tokens(self.tokenizer, [self.tokenizer.eos_token_id])

4
modules/loaders.py
View file

@ -205,12 +205,16 @@ loaders_samplers = {
'HQQ': transformers_samplers(),
'ExLlamav2': {
'temperature',
'temperature_last',
'top_p',
'min_p',
'top_k',
'typical_p',
'tfs',
'top_a',
'repetition_penalty',
'presence_penalty',
'frequency_penalty',
'repetition_penalty_range',
'seed',
'mirostat_mode',