mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Bump ExLlamaV2 to 0.0.5 (#4186)

Browse source
This commit is contained in:
turboderp 2023-10-06 00:12:22 +02:00 • committed by GitHub
parent 7ffb424c7b
commit 8a98646a21
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 16 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/exllamav2.py
View file

@ -110,7 +110,7 @@ class Exllamav2Model:
has_leading_space = False
for i in range(max_new_tokens):
logits = self.model.forward(ids[:, -1:], self.cache, input_mask=None).float().cpu()
token, _ = ExLlamaV2Sampler.sample(logits, settings, ids, random.random())
token, _, _= ExLlamaV2Sampler.sample(logits, settings, ids, random.random(), self.tokenizer)
ids = torch.cat([ids, token], dim=1)
if i == 0 and self.tokenizer.tokenizer.IdToPiece(int(token)).startswith('▁'):