This commit is contained in:
oobabooga 2023-07-12 11:33:25 -07:00
parent 9b55d3a9f9
commit e202190c4f
24 changed files with 146 additions and 125 deletions

View file

@ -106,11 +106,11 @@ def load_tokenizer(model_name, model):
use_fast=False
)
except ValueError:
tokenizer = AutoTokenizer.from_pretrained(
tokenizer = AutoTokenizer.from_pretrained(
path_to_model,
trust_remote_code=shared.args.trust_remote_code,
use_fast=True
)
)
if tokenizer.__class__.__name__ == 'LlamaTokenizer':
pairs = [