lint
This commit is contained in:
parent
9b55d3a9f9
commit
e202190c4f
24 changed files with 146 additions and 125 deletions
|
@ -106,11 +106,11 @@ def load_tokenizer(model_name, model):
|
|||
use_fast=False
|
||||
)
|
||||
except ValueError:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
path_to_model,
|
||||
trust_remote_code=shared.args.trust_remote_code,
|
||||
use_fast=True
|
||||
)
|
||||
)
|
||||
|
||||
if tokenizer.__class__.__name__ == 'LlamaTokenizer':
|
||||
pairs = [
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue