diff --git a/download-model.py b/download-model.py index 98b57bb..8be398c 100644 --- a/download-model.py +++ b/download-model.py @@ -113,9 +113,10 @@ def get_download_links_from_huggingface(model, branch): is_pytorch = re.match("pytorch_model.*\.bin", fname) is_safetensors = re.match("model.*\.safetensors", fname) - is_text = re.match(".*\.(txt|json)", fname) + is_tokenizer = re.match("tokenizer.*\.model", fname) + is_text = re.match(".*\.(txt|json)", fname) or is_tokenizer - if is_text or is_safetensors or is_pytorch: + if any((is_pytorch, is_safetensors, is_text, is_tokenizer)): if is_text: links.append(f"https://huggingface.co/{model}/resolve/{branch}/{fname}") classifications.append('text')