From 875847bf88c52166c4e9a0cc35f7e6c535b88d97 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 10 Mar 2023 00:45:28 -0300
Subject: [PATCH] Consider tokenizer a type of text

---
 download-model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/download-model.py b/download-model.py
index 27fbffd..bf94be7 100644
--- a/download-model.py
+++ b/download-model.py
@@ -108,7 +108,7 @@ def get_download_links_from_huggingface(model, branch):
             is_pytorch = re.match("pytorch_model.*\.bin", fname)
             is_safetensors = re.match("model.*\.safetensors", fname)
             is_tokenizer = re.match("tokenizer.*\.model", fname)
-            is_text = re.match(".*\.(txt|json)", fname)
+            is_text = re.match(".*\.(txt|json)", fname) or is_tokenizer
 
             if any((is_pytorch, is_safetensors, is_text, is_tokenizer)):
                 if is_text: