mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Create llamacpp_HF loader (#3062)

Browse source
This commit is contained in:
oobabooga 2023-07-16 02:21:13 -03:00 • committed by GitHub
parent 7c4d4fc7d3
commit 5e3f7e00a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 143 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

22
modules/models.py
View file

@ -55,6 +55,7 @@ def load_model(model_name, loader=None):
'AutoGPTQ': AutoGPTQ_loader,
'GPTQ-for-LLaMa': GPTQ_loader,
'llama.cpp': llamacpp_loader,
'llamacpp_HF': llamacpp_HF_loader,
'FlexGen': flexgen_loader,
'RWKV': RWKV_loader,
'ExLlama': ExLlama_loader,
@ -268,6 +269,27 @@ def llamacpp_loader(model_name):
return model, tokenizer
def llamacpp_HF_loader(model_name):
from modules.llamacpp_hf import LlamacppHF
for fname in ["oobabooga_llama-tokenizer", "llama-tokenizer"]:
path = Path(f'{shared.args.model_dir}/{fname}')
if path.exists():
break
else:
logger.error("Could not load the model because a tokenizer in transformers format was not found. Please download oobabooga/llama-tokenizer.")
return None, None
tokenizer = AutoTokenizer.from_pretrained(
path,
trust_remote_code=shared.args.trust_remote_code,
use_fast=False
)
model = LlamacppHF.from_pretrained(model_name)
return model, tokenizer
def GPTQ_loader(model_name):
# Monkey patch