Bump llama-cpp-python to 0.2.18 (#4611)
This commit is contained in:
oobabooga
GitHub
parent
61f429563e
commit
923c8e25fb
17 changed files with 92 additions and 174 deletions
|
|
@ -2,6 +2,7 @@ import os
|
|||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
import llama_cpp
|
||||
import torch
|
||||
from torch.nn import CrossEntropyLoss
|
||||
from transformers import GenerationConfig, PretrainedConfig, PreTrainedModel
|
||||
|
|
@ -10,23 +11,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
|
|||
from modules import RoPE, shared
|
||||
from modules.logging_colors import logger
|
||||
|
||||
try:
|
||||
import llama_cpp
|
||||
except:
|
||||
llama_cpp = None
|
||||
|
||||
try:
|
||||
import llama_cpp_cuda
|
||||
except:
|
||||
llama_cpp_cuda = None
|
||||
|
||||
|
||||
def llama_cpp_lib():
|
||||
if (shared.args.cpu and llama_cpp is not None) or llama_cpp_cuda is None:
|
||||
return llama_cpp
|
||||
else:
|
||||
return llama_cpp_cuda
|
||||
|
||||
|
||||
class LlamacppHF(PreTrainedModel):
|
||||
def __init__(self, model, path):
|
||||
|
|
@ -39,7 +23,7 @@ class LlamacppHF(PreTrainedModel):
|
|||
'n_tokens': self.model.n_tokens,
|
||||
'input_ids': self.model.input_ids,
|
||||
'scores': self.model.scores,
|
||||
'ctx': self.model.ctx
|
||||
'ctx': self.model._ctx.ctx
|
||||
}
|
||||
|
||||
if shared.args.cfg_cache:
|
||||
|
|
@ -48,7 +32,7 @@ class LlamacppHF(PreTrainedModel):
|
|||
'n_tokens': self.model.n_tokens,
|
||||
'input_ids': self.model.input_ids.copy(),
|
||||
'scores': self.model.scores.copy(),
|
||||
'ctx': llama_cpp_lib().llama_new_context_with_model(model.model, model.context_params)
|
||||
'ctx': llama_cpp.llama_new_context_with_model(model.model, model.context_params)
|
||||
}
|
||||
|
||||
def _validate_model_class(self):
|
||||
|
|
@ -65,7 +49,7 @@ class LlamacppHF(PreTrainedModel):
|
|||
'n_tokens': self.model.n_tokens,
|
||||
'input_ids': self.model.input_ids,
|
||||
'scores': self.model.scores,
|
||||
'ctx': self.model.ctx
|
||||
'ctx': self.model._ctx.ctx
|
||||
})
|
||||
|
||||
def save_negative_cache(self):
|
||||
|
|
@ -73,20 +57,20 @@ class LlamacppHF(PreTrainedModel):
|
|||
'n_tokens': self.model.n_tokens,
|
||||
'input_ids': self.model.input_ids,
|
||||
'scores': self.model.scores,
|
||||
'ctx': self.model.ctx
|
||||
'ctx': self.model._ctx.ctx
|
||||
})
|
||||
|
||||
def load_cache(self):
|
||||
self.model.n_tokens = self.llamacpp_cache['n_tokens']
|
||||
self.model.input_ids = self.llamacpp_cache['input_ids']
|
||||
self.model.scores = self.llamacpp_cache['scores']
|
||||
self.model.ctx = self.llamacpp_cache['ctx']
|
||||
self.model._ctx.ctx = self.llamacpp_cache['ctx']
|
||||
|
||||
def load_negative_cache(self):
|
||||
self.model.n_tokens = self.llamacpp_cache_negative['n_tokens']
|
||||
self.model.input_ids = self.llamacpp_cache_negative['input_ids']
|
||||
self.model.scores = self.llamacpp_cache_negative['scores']
|
||||
self.model.ctx = self.llamacpp_cache_negative['ctx']
|
||||
self.model._ctx.ctx = self.llamacpp_cache_negative['ctx']
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
|
|
@ -192,7 +176,6 @@ class LlamacppHF(PreTrainedModel):
|
|||
params = {
|
||||
'model_path': str(model_file),
|
||||
'n_ctx': shared.args.n_ctx,
|
||||
'seed': int(shared.args.llama_cpp_seed),
|
||||
'n_threads': shared.args.threads or None,
|
||||
'n_threads_batch': shared.args.threads_batch or None,
|
||||
'n_batch': shared.args.n_batch,
|
||||
|
|
@ -207,7 +190,5 @@ class LlamacppHF(PreTrainedModel):
|
|||
'logits_all': shared.args.logits_all,
|
||||
}
|
||||
|
||||
Llama = llama_cpp_lib().Llama
|
||||
model = Llama(**params)
|
||||
|
||||
model = llama_cpp.Llama(**params)
|
||||
return LlamacppHF(model, model_file)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import re
|
||||
from functools import partial
|
||||
|
||||
import llama_cpp
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
|
@ -9,23 +10,6 @@ from modules.callbacks import Iteratorize
|
|||
from modules.logging_colors import logger
|
||||
from modules.text_generation import get_max_prompt_length
|
||||
|
||||
try:
|
||||
import llama_cpp
|
||||
except:
|
||||
llama_cpp = None
|
||||
|
||||
try:
|
||||
import llama_cpp_cuda
|
||||
except:
|
||||
llama_cpp_cuda = None
|
||||
|
||||
|
||||
def llama_cpp_lib():
|
||||
if (shared.args.cpu and llama_cpp is not None) or llama_cpp_cuda is None:
|
||||
return llama_cpp
|
||||
else:
|
||||
return llama_cpp_cuda
|
||||
|
||||
|
||||
def ban_eos_logits_processor(eos_token, input_ids, logits):
|
||||
logits[eos_token] = -float('inf')
|
||||
|
|
@ -50,10 +34,6 @@ class LlamaCppModel:
|
|||
|
||||
@classmethod
|
||||
def from_pretrained(self, path):
|
||||
|
||||
Llama = llama_cpp_lib().Llama
|
||||
LlamaCache = llama_cpp_lib().LlamaCache
|
||||
|
||||
result = self()
|
||||
cache_capacity = 0
|
||||
if shared.args.cache_capacity is not None:
|
||||
|
|
@ -74,7 +54,6 @@ class LlamaCppModel:
|
|||
params = {
|
||||
'model_path': str(path),
|
||||
'n_ctx': shared.args.n_ctx,
|
||||
'seed': int(shared.args.llama_cpp_seed),
|
||||
'n_threads': shared.args.threads or None,
|
||||
'n_threads_batch': shared.args.threads_batch or None,
|
||||
'n_batch': shared.args.n_batch,
|
||||
|
|
@ -88,9 +67,9 @@ class LlamaCppModel:
|
|||
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
|
||||
}
|
||||
|
||||
result.model = Llama(**params)
|
||||
result.model = llama_cpp.Llama(**params)
|
||||
if cache_capacity > 0:
|
||||
result.model.set_cache(LlamaCache(capacity_bytes=cache_capacity))
|
||||
result.model.set_cache(llama_cpp.LlamaCache(capacity_bytes=cache_capacity))
|
||||
|
||||
# This is ugly, but the model and the tokenizer are the same object in this library.
|
||||
return result, result
|
||||
|
|
@ -114,13 +93,13 @@ class LlamaCppModel:
|
|||
if string != self.grammar_string:
|
||||
self.grammar_string = string
|
||||
if string.strip() != '':
|
||||
self.grammar = llama_cpp_lib().LlamaGrammar.from_string(string)
|
||||
self.grammar = llama_cpp.LlamaGrammar.from_string(string)
|
||||
else:
|
||||
self.grammar = None
|
||||
|
||||
def generate(self, prompt, state, callback=None):
|
||||
|
||||
LogitsProcessorList = llama_cpp_lib().LogitsProcessorList
|
||||
LogitsProcessorList = llama_cpp.LogitsProcessorList
|
||||
|
||||
prompt = prompt if type(prompt) is str else prompt.decode()
|
||||
|
||||
|
|
@ -144,15 +123,16 @@ class LlamaCppModel:
|
|||
max_tokens=state['max_new_tokens'],
|
||||
temperature=state['temperature'],
|
||||
top_p=state['top_p'],
|
||||
top_k=state['top_k'],
|
||||
repeat_penalty=state['repetition_penalty'],
|
||||
presence_penalty=state['presence_penalty'],
|
||||
frequency_penalty=state['frequency_penalty'],
|
||||
presence_penalty=state['presence_penalty'],
|
||||
repeat_penalty=state['repetition_penalty'],
|
||||
top_k=state['top_k'],
|
||||
stream=True,
|
||||
seed=int(state['seed']) if state['seed'] != -1 else None,
|
||||
tfs_z=state['tfs'],
|
||||
mirostat_mode=int(state['mirostat_mode']),
|
||||
mirostat_tau=state['mirostat_tau'],
|
||||
mirostat_eta=state['mirostat_eta'],
|
||||
stream=True,
|
||||
logits_processor=logit_processors,
|
||||
grammar=self.grammar
|
||||
)
|
||||
|
|
|
|||
|
|
@ -99,11 +99,9 @@ loaders_and_params = OrderedDict({
|
|||
'no_mmap',
|
||||
'mlock',
|
||||
'no_mul_mat_q',
|
||||
'llama_cpp_seed',
|
||||
'alpha_value',
|
||||
'rope_freq_base',
|
||||
'compress_pos_emb',
|
||||
'cpu',
|
||||
'numa',
|
||||
],
|
||||
'llamacpp_HF': [
|
||||
|
|
@ -119,7 +117,6 @@ loaders_and_params = OrderedDict({
|
|||
'alpha_value',
|
||||
'rope_freq_base',
|
||||
'compress_pos_emb',
|
||||
'cpu',
|
||||
'numa',
|
||||
'cfg_cache',
|
||||
'use_fast',
|
||||
|
|
@ -366,6 +363,7 @@ loaders_samplers = {
|
|||
'repetition_penalty',
|
||||
'presence_penalty',
|
||||
'frequency_penalty',
|
||||
'seed',
|
||||
'mirostat_mode',
|
||||
'mirostat_tau',
|
||||
'mirostat_eta',
|
||||
|
|
|
|||
|
|
@ -112,7 +112,6 @@ parser.add_argument('--no-mmap', action='store_true', help='Prevent mmap from be
|
|||
parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.')
|
||||
parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.')
|
||||
parser.add_argument('--tensor_split', type=str, default=None, help='Split the model across multiple GPUs. Comma-separated list of proportions. Example: 18,17.')
|
||||
parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default is 0 (random).')
|
||||
parser.add_argument('--numa', action='store_true', help='Activate NUMA task allocation for llama.cpp.')
|
||||
parser.add_argument('--logits_all', action='store_true', help='Needs to be set for perplexity evaluation to work. Otherwise, ignore it, as it makes prompt processing slower.')
|
||||
parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity (llama-cpp-python). Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.')
|
||||
|
|
@ -182,6 +181,7 @@ parser.add_argument('--no-stream', action='store_true', help='DEPRECATED')
|
|||
parser.add_argument('--mul_mat_q', action='store_true', help='DEPRECATED')
|
||||
parser.add_argument('--api-blocking-port', type=int, default=5000, help='DEPRECATED')
|
||||
parser.add_argument('--api-streaming-port', type=int, default=5005, help='DEPRECATED')
|
||||
parser.add_argument('--llama_cpp_seed', type=int, default=0, help='DEPRECATED')
|
||||
|
||||
args = parser.parse_args()
|
||||
args_defaults = parser.parse_args([])
|
||||
|
|
|
|||
|
|
@ -80,7 +80,6 @@ def list_model_elements():
|
|||
'n_gpu_layers',
|
||||
'tensor_split',
|
||||
'n_ctx',
|
||||
'llama_cpp_seed',
|
||||
'gpu_split',
|
||||
'max_seq_len',
|
||||
'compress_pos_emb',
|
||||
|
|
|
|||
|
|
@ -121,7 +121,6 @@ def create_ui():
|
|||
shared.gradio['load_in_4bit'] = gr.Checkbox(label="load-in-4bit", value=shared.args.load_in_4bit)
|
||||
shared.gradio['use_double_quant'] = gr.Checkbox(label="use_double_quant", value=shared.args.use_double_quant)
|
||||
shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17')
|
||||
shared.gradio['llama_cpp_seed'] = gr.Number(label='Seed (0 for random)', value=shared.args.llama_cpp_seed)
|
||||
shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='To enable this option, start the web UI with the --trust-remote-code flag. It is necessary for some models.', interactive=shared.args.trust_remote_code)
|
||||
shared.gradio['use_fast'] = gr.Checkbox(label="use_fast", value=shared.args.use_fast, info='Set use_fast=True while loading the tokenizer. May trigger a conversion that takes several minutes.')
|
||||
shared.gradio['logits_all'] = gr.Checkbox(label="logits_all", value=shared.args.logits_all, info='Needs to be set for perplexity evaluation to work. Otherwise, ignore it, as it makes prompt processing slower.')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue