Merge branch 'main' into main
This commit is contained in:
Alexander Hristov Hristov
GitHub
commit
63c5a139a2
10 changed files with 158 additions and 81 deletions
|
|
@ -25,10 +25,10 @@ class RWKVModel:
|
|||
tokenizer_path = Path(f"{path.parent}/20B_tokenizer.json")
|
||||
|
||||
if shared.args.rwkv_strategy is None:
|
||||
model = RWKV(model=os.path.abspath(path), strategy=f'{device} {dtype}')
|
||||
model = RWKV(model=str(path), strategy=f'{device} {dtype}')
|
||||
else:
|
||||
model = RWKV(model=os.path.abspath(path), strategy=shared.args.rwkv_strategy)
|
||||
pipeline = PIPELINE(model, os.path.abspath(tokenizer_path))
|
||||
model = RWKV(model=str(path), strategy=shared.args.rwkv_strategy)
|
||||
pipeline = PIPELINE(model, str(tokenizer_path))
|
||||
|
||||
result = self()
|
||||
result.pipeline = pipeline
|
||||
|
|
@ -61,7 +61,7 @@ class RWKVTokenizer:
|
|||
@classmethod
|
||||
def from_pretrained(self, path):
|
||||
tokenizer_path = path / "20B_tokenizer.json"
|
||||
tokenizer = Tokenizer.from_file(os.path.abspath(tokenizer_path))
|
||||
tokenizer = Tokenizer.from_file(str(tokenizer_path))
|
||||
|
||||
result = self()
|
||||
result.tokenizer = tokenizer
|
||||
|
|
|
|||
|
|
@ -22,6 +22,12 @@ def clean_chat_message(text):
|
|||
text = text.strip()
|
||||
return text
|
||||
|
||||
def generate_chat_output(history, name1, name2, character):
|
||||
if shared.args.cai_chat:
|
||||
return generate_chat_html(history, name1, name2, character)
|
||||
else:
|
||||
return history
|
||||
|
||||
def generate_chat_prompt(user_input, max_new_tokens, name1, name2, context, chat_prompt_size, impersonate=False):
|
||||
user_input = clean_chat_message(user_input)
|
||||
rows = [f"{context.strip()}\n"]
|
||||
|
|
@ -53,7 +59,6 @@ def generate_chat_prompt(user_input, max_new_tokens, name1, name2, context, chat
|
|||
|
||||
def extract_message_from_reply(question, reply, name1, name2, check, impersonate=False):
|
||||
next_character_found = False
|
||||
substring_found = False
|
||||
|
||||
asker = name1 if not impersonate else name2
|
||||
replier = name2 if not impersonate else name1
|
||||
|
|
@ -79,15 +84,15 @@ def extract_message_from_reply(question, reply, name1, name2, check, impersonate
|
|||
next_character_found = True
|
||||
reply = clean_chat_message(reply)
|
||||
|
||||
# Detect if something like "\nYo" is generated just before
|
||||
# "\nYou:" is completed
|
||||
tmp = f"\n{asker}:"
|
||||
for j in range(1, len(tmp)):
|
||||
if reply[-j:] == tmp[:j]:
|
||||
# If something like "\nYo" is generated just before "\nYou:"
|
||||
# is completed, trim it
|
||||
next_turn = f"\n{asker}:"
|
||||
for j in range(len(next_turn)-1, 0, -1):
|
||||
if reply[-j:] == next_turn[:j]:
|
||||
reply = reply[:-j]
|
||||
substring_found = True
|
||||
break
|
||||
|
||||
return reply, next_character_found, substring_found
|
||||
return reply, next_character_found
|
||||
|
||||
def stop_everything_event():
|
||||
shared.stop_everything = True
|
||||
|
|
@ -122,7 +127,6 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical
|
|||
prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size)
|
||||
|
||||
if not regenerate:
|
||||
# Display user input and "*is typing...*" imediately
|
||||
yield shared.history['visible']+[[visible_text, '*Is typing...*']]
|
||||
|
||||
# Generate
|
||||
|
|
@ -131,7 +135,7 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical
|
|||
for reply in generate_reply(f"{prompt}{' ' if len(reply) > 0 else ''}{reply}", max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name1}:"):
|
||||
|
||||
# Extracting the reply
|
||||
reply, next_character_found, substring_found = extract_message_from_reply(prompt, reply, name1, name2, check)
|
||||
reply, next_character_found = extract_message_from_reply(prompt, reply, name1, name2, check)
|
||||
visible_reply = re.sub("(<USER>|<user>|{{user}})", name1_original, reply)
|
||||
visible_reply = apply_extensions(visible_reply, "output")
|
||||
if shared.args.chat:
|
||||
|
|
@ -148,7 +152,7 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical
|
|||
|
||||
shared.history['internal'][-1] = [text, reply]
|
||||
shared.history['visible'][-1] = [visible_text, visible_reply]
|
||||
if not substring_found and not shared.args.no_stream:
|
||||
if not shared.args.no_stream:
|
||||
yield shared.history['visible']
|
||||
if next_character_found:
|
||||
break
|
||||
|
|
@ -163,15 +167,12 @@ def impersonate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typ
|
|||
|
||||
prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size, impersonate=True)
|
||||
|
||||
# Display "*is typing...*" imediately
|
||||
yield '*Is typing...*'
|
||||
|
||||
reply = ''
|
||||
yield '*Is typing...*'
|
||||
for i in range(chat_generation_attempts):
|
||||
for reply in generate_reply(prompt+reply, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name2}:"):
|
||||
reply, next_character_found, substring_found = extract_message_from_reply(prompt, reply, name1, name2, check, impersonate=True)
|
||||
if not substring_found:
|
||||
yield reply
|
||||
reply, next_character_found = extract_message_from_reply(prompt, reply, name1, name2, check, impersonate=True)
|
||||
yield reply
|
||||
if next_character_found:
|
||||
break
|
||||
yield reply
|
||||
|
|
@ -182,21 +183,18 @@ def cai_chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typ
|
|||
|
||||
def regenerate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, name1, name2, context, check, chat_prompt_size, chat_generation_attempts=1):
|
||||
if (shared.character != 'None' and len(shared.history['visible']) == 1) or len(shared.history['internal']) == 0:
|
||||
if shared.args.cai_chat:
|
||||
yield generate_chat_html(shared.history['visible'], name1, name2, shared.character)
|
||||
else:
|
||||
yield shared.history['visible']
|
||||
yield generate_chat_output(shared.history['visible'], name1, name2, shared.character)
|
||||
else:
|
||||
last_visible = shared.history['visible'].pop()
|
||||
last_internal = shared.history['internal'].pop()
|
||||
|
||||
yield generate_chat_output(shared.history['visible']+[[last_visible[0], '*Is typing...*']], name1, name2, shared.character)
|
||||
for _history in chatbot_wrapper(last_internal[0], max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, name1, name2, context, check, chat_prompt_size, chat_generation_attempts, regenerate=True):
|
||||
if shared.args.cai_chat:
|
||||
shared.history['visible'][-1] = [last_visible[0], _history[-1][1]]
|
||||
yield generate_chat_html(shared.history['visible'], name1, name2, shared.character)
|
||||
else:
|
||||
shared.history['visible'][-1] = (last_visible[0], _history[-1][1])
|
||||
yield shared.history['visible']
|
||||
yield generate_chat_output(shared.history['visible'], name1, name2, shared.character)
|
||||
|
||||
def remove_last_message(name1, name2):
|
||||
if len(shared.history['visible']) > 0 and not shared.history['internal'][-1][0] == '<|BEGIN-VISIBLE-CHAT|>':
|
||||
|
|
@ -204,6 +202,7 @@ def remove_last_message(name1, name2):
|
|||
shared.history['internal'].pop()
|
||||
else:
|
||||
last = ['', '']
|
||||
|
||||
if shared.args.cai_chat:
|
||||
return generate_chat_html(shared.history['visible'], name1, name2, shared.character), last[0]
|
||||
else:
|
||||
|
|
@ -223,10 +222,7 @@ def replace_last_reply(text, name1, name2):
|
|||
shared.history['visible'][-1] = (shared.history['visible'][-1][0], text)
|
||||
shared.history['internal'][-1][1] = apply_extensions(text, "input")
|
||||
|
||||
if shared.args.cai_chat:
|
||||
return generate_chat_html(shared.history['visible'], name1, name2, shared.character)
|
||||
else:
|
||||
return shared.history['visible']
|
||||
return generate_chat_output(shared.history['visible'], name1, name2, shared.character)
|
||||
|
||||
def clear_html():
|
||||
return generate_chat_html([], "", "", shared.character)
|
||||
|
|
@ -246,10 +242,8 @@ def clear_chat_log(name1, name2):
|
|||
else:
|
||||
shared.history['internal'] = []
|
||||
shared.history['visible'] = []
|
||||
if shared.args.cai_chat:
|
||||
return generate_chat_html(shared.history['visible'], name1, name2, shared.character)
|
||||
else:
|
||||
return shared.history['visible']
|
||||
|
||||
return generate_chat_output(shared.history['visible'], name1, name2, shared.character)
|
||||
|
||||
def redraw_html(name1, name2):
|
||||
return generate_chat_html(shared.history['visible'], name1, name2, shared.character)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -7,7 +6,7 @@ import torch
|
|||
|
||||
import modules.shared as shared
|
||||
|
||||
sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa")))
|
||||
sys.path.insert(0, str(Path("repositories/GPTQ-for-LLaMa")))
|
||||
from llama import load_quant
|
||||
|
||||
|
||||
|
|
@ -41,9 +40,9 @@ def load_quantized_LLaMA(model_name):
|
|||
print(f"Could not find {pt_model}, exiting...")
|
||||
exit()
|
||||
|
||||
model = load_quant(path_to_model, os.path.abspath(pt_path), bits)
|
||||
model = load_quant(str(path_to_model), str(pt_path), bits)
|
||||
|
||||
# Multi-GPU setup
|
||||
# Multiple GPUs or GPU+CPU
|
||||
if shared.args.gpu_memory:
|
||||
max_memory = {}
|
||||
for i in range(len(shared.args.gpu_memory)):
|
||||
|
|
|
|||
|
|
@ -85,12 +85,12 @@ parser.add_argument('--nvme-offload-dir', type=str, help='DeepSpeed: Directory t
|
|||
parser.add_argument('--local_rank', type=int, default=0, help='DeepSpeed: Optional argument for distributed setups.')
|
||||
parser.add_argument('--rwkv-strategy', type=str, default=None, help='RWKV: The strategy to use while loading the model. Examples: "cpu fp32", "cuda fp16", "cuda fp16i8".')
|
||||
parser.add_argument('--rwkv-cuda-on', action='store_true', help='RWKV: Compile the CUDA kernel for better performance.')
|
||||
parser.add_argument('--no-stream', action='store_true', help='Don\'t stream the text output in real time. This improves the text generation performance.')
|
||||
parser.add_argument('--no-stream', action='store_true', help='Don\'t stream the text output in real time.')
|
||||
parser.add_argument('--settings', type=str, help='Load the default interface settings from this json file. See settings-template.json for an example. If you create a file called settings.json, this file will be loaded by default without the need to use the --settings flag.')
|
||||
parser.add_argument('--extensions', type=str, nargs="+", help='The list of extensions to load. If you want to load more than one extension, write the names separated by spaces.')
|
||||
parser.add_argument('--listen', action='store_true', help='Make the web UI reachable from your local network.')
|
||||
parser.add_argument('--listen-port', type=int, help='The listening port that the server will use.')
|
||||
parser.add_argument('--share', action='store_true', help='Create a public URL. This is useful for running the web UI on Google Colab or similar.')
|
||||
parser.add_argument('--auto-launch', action='store_true', default=False, help='Open the web UI in the default browser upon launch.')
|
||||
parser.add_argument('--verbose', action='store_true', help='Print the prompts to the terminal.')
|
||||
parser.add_argument('--auto-launch', action='store_true', default=False, help='Open the web UI in the default browser upon launch')
|
||||
args = parser.parse_args()
|
||||
|
|
|
|||
|
|
@ -37,9 +37,13 @@ def encode(prompt, tokens_to_generate=0, add_special_tokens=True):
|
|||
return input_ids.cuda()
|
||||
|
||||
def decode(output_ids):
|
||||
reply = shared.tokenizer.decode(output_ids, skip_special_tokens=True)
|
||||
reply = reply.replace(r'<|endoftext|>', '')
|
||||
return reply
|
||||
# Open Assistant relies on special tokens like <|endoftext|>
|
||||
if re.match('oasst-*', shared.model_name.lower()):
|
||||
return shared.tokenizer.decode(output_ids, skip_special_tokens=False)
|
||||
else:
|
||||
reply = shared.tokenizer.decode(output_ids, skip_special_tokens=True)
|
||||
reply = reply.replace(r'<|endoftext|>', '')
|
||||
return reply
|
||||
|
||||
def generate_softprompt_input_tensors(input_ids):
|
||||
inputs_embeds = shared.model.transformer.wte(input_ids)
|
||||
|
|
@ -119,7 +123,9 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
|
|||
original_input_ids = input_ids
|
||||
output = input_ids[0]
|
||||
cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()"
|
||||
n = shared.tokenizer.eos_token_id if eos_token is None else int(encode(eos_token)[0][-1])
|
||||
eos_token_ids = [shared.tokenizer.eos_token_id] if shared.tokenizer.eos_token_id is not None else []
|
||||
if eos_token is not None:
|
||||
eos_token_ids.append(int(encode(eos_token)[0][-1]))
|
||||
stopping_criteria_list = transformers.StoppingCriteriaList()
|
||||
if stopping_string is not None:
|
||||
# Copied from https://github.com/PygmalionAI/gradio-ui/blob/master/src/model.py
|
||||
|
|
@ -129,7 +135,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
|
|||
if not shared.args.flexgen:
|
||||
generate_params = [
|
||||
f"max_new_tokens=max_new_tokens",
|
||||
f"eos_token_id={n}",
|
||||
f"eos_token_id={eos_token_ids}",
|
||||
f"stopping_criteria=stopping_criteria_list",
|
||||
f"do_sample={do_sample}",
|
||||
f"temperature={temperature}",
|
||||
|
|
@ -149,7 +155,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
|
|||
f"max_new_tokens={max_new_tokens if shared.args.no_stream else 8}",
|
||||
f"do_sample={do_sample}",
|
||||
f"temperature={temperature}",
|
||||
f"stop={n}",
|
||||
f"stop={eos_token_ids[-1]}",
|
||||
]
|
||||
if shared.args.deepspeed:
|
||||
generate_params.append("synced_gpus=True")
|
||||
|
|
@ -196,10 +202,12 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
|
|||
|
||||
if not (shared.args.chat or shared.args.cai_chat):
|
||||
reply = original_question + apply_extensions(reply[len(question):], "output")
|
||||
|
||||
if output[-1] in eos_token_ids:
|
||||
break
|
||||
yield formatted_outputs(reply, shared.model_name)
|
||||
|
||||
if output[-1] == n:
|
||||
break
|
||||
yield formatted_outputs(reply, shared.model_name)
|
||||
|
||||
# Stream the output naively for FlexGen since it doesn't support 'stopping_criteria'
|
||||
else:
|
||||
|
|
@ -213,15 +221,17 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
|
|||
|
||||
if not (shared.args.chat or shared.args.cai_chat):
|
||||
reply = original_question + apply_extensions(reply[len(question):], "output")
|
||||
yield formatted_outputs(reply, shared.model_name)
|
||||
|
||||
if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n):
|
||||
if np.count_nonzero(np.isin(input_ids[0], eos_token_ids)) < np.count_nonzero(np.isin(output, eos_token_ids)):
|
||||
break
|
||||
yield formatted_outputs(reply, shared.model_name)
|
||||
|
||||
input_ids = np.reshape(output, (1, output.shape[0]))
|
||||
if shared.soft_prompt:
|
||||
inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids)
|
||||
|
||||
yield formatted_outputs(reply, shared.model_name)
|
||||
|
||||
finally:
|
||||
t1 = time.time()
|
||||
print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(original_input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(original_input_ids[0])} tokens)")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue