Implement sessions + add basic multi-user support (#2991)
This commit is contained in:
oobabooga
GitHub
parent
1f8cae14f9
commit
4b1804a438
17 changed files with 595 additions and 414 deletions
260
modules/chat.py
260
modules/chat.py
|
|
@ -3,7 +3,6 @@ import copy
|
|||
import functools
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import gradio as gr
|
||||
|
|
@ -11,7 +10,6 @@ import yaml
|
|||
from PIL import Image
|
||||
|
||||
import modules.shared as shared
|
||||
from modules import utils
|
||||
from modules.extensions import apply_extensions
|
||||
from modules.html_generator import chat_html_wrapper, make_thumbnail
|
||||
from modules.logging_colors import logger
|
||||
|
|
@ -20,7 +18,12 @@ from modules.text_generation import (
|
|||
get_encoded_length,
|
||||
get_max_prompt_length
|
||||
)
|
||||
from modules.utils import delete_file, replace_all, save_file
|
||||
from modules.utils import (
|
||||
delete_file,
|
||||
get_available_characters,
|
||||
replace_all,
|
||||
save_file
|
||||
)
|
||||
|
||||
|
||||
def get_turn_substrings(state, instruct=False):
|
||||
|
|
@ -54,7 +57,7 @@ def generate_chat_prompt(user_input, state, **kwargs):
|
|||
impersonate = kwargs.get('impersonate', False)
|
||||
_continue = kwargs.get('_continue', False)
|
||||
also_return_rows = kwargs.get('also_return_rows', False)
|
||||
history = kwargs.get('history', shared.history)['internal']
|
||||
history = kwargs.get('history', state['history'])['internal']
|
||||
is_instruct = state['mode'] == 'instruct'
|
||||
|
||||
# Find the maximum prompt size
|
||||
|
|
@ -76,10 +79,10 @@ def generate_chat_prompt(user_input, state, **kwargs):
|
|||
if impersonate:
|
||||
wrapper += substrings['user_turn_stripped'].rstrip(' ')
|
||||
elif _continue:
|
||||
wrapper += apply_extensions("bot_prefix", substrings['bot_turn_stripped'])
|
||||
wrapper += apply_extensions('bot_prefix', substrings['bot_turn_stripped'], state)
|
||||
wrapper += history[-1][1]
|
||||
else:
|
||||
wrapper += apply_extensions("bot_prefix", substrings['bot_turn_stripped'].rstrip(' '))
|
||||
wrapper += apply_extensions('bot_prefix', substrings['bot_turn_stripped'].rstrip(' '), state)
|
||||
else:
|
||||
wrapper = '<|prompt|>'
|
||||
|
||||
|
|
@ -113,7 +116,7 @@ def generate_chat_prompt(user_input, state, **kwargs):
|
|||
|
||||
# Add the character prefix
|
||||
if state['mode'] != 'chat-instruct':
|
||||
rows.append(apply_extensions("bot_prefix", substrings['bot_turn_stripped'].rstrip(' ')))
|
||||
rows.append(apply_extensions('bot_prefix', substrings['bot_turn_stripped'].rstrip(' '), state))
|
||||
|
||||
while len(rows) > min_rows and get_encoded_length(wrapper.replace('<|prompt|>', ''.join(rows))) >= max_length:
|
||||
rows.pop(1)
|
||||
|
|
@ -153,7 +156,8 @@ def get_stopping_strings(state):
|
|||
return stopping_strings
|
||||
|
||||
|
||||
def chatbot_wrapper(text, history, state, regenerate=False, _continue=False, loading_message=True):
|
||||
def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_message=True):
|
||||
history = state['history']
|
||||
output = copy.deepcopy(history)
|
||||
output = apply_extensions('history', output)
|
||||
state = apply_extensions('state', state)
|
||||
|
|
@ -174,11 +178,11 @@ def chatbot_wrapper(text, history, state, regenerate=False, _continue=False, loa
|
|||
if visible_text is None:
|
||||
visible_text = text
|
||||
|
||||
text = apply_extensions('input', text, state)
|
||||
|
||||
# *Is typing...*
|
||||
if loading_message:
|
||||
yield {'visible': output['visible'] + [[visible_text, shared.processing_message]], 'internal': output['internal']}
|
||||
|
||||
text = apply_extensions('input', text)
|
||||
else:
|
||||
text, visible_text = output['internal'][-1][0], output['visible'][-1][0]
|
||||
if regenerate:
|
||||
|
|
@ -215,7 +219,7 @@ def chatbot_wrapper(text, history, state, regenerate=False, _continue=False, loa
|
|||
# We need this global variable to handle the Stop event,
|
||||
# otherwise gradio gets confused
|
||||
if shared.stop_everything:
|
||||
output['visible'][-1][1] = apply_extensions("output", output['visible'][-1][1])
|
||||
output['visible'][-1][1] = apply_extensions('output', output['visible'][-1][1], state)
|
||||
yield output
|
||||
return
|
||||
|
||||
|
|
@ -241,7 +245,7 @@ def chatbot_wrapper(text, history, state, regenerate=False, _continue=False, loa
|
|||
else:
|
||||
cumulative_reply = reply
|
||||
|
||||
output['visible'][-1][1] = apply_extensions("output", output['visible'][-1][1])
|
||||
output['visible'][-1][1] = apply_extensions('output', output['visible'][-1][1], state)
|
||||
yield output
|
||||
|
||||
|
||||
|
|
@ -274,14 +278,15 @@ def impersonate_wrapper(text, start_with, state):
|
|||
yield cumulative_reply.lstrip(' ')
|
||||
|
||||
|
||||
def generate_chat_reply(text, history, state, regenerate=False, _continue=False, loading_message=True):
|
||||
def generate_chat_reply(text, state, regenerate=False, _continue=False, loading_message=True):
|
||||
history = state['history']
|
||||
if regenerate or _continue:
|
||||
text = ''
|
||||
if (len(history['visible']) == 1 and not history['visible'][0][0]) or len(history['internal']) == 0:
|
||||
yield history
|
||||
return
|
||||
|
||||
for history in chatbot_wrapper(text, history, state, regenerate=regenerate, _continue=_continue, loading_message=loading_message):
|
||||
for history in chatbot_wrapper(text, state, regenerate=regenerate, _continue=_continue, loading_message=loading_message):
|
||||
yield history
|
||||
|
||||
|
||||
|
|
@ -296,144 +301,116 @@ def generate_chat_reply_wrapper(text, start_with, state, regenerate=False, _cont
|
|||
send_dummy_message(text)
|
||||
send_dummy_reply(start_with)
|
||||
|
||||
for i, history in enumerate(generate_chat_reply(text, shared.history, state, regenerate, _continue, loading_message=True)):
|
||||
if i != 0:
|
||||
shared.history = copy.deepcopy(history)
|
||||
|
||||
yield chat_html_wrapper(history['visible'], state['name1'], state['name2'], state['mode'], state['chat_style'])
|
||||
for i, history in enumerate(generate_chat_reply(text, state, regenerate, _continue, loading_message=True)):
|
||||
yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style']), history
|
||||
|
||||
|
||||
def remove_last_message():
|
||||
if len(shared.history['visible']) > 0 and shared.history['internal'][-1][0] != '<|BEGIN-VISIBLE-CHAT|>':
|
||||
last = shared.history['visible'].pop()
|
||||
shared.history['internal'].pop()
|
||||
def remove_last_message(history):
|
||||
if len(history['visible']) > 0 and history['internal'][-1][0] != '<|BEGIN-VISIBLE-CHAT|>':
|
||||
last = history['visible'].pop()
|
||||
history['internal'].pop()
|
||||
else:
|
||||
last = ['', '']
|
||||
|
||||
return last[0]
|
||||
return last[0], history
|
||||
|
||||
|
||||
def send_last_reply_to_input():
|
||||
if len(shared.history['internal']) > 0:
|
||||
return shared.history['internal'][-1][1]
|
||||
def send_last_reply_to_input(history):
|
||||
if len(history['internal']) > 0:
|
||||
return history['internal'][-1][1]
|
||||
else:
|
||||
return ''
|
||||
|
||||
|
||||
def replace_last_reply(text):
|
||||
if len(shared.history['visible']) > 0:
|
||||
shared.history['visible'][-1][1] = text
|
||||
shared.history['internal'][-1][1] = apply_extensions("input", text)
|
||||
|
||||
|
||||
def send_dummy_message(text):
|
||||
shared.history['visible'].append([text, ''])
|
||||
shared.history['internal'].append([apply_extensions("input", text), ''])
|
||||
|
||||
|
||||
def send_dummy_reply(text):
|
||||
if len(shared.history['visible']) > 0 and not shared.history['visible'][-1][1] == '':
|
||||
shared.history['visible'].append(['', ''])
|
||||
shared.history['internal'].append(['', ''])
|
||||
|
||||
shared.history['visible'][-1][1] = text
|
||||
shared.history['internal'][-1][1] = apply_extensions("input", text)
|
||||
|
||||
|
||||
def clear_chat_log(greeting, mode):
|
||||
shared.history['visible'] = []
|
||||
shared.history['internal'] = []
|
||||
|
||||
if mode != 'instruct':
|
||||
if greeting != '':
|
||||
shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', greeting]]
|
||||
shared.history['visible'] += [['', apply_extensions("output", greeting)]]
|
||||
|
||||
save_history(mode)
|
||||
|
||||
|
||||
def redraw_html(name1, name2, mode, style, reset_cache=False):
|
||||
return chat_html_wrapper(shared.history['visible'], name1, name2, mode, style, reset_cache=reset_cache)
|
||||
|
||||
|
||||
def tokenize_dialogue(dialogue, name1, name2):
|
||||
history = []
|
||||
messages = []
|
||||
dialogue = re.sub('<START>', '', dialogue)
|
||||
dialogue = re.sub('<start>', '', dialogue)
|
||||
dialogue = re.sub('(\n|^)[Aa]non:', '\\1You:', dialogue)
|
||||
dialogue = re.sub('(\n|^)\[CHARACTER\]:', f'\\g<1>{name2}:', dialogue)
|
||||
idx = [m.start() for m in re.finditer(f"(^|\n)({re.escape(name1)}|{re.escape(name2)}):", dialogue)]
|
||||
if len(idx) == 0:
|
||||
return history
|
||||
|
||||
for i in range(len(idx) - 1):
|
||||
messages.append(dialogue[idx[i]:idx[i + 1]].strip())
|
||||
|
||||
messages.append(dialogue[idx[-1]:].strip())
|
||||
entry = ['', '']
|
||||
for i in messages:
|
||||
if i.startswith(f'{name1}:'):
|
||||
entry[0] = i[len(f'{name1}:'):].strip()
|
||||
elif i.startswith(f'{name2}:'):
|
||||
entry[1] = i[len(f'{name2}:'):].strip()
|
||||
if not (len(entry[0]) == 0 and len(entry[1]) == 0):
|
||||
history.append(entry)
|
||||
|
||||
entry = ['', '']
|
||||
|
||||
print("\033[1;32;1m\nDialogue tokenized to:\033[0;37;0m\n", end='')
|
||||
for row in history:
|
||||
for column in row:
|
||||
print("\n")
|
||||
for line in column.strip().split('\n'):
|
||||
print("| " + line + "\n")
|
||||
|
||||
print("|\n")
|
||||
print("------------------------------")
|
||||
def replace_last_reply(text, state):
|
||||
history = state['history']
|
||||
if len(history['visible']) > 0:
|
||||
history['visible'][-1][1] = text
|
||||
history['internal'][-1][1] = apply_extensions('input', text, state)
|
||||
|
||||
return history
|
||||
|
||||
|
||||
def save_history(mode, timestamp=False, user_request=False):
|
||||
# Instruct mode histories should not be saved as if
|
||||
# Alpaca or Vicuna were characters
|
||||
if mode == 'instruct':
|
||||
if not timestamp:
|
||||
return
|
||||
|
||||
fname = f"Instruct_{datetime.now().strftime('%Y%m%d-%H%M%S')}.json"
|
||||
else:
|
||||
if shared.character == 'None' and not user_request:
|
||||
return
|
||||
|
||||
if timestamp:
|
||||
fname = f"{shared.character}_{datetime.now().strftime('%Y%m%d-%H%M%S')}.json"
|
||||
else:
|
||||
fname = f"{shared.character}_persistent.json"
|
||||
|
||||
if not Path('logs').exists():
|
||||
Path('logs').mkdir()
|
||||
|
||||
with open(Path(f'logs/{fname}'), 'w', encoding='utf-8') as f:
|
||||
f.write(json.dumps({'data': shared.history['internal'], 'data_visible': shared.history['visible']}, indent=2))
|
||||
|
||||
return Path(f'logs/{fname}')
|
||||
def send_dummy_message(text, state):
|
||||
history = state['history']
|
||||
history['visible'].append([text, ''])
|
||||
history['internal'].append([apply_extensions('input', text, state), ''])
|
||||
return history
|
||||
|
||||
|
||||
def load_history(file, name1, name2):
|
||||
file = file.decode('utf-8')
|
||||
def send_dummy_reply(text, state):
|
||||
history = state['history']
|
||||
if len(history['visible']) > 0 and not history['visible'][-1][1] == '':
|
||||
history['visible'].append(['', ''])
|
||||
history['internal'].append(['', ''])
|
||||
|
||||
history['visible'][-1][1] = text
|
||||
history['internal'][-1][1] = apply_extensions('input', text, state)
|
||||
return history
|
||||
|
||||
|
||||
def clear_chat_log(state):
|
||||
greeting = state['greeting']
|
||||
mode = state['mode']
|
||||
history = state['history']
|
||||
|
||||
history['visible'] = []
|
||||
history['internal'] = []
|
||||
if mode != 'instruct':
|
||||
if greeting != '':
|
||||
history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', greeting]]
|
||||
history['visible'] += [['', apply_extensions('output', greeting, state)]]
|
||||
|
||||
return history
|
||||
|
||||
|
||||
def redraw_html(history, name1, name2, mode, style, reset_cache=False):
|
||||
return chat_html_wrapper(history, name1, name2, mode, style, reset_cache=reset_cache)
|
||||
|
||||
|
||||
def save_history(history, path=None):
|
||||
p = path or Path('logs/exported_history.json')
|
||||
with open(p, 'w', encoding='utf-8') as f:
|
||||
f.write(json.dumps(history, indent=4))
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def load_history(file, history):
|
||||
try:
|
||||
file = file.decode('utf-8')
|
||||
j = json.loads(file)
|
||||
if 'data' in j:
|
||||
shared.history['internal'] = j['data']
|
||||
if 'data_visible' in j:
|
||||
shared.history['visible'] = j['data_visible']
|
||||
else:
|
||||
shared.history['visible'] = copy.deepcopy(shared.history['internal'])
|
||||
if 'internal' in j and 'visible' in j:
|
||||
return j
|
||||
else:
|
||||
return history
|
||||
except:
|
||||
shared.history['internal'] = tokenize_dialogue(file, name1, name2)
|
||||
shared.history['visible'] = copy.deepcopy(shared.history['internal'])
|
||||
return history
|
||||
|
||||
|
||||
def save_persistent_history(history, character, mode):
|
||||
if mode in ['chat', 'chat-instruct'] and character not in ['', 'None', None] and not shared.args.multi_user:
|
||||
save_history(history, path=Path(f'logs/{character}_persistent.json'))
|
||||
|
||||
|
||||
def load_persistent_history(state):
|
||||
if shared.args.multi_user or state['mode'] == 'instruct':
|
||||
return state['history']
|
||||
|
||||
character = state['character_menu']
|
||||
greeting = state['greeting']
|
||||
p = Path(f'logs/{character}_persistent.json')
|
||||
if character not in ['None', '', None] and p.exists():
|
||||
f = json.loads(open(p, 'rb').read())
|
||||
if 'internal' in f and 'visible' in f:
|
||||
history = f
|
||||
else:
|
||||
history = {'internal': [], 'visible': []}
|
||||
if greeting != "":
|
||||
history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', greeting]]
|
||||
history['visible'] += [['', apply_extensions('output', greeting, state)]]
|
||||
|
||||
return history
|
||||
|
||||
|
||||
def replace_character_names(text, name1, name2):
|
||||
|
|
@ -468,7 +445,6 @@ def generate_pfp_cache(character):
|
|||
|
||||
|
||||
def load_character(character, name1, name2, instruct=False):
|
||||
shared.character = character
|
||||
context = greeting = turn_template = ""
|
||||
greeting_field = 'greeting'
|
||||
picture = None
|
||||
|
|
@ -477,7 +453,7 @@ def load_character(character, name1, name2, instruct=False):
|
|||
if Path("cache/pfp_character.png").exists():
|
||||
Path("cache/pfp_character.png").unlink()
|
||||
|
||||
if character != 'None':
|
||||
if character not in ['None', '', None]:
|
||||
folder = 'characters' if not instruct else 'characters/instruction-following'
|
||||
picture = generate_pfp_cache(character)
|
||||
for extension in ["yml", "yaml", "json"]:
|
||||
|
|
@ -527,20 +503,6 @@ def load_character(character, name1, name2, instruct=False):
|
|||
greeting = shared.settings['greeting']
|
||||
turn_template = shared.settings['turn_template']
|
||||
|
||||
if not instruct:
|
||||
shared.history['internal'] = []
|
||||
shared.history['visible'] = []
|
||||
if shared.character != 'None' and Path(f'logs/{shared.character}_persistent.json').exists():
|
||||
load_history(open(Path(f'logs/{shared.character}_persistent.json'), 'rb').read(), name1, name2)
|
||||
else:
|
||||
# Insert greeting if it exists
|
||||
if greeting != "":
|
||||
shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', greeting]]
|
||||
shared.history['visible'] += [['', apply_extensions("output", greeting)]]
|
||||
|
||||
# Create .json log files since they don't already exist
|
||||
save_history('instruct' if instruct else 'chat')
|
||||
|
||||
return name1, name2, picture, greeting, context, turn_template.replace("\n", r"\n")
|
||||
|
||||
|
||||
|
|
@ -568,7 +530,7 @@ def upload_character(json_file, img, tavern=False):
|
|||
img.save(Path(f'characters/{outfile_name}.png'))
|
||||
|
||||
logger.info(f'New character saved to "characters/{outfile_name}.json".')
|
||||
return gr.update(value=outfile_name, choices=utils.get_available_characters())
|
||||
return gr.update(value=outfile_name, choices=get_available_characters())
|
||||
|
||||
|
||||
def upload_tavern_character(img, _json):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import gradio as gr
|
|||
import extensions
|
||||
import modules.shared as shared
|
||||
from modules.logging_colors import logger
|
||||
from inspect import signature
|
||||
|
||||
|
||||
state = {}
|
||||
available_extensions = []
|
||||
|
|
@ -52,10 +54,14 @@ def iterator():
|
|||
|
||||
|
||||
# Extension functions that map string -> string
|
||||
def _apply_string_extensions(function_name, text):
|
||||
def _apply_string_extensions(function_name, text, state):
|
||||
for extension, _ in iterator():
|
||||
if hasattr(extension, function_name):
|
||||
text = getattr(extension, function_name)(text)
|
||||
func = getattr(extension, function_name)
|
||||
if len(signature(func).parameters) == 2:
|
||||
text = func(text, state)
|
||||
else:
|
||||
text = func(text)
|
||||
|
||||
return text
|
||||
|
||||
|
|
|
|||
|
|
@ -14,16 +14,20 @@ def clone_or_pull_repository(github_url):
|
|||
|
||||
# Check if the repository is already cloned
|
||||
if os.path.exists(repo_path):
|
||||
yield f"Updating {github_url}..."
|
||||
# Perform a 'git pull' to update the repository
|
||||
try:
|
||||
pull_output = subprocess.check_output(["git", "-C", repo_path, "pull"], stderr=subprocess.STDOUT)
|
||||
yield "Done."
|
||||
return pull_output.decode()
|
||||
except subprocess.CalledProcessError as e:
|
||||
return str(e)
|
||||
|
||||
# Clone the repository
|
||||
try:
|
||||
yield f"Cloning {github_url}..."
|
||||
clone_output = subprocess.check_output(["git", "clone", github_url, repo_path], stderr=subprocess.STDOUT)
|
||||
yield "Done."
|
||||
return clone_output.decode()
|
||||
except subprocess.CalledProcessError as e:
|
||||
return str(e)
|
||||
|
|
|
|||
|
|
@ -266,8 +266,8 @@ def generate_chat_html(history, name1, name2, reset_cache=False):
|
|||
|
||||
def chat_html_wrapper(history, name1, name2, mode, style, reset_cache=False):
|
||||
if mode == 'instruct':
|
||||
return generate_instruct_html(history)
|
||||
return generate_instruct_html(history['visible'])
|
||||
elif style == 'wpp':
|
||||
return generate_chat_html(history, name1, name2)
|
||||
return generate_chat_html(history['visible'], name1, name2)
|
||||
else:
|
||||
return generate_cai_chat_html(history, name1, name2, style, reset_cache)
|
||||
return generate_cai_chat_html(history['visible'], name1, name2, style, reset_cache)
|
||||
|
|
|
|||
|
|
@ -29,11 +29,12 @@ def load_preset(name):
|
|||
'mirostat_eta': 0.1,
|
||||
}
|
||||
|
||||
with open(Path(f'presets/{name}.yaml'), 'r') as infile:
|
||||
preset = yaml.safe_load(infile)
|
||||
if name not in ['None', None, '']:
|
||||
with open(Path(f'presets/{name}.yaml'), 'r') as infile:
|
||||
preset = yaml.safe_load(infile)
|
||||
|
||||
for k in preset:
|
||||
generate_params[k] = preset[k]
|
||||
for k in preset:
|
||||
generate_params[k] = preset[k]
|
||||
|
||||
generate_params['temperature'] = min(1.99, generate_params['temperature'])
|
||||
return generate_params
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ model_name = "None"
|
|||
lora_names = []
|
||||
|
||||
# Chat variables
|
||||
history = {'internal': [], 'visible': []}
|
||||
character = 'None'
|
||||
stop_everything = False
|
||||
processing_message = '*Is typing...*'
|
||||
|
||||
|
|
@ -83,6 +81,7 @@ parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.HelpForma
|
|||
# Basic settings
|
||||
parser.add_argument('--notebook', action='store_true', help='Launch the web UI in notebook mode, where the output is written to the same text box as the input.')
|
||||
parser.add_argument('--chat', action='store_true', help='Launch the web UI in chat mode with a style similar to the Character.AI website.')
|
||||
parser.add_argument('--multi-user', action='store_true', help='Multi-user mode. Chat histories are not saved or automatically loaded. WARNING: this is highly experimental.')
|
||||
parser.add_argument('--character', type=str, help='The name of the character to load in chat mode by default.')
|
||||
parser.add_argument('--model', type=str, help='Name of the model to load by default.')
|
||||
parser.add_argument('--lora', type=str, nargs="+", help='The list of LoRAs to load. If you want to load more than one LoRA, write the names separated by spaces.')
|
||||
|
|
@ -204,6 +203,8 @@ if args.trust_remote_code:
|
|||
logger.warning("trust_remote_code is enabled. This is dangerous.")
|
||||
if args.share:
|
||||
logger.warning("The gradio \"share link\" feature uses a proprietary executable to create a reverse tunnel. Use it with care.")
|
||||
if args.multi_user:
|
||||
logger.warning("The multi-user mode is highly experimental. DO NOT EXPOSE IT TO THE INTERNET.")
|
||||
|
||||
|
||||
def fix_loader_name(name):
|
||||
|
|
@ -246,6 +247,15 @@ def is_chat():
|
|||
return args.chat
|
||||
|
||||
|
||||
def get_mode():
|
||||
if args.chat:
|
||||
return 'chat'
|
||||
elif args.notebook:
|
||||
return 'notebook'
|
||||
else:
|
||||
return 'default'
|
||||
|
||||
|
||||
# Loading model-specific settings
|
||||
with Path(f'{args.model_dir}/config.yaml') as p:
|
||||
if p.exists():
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False):
|
|||
original_question = question
|
||||
if not is_chat:
|
||||
state = apply_extensions('state', state)
|
||||
question = apply_extensions('input', question)
|
||||
question = apply_extensions('input', question, state)
|
||||
|
||||
# Finding the stopping strings
|
||||
all_stop_strings = []
|
||||
|
|
@ -223,7 +223,7 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False):
|
|||
break
|
||||
|
||||
if not is_chat:
|
||||
reply = apply_extensions('output', reply)
|
||||
reply = apply_extensions('output', reply, state)
|
||||
|
||||
yield reply
|
||||
|
||||
|
|
@ -262,7 +262,7 @@ def generate_reply_HF(question, original_question, seed, state, stopping_strings
|
|||
eos_token_ids = [shared.tokenizer.eos_token_id] if shared.tokenizer.eos_token_id is not None else []
|
||||
generate_params['eos_token_id'] = eos_token_ids
|
||||
generate_params['stopping_criteria'] = transformers.StoppingCriteriaList()
|
||||
generate_params['stopping_criteria'].append(_StopEverythingStoppingCriteria());
|
||||
generate_params['stopping_criteria'].append(_StopEverythingStoppingCriteria())
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
|
|
|
|||
114
modules/ui.py
114
modules/ui.py
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import gradio as gr
|
||||
|
|
@ -5,6 +6,7 @@ import torch
|
|||
|
||||
from modules import shared
|
||||
|
||||
|
||||
with open(Path(__file__).resolve().parent / '../css/main.css', 'r') as f:
|
||||
css = f.read()
|
||||
with open(Path(__file__).resolve().parent / '../css/chat.css', 'r') as f:
|
||||
|
|
@ -14,7 +16,7 @@ with open(Path(__file__).resolve().parent / '../css/main.js', 'r') as f:
|
|||
with open(Path(__file__).resolve().parent / '../css/chat.js', 'r') as f:
|
||||
chat_js = f.read()
|
||||
|
||||
refresh_symbol = '\U0001f504' # 🔄
|
||||
refresh_symbol = '🔄'
|
||||
delete_symbol = '🗑️'
|
||||
save_symbol = '💾'
|
||||
|
||||
|
|
@ -30,17 +32,103 @@ theme = gr.themes.Default(
|
|||
|
||||
|
||||
def list_model_elements():
|
||||
elements = ['loader', 'cpu_memory', 'auto_devices', 'disk', 'cpu', 'bf16', 'load_in_8bit', 'trust_remote_code', 'load_in_4bit', 'compute_dtype', 'quant_type', 'use_double_quant', 'wbits', 'groupsize', 'model_type', 'pre_layer', 'triton', 'desc_act', 'no_inject_fused_attention', 'no_inject_fused_mlp', 'no_use_cuda_fp16', 'threads', 'n_batch', 'no_mmap', 'mlock', 'n_gpu_layers', 'n_ctx', 'llama_cpp_seed', 'gpu_split', 'max_seq_len', 'compress_pos_emb']
|
||||
elements = [
|
||||
'loader',
|
||||
'cpu_memory',
|
||||
'auto_devices',
|
||||
'disk',
|
||||
'cpu',
|
||||
'bf16',
|
||||
'load_in_8bit',
|
||||
'trust_remote_code',
|
||||
'load_in_4bit',
|
||||
'compute_dtype',
|
||||
'quant_type',
|
||||
'use_double_quant',
|
||||
'wbits',
|
||||
'groupsize',
|
||||
'model_type',
|
||||
'pre_layer',
|
||||
'triton',
|
||||
'desc_act',
|
||||
'no_inject_fused_attention',
|
||||
'no_inject_fused_mlp',
|
||||
'no_use_cuda_fp16',
|
||||
'threads',
|
||||
'n_batch',
|
||||
'no_mmap',
|
||||
'mlock',
|
||||
'n_gpu_layers',
|
||||
'n_ctx',
|
||||
'llama_cpp_seed',
|
||||
'gpu_split',
|
||||
'max_seq_len',
|
||||
'compress_pos_emb'
|
||||
]
|
||||
|
||||
for i in range(torch.cuda.device_count()):
|
||||
elements.append(f'gpu_memory_{i}')
|
||||
|
||||
return elements
|
||||
|
||||
|
||||
def list_interface_input_elements(chat=False):
|
||||
elements = ['max_new_tokens', 'seed', 'temperature', 'top_p', 'top_k', 'typical_p', 'epsilon_cutoff', 'eta_cutoff', 'repetition_penalty', 'repetition_penalty_range', 'encoder_repetition_penalty', 'no_repeat_ngram_size', 'min_length', 'do_sample', 'penalty_alpha', 'num_beams', 'length_penalty', 'early_stopping', 'mirostat_mode', 'mirostat_tau', 'mirostat_eta', 'add_bos_token', 'ban_eos_token', 'truncation_length', 'custom_stopping_strings', 'skip_special_tokens', 'preset_menu', 'stream', 'tfs', 'top_a']
|
||||
if chat:
|
||||
elements += ['name1', 'name2', 'greeting', 'context', 'chat_generation_attempts', 'stop_at_newline', 'mode', 'instruction_template', 'character_menu', 'name1_instruct', 'name2_instruct', 'context_instruct', 'turn_template', 'chat_style', 'chat-instruct_command']
|
||||
def list_interface_input_elements():
|
||||
elements = [
|
||||
'preset_menu',
|
||||
'max_new_tokens',
|
||||
'seed',
|
||||
'temperature',
|
||||
'top_p',
|
||||
'top_k',
|
||||
'typical_p',
|
||||
'epsilon_cutoff',
|
||||
'eta_cutoff',
|
||||
'repetition_penalty',
|
||||
'repetition_penalty_range',
|
||||
'encoder_repetition_penalty',
|
||||
'no_repeat_ngram_size',
|
||||
'min_length',
|
||||
'do_sample',
|
||||
'penalty_alpha',
|
||||
'num_beams',
|
||||
'length_penalty',
|
||||
'early_stopping',
|
||||
'mirostat_mode',
|
||||
'mirostat_tau',
|
||||
'mirostat_eta',
|
||||
'add_bos_token',
|
||||
'ban_eos_token',
|
||||
'truncation_length',
|
||||
'custom_stopping_strings',
|
||||
'skip_special_tokens',
|
||||
'stream',
|
||||
'tfs',
|
||||
'top_a',
|
||||
]
|
||||
|
||||
if shared.args.chat:
|
||||
elements += [
|
||||
'character_menu',
|
||||
'history',
|
||||
'name1',
|
||||
'name2',
|
||||
'greeting',
|
||||
'context',
|
||||
'chat_generation_attempts',
|
||||
'stop_at_newline',
|
||||
'mode',
|
||||
'instruction_template',
|
||||
'name1_instruct',
|
||||
'name2_instruct',
|
||||
'context_instruct',
|
||||
'turn_template',
|
||||
'chat_style',
|
||||
'chat-instruct_command',
|
||||
]
|
||||
else:
|
||||
elements.append('textbox')
|
||||
if not shared.args.notebook:
|
||||
elements.append('output_textbox')
|
||||
|
||||
elements += list_model_elements()
|
||||
return elements
|
||||
|
|
@ -48,10 +136,14 @@ def list_interface_input_elements(chat=False):
|
|||
|
||||
def gather_interface_values(*args):
|
||||
output = {}
|
||||
for i, element in enumerate(shared.input_elements):
|
||||
for i, element in enumerate(list_interface_input_elements()):
|
||||
output[element] = args[i]
|
||||
|
||||
shared.persistent_interface_state = output
|
||||
if not shared.args.multi_user:
|
||||
shared.persistent_interface_state = output
|
||||
with open(Path(f'logs/session_{shared.get_mode()}_autosave.json'), 'w') as f:
|
||||
f.write(json.dumps(output, indent=4))
|
||||
|
||||
return output
|
||||
|
||||
|
||||
|
|
@ -59,11 +151,12 @@ def apply_interface_values(state, use_persistent=False):
|
|||
if use_persistent:
|
||||
state = shared.persistent_interface_state
|
||||
|
||||
elements = list_interface_input_elements(chat=shared.is_chat())
|
||||
elements = list_interface_input_elements()
|
||||
if len(state) == 0:
|
||||
return [gr.update() for k in elements] # Dummy, do nothing
|
||||
else:
|
||||
return [state[k] if k in state else gr.update() for k in elements]
|
||||
ans = [state[k] if k in state else gr.update() for k in elements]
|
||||
return ans
|
||||
|
||||
|
||||
class ToolButton(gr.Button, gr.components.FormComponent):
|
||||
|
|
@ -92,6 +185,7 @@ def create_refresh_button(refresh_component, refresh_method, refreshed_args, ele
|
|||
inputs=[],
|
||||
outputs=[refresh_component]
|
||||
)
|
||||
|
||||
return refresh_button
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,14 @@ from modules import shared
|
|||
from modules.logging_colors import logger
|
||||
|
||||
|
||||
# Helper function to get multiple values from shared.gradio
|
||||
def gradio(*keys):
|
||||
if len(keys) == 1 and type(keys[0]) is list:
|
||||
keys = keys[0]
|
||||
|
||||
return [shared.gradio[k] for k in keys]
|
||||
|
||||
|
||||
def save_file(fname, contents):
|
||||
if fname == '':
|
||||
logger.error('File name is empty!')
|
||||
|
|
@ -111,3 +119,8 @@ def get_datasets(path: str, ext: str):
|
|||
|
||||
def get_available_chat_styles():
|
||||
return sorted(set(('-'.join(k.stem.split('-')[1:]) for k in Path('css').glob('chat_style*.css'))), key=natural_keys)
|
||||
|
||||
|
||||
def get_available_sessions():
|
||||
items = sorted(set(k.stem for k in Path('logs').glob(f'session_{shared.get_mode()}*')), key=natural_keys, reverse=True)
|
||||
return [item for item in items if 'autosave' in item] + [item for item in items if 'autosave' not in item]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue