mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add context_instruct to API. Load default model instruction template … (#2688)

Browse source
This commit is contained in:
atriantafy 2023-07-12 04:01:03 +01:00 • committed by GitHub
parent 324e45b848
commit d9fabdde40
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 37 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

3
api-examples/api-example-chat-stream.py
View file

@ -23,7 +23,8 @@ async def run(user_input, history):
'history': history, 'history': history,
'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct' 'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct'
'character': 'Example', 'character': 'Example',
'instruction_template': 'Vicuna-v1.1', 'instruction_template': 'Vicuna-v1.1', # Will get autodetected if unset
# 'context_instruct': '', # Optional
'your_name': 'You', 'your_name': 'You',
'regenerate': False, 'regenerate': False,

3
api-examples/api-example-chat.py
View file

@ -17,7 +17,8 @@ def run(user_input, history):
'history': history, 'history': history,
'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct' 'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct'
'character': 'Example', 'character': 'Example',
'instruction_template': 'Vicuna-v1.1', 'instruction_template': 'Vicuna-v1.1', # Will get autodetected if unset
# 'context_instruct': '', # Optional
'your_name': 'You', 'your_name': 'You',
'regenerate': False, 'regenerate': False,

36
api-examples/api-example-model.py
View file

@ -4,8 +4,9 @@ import requests
HOST = '0.0.0.0:5000' HOST = '0.0.0.0:5000'
def generate(prompt, tokens = 200):
request = { 'prompt': prompt, 'max_new_tokens': tokens } def generate(prompt, tokens=200):
request = {'prompt': prompt, 'max_new_tokens': tokens}
response = requests.post(f'http://{HOST}/api/v1/generate', json=request) response = requests.post(f'http://{HOST}/api/v1/generate', json=request)
if response.status_code == 200: if response.status_code == 200:
@ -75,17 +76,17 @@ def complex_model_load(model):
'rwkv_cuda_on': False, 'rwkv_cuda_on': False,
# b&b 4-bit # b&b 4-bit
#'load_in_4bit': False, # 'load_in_4bit': False,
#'compute_dtype': 'float16', # 'compute_dtype': 'float16',
#'quant_type': 'nf4', # 'quant_type': 'nf4',
#'use_double_quant': False, # 'use_double_quant': False,
#"cpu": false, # "cpu": false,
#"auto_devices": false, # "auto_devices": false,
#"gpu_memory": null, # "gpu_memory": null,
#"cpu_memory": null, # "cpu_memory": null,
#"disk": false, # "disk": false,
#"disk_cache_dir": "cache", # "disk_cache_dir": "cache",
}, },
} }
@ -108,7 +109,7 @@ def complex_model_load(model):
elif '13b' in model: elif '13b' in model:
req['args']['load_in_8bit'] = True # for 24GB req['args']['load_in_8bit'] = True # for 24GB
elif 'ggml' in model: elif 'ggml' in model:
#req['args']['threads'] = 16 # req['args']['threads'] = 16
if '7b' in model: if '7b' in model:
req['args']['n_gpu_layers'] = 100 req['args']['n_gpu_layers'] = 100
elif '13b' in model: elif '13b' in model:
@ -124,7 +125,6 @@ def complex_model_load(model):
else: else:
req['args']['rwkv_strategy'] = 'cuda f16' # 24GB req['args']['rwkv_strategy'] = 'cuda f16' # 24GB
return model_api(req) return model_api(req)
@ -134,7 +134,7 @@ if __name__ == '__main__':
resp = complex_model_load(model) resp = complex_model_load(model)
if 'error' in resp: if 'error' in resp:
print (f"❌ {model} FAIL Error: {resp['error']['message']}") print(f"❌ {model} FAIL Error: {resp['error']['message']}")
continue continue
else: else:
print_basic_model_info(resp) print_basic_model_info(resp)
@ -142,12 +142,12 @@ if __name__ == '__main__':
ans = generate("0,1,1,2,3,5,8,13,", tokens=2) ans = generate("0,1,1,2,3,5,8,13,", tokens=2)
if '21' in ans: if '21' in ans:
print (f"✅ {model} PASS ({ans})") print(f"✅ {model} PASS ({ans})")
else: else:
print (f"❌ {model} FAIL ({ans})") print(f"❌ {model} FAIL ({ans})")
except Exception as e: except Exception as e:
print (f"❌ {model} FAIL Exception: {repr(e)}") print(f"❌ {model} FAIL Exception: {repr(e)}")
# 0,1,1,2,3,5,8,13, is the fibonacci sequence, the next number is 21. # 0,1,1,2,3,5,8,13, is the fibonacci sequence, the next number is 21.

7
extensions/api/util.py
View file

@ -59,7 +59,10 @@ def build_parameters(body, chat=False):
if chat: if chat:
character = body.get('character') character = body.get('character')
instruction_template = body.get('instruction_template') instruction_template = body.get('instruction_template', shared.settings['instruction_template'])
if str(instruction_template) == "None":
instruction_template = "Vicuna-v1.1"
name1, name2, _, greeting, context, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), shared.settings['name2'], instruct=False) name1, name2, _, greeting, context, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), shared.settings['name2'], instruct=False)
name1_instruct, name2_instruct, _, _, context_instruct, turn_template = load_character_memoized(instruction_template, '', '', instruct=True) name1_instruct, name2_instruct, _, _, context_instruct, turn_template = load_character_memoized(instruction_template, '', '', instruct=True)
generate_params.update({ generate_params.update({
@ -72,7 +75,7 @@ def build_parameters(body, chat=False):
'greeting': greeting, 'greeting': greeting,
'name1_instruct': name1_instruct, 'name1_instruct': name1_instruct,
'name2_instruct': name2_instruct, 'name2_instruct': name2_instruct,
'context_instruct': context_instruct, 'context_instruct': body.get('context_instruct', context_instruct),
'turn_template': turn_template, 'turn_template': turn_template,
'chat-instruct_command': str(body.get('chat-instruct_command', shared.settings['chat-instruct_command'])), 'chat-instruct_command': str(body.get('chat-instruct_command', shared.settings['chat-instruct_command'])),
'history': body.get('history', {'internal': [], 'visible': []}) 'history': body.get('history', {'internal': [], 'visible': []})