Allow API requests to use parameter presets

api-examples/api-example-chat-stream.py

@@ -19,6 +19,7 @@ async def run(user_input, history):
     # Note: the selected defaults change from time to time.
     request = {
         'user_input': user_input,
+        'max_new_tokens': 250,
         'history': history,
         'mode': 'instruct',  # Valid options: 'chat', 'chat-instruct', 'instruct'
         'character': 'Example',
@@ -32,7 +33,9 @@ async def run(user_input, history):
         'chat_generation_attempts': 1,
         'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
 
-        'max_new_tokens': 250,
+        # Generation params. If 'preset' is set to different than 'None', the values
+        # in presets/preset-name.yaml are used instead of the individual numbers.
+        'preset': 'None',  
         'do_sample': True,
         'temperature': 0.7,
         'top_p': 0.1,
@@ -52,6 +55,7 @@ async def run(user_input, history):
         'mirostat_mode': 0,
         'mirostat_tau': 5,
         'mirostat_eta': 0.1,
+
         'seed': -1,
         'add_bos_token': True,
         'truncation_length': 2048,

api-examples/api-example-chat.py

@@ -13,6 +13,7 @@ URI = f'http://{HOST}/api/v1/chat'
 def run(user_input, history):
     request = {
         'user_input': user_input,
+        'max_new_tokens': 250,
         'history': history,
         'mode': 'instruct',  # Valid options: 'chat', 'chat-instruct', 'instruct'
         'character': 'Example',
@@ -26,7 +27,9 @@ def run(user_input, history):
         'chat_generation_attempts': 1,
         'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
 
-        'max_new_tokens': 250,
+        # Generation params. If 'preset' is set to different than 'None', the values
+        # in presets/preset-name.yaml are used instead of the individual numbers.
+        'preset': 'None',
         'do_sample': True,
         'temperature': 0.7,
         'top_p': 0.1,
@@ -46,6 +49,7 @@ def run(user_input, history):
         'mirostat_mode': 0,
         'mirostat_tau': 5,
         'mirostat_eta': 0.1,
+
         'seed': -1,
         'add_bos_token': True,
         'truncation_length': 2048,

api-examples/api-example-stream.py

@@ -20,8 +20,12 @@ async def run(context):
     request = {
         'prompt': context,
         'max_new_tokens': 250,
+
+        # Generation params. If 'preset' is set to different than 'None', the values
+        # in presets/preset-name.yaml are used instead of the individual numbers.
+        'preset': 'None',  
         'do_sample': True,
-        'temperature': 1.3,
+        'temperature': 0.7,
         'top_p': 0.1,
         'typical_p': 1,
         'epsilon_cutoff': 0,  # In units of 1e-4
@@ -39,6 +43,7 @@ async def run(context):
         'mirostat_mode': 0,
         'mirostat_tau': 5,
         'mirostat_eta': 0.1,
+
         'seed': -1,
         'add_bos_token': True,
         'truncation_length': 2048,

api-examples/api-example.py

@@ -12,8 +12,12 @@ def run(prompt):
     request = {
         'prompt': prompt,
         'max_new_tokens': 250,
+
+        # Generation params. If 'preset' is set to different than 'None', the values
+        # in presets/preset-name.yaml are used instead of the individual numbers.
+        'preset': 'None',  
         'do_sample': True,
-        'temperature': 1.3,
+        'temperature': 0.7,
         'top_p': 0.1,
         'typical_p': 1,
         'epsilon_cutoff': 0,  # In units of 1e-4
@@ -31,6 +35,7 @@ def run(prompt):
         'mirostat_mode': 0,
         'mirostat_tau': 5,
         'mirostat_eta': 0.1,
+
         'seed': -1,
         'add_bos_token': True,
         'truncation_length': 2048,