Add max_tokens_second param (#3533)

2023-08-29 17:44:31 -03:00 · 2023-08-29 17:44:31 -03:00 · cec8db52e5
commit cec8db52e5
parent fe1f7c6513
11 changed files with 24 additions and 3 deletions
--- a/api-examples/api-example-stream.py
+++ b/api-examples/api-example-stream.py
@ -21,6 +21,7 @@ async def run(context):
        'prompt': context,
        'max_new_tokens': 250,
        'auto_max_new_tokens': False,
+        'max_tokens_second': 0,

        # Generation params. If 'preset' is set to different than 'None', the values
        # in presets/preset-name.yaml are used instead of the individual numbers.