Add max_tokens_second param (#3533)

extensions/api/util.py

@@ -22,6 +22,7 @@ def build_parameters(body, chat=False):
     generate_params = {
         'max_new_tokens': int(body.get('max_new_tokens', body.get('max_length', 200))),
         'auto_max_new_tokens': bool(body.get('auto_max_new_tokens', False)),
+        'max_tokens_second': int(body.get('max_tokens_second', 0)),
         'do_sample': bool(body.get('do_sample', True)),
         'temperature': float(body.get('temperature', 0.5)),
         'top_p': float(body.get('top_p', 1)),

extensions/openai/defaults.py

@@ -5,6 +5,7 @@ import copy
 default_req_params = {
     'max_new_tokens': 16,  # 'Inf' for chat
     'auto_max_new_tokens': False,
+    'max_tokens_second': 0,
     'temperature': 1.0,
     'top_p': 1.0,
     'top_k': 1,  # choose 20 for chat in absence of another default