mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

lint

Browse source
This commit is contained in:
oobabooga 2023-07-12 11:33:25 -07:00
parent 9b55d3a9f9
commit e202190c4f
24 changed files with 146 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

11
extensions/openai/tokens.py
View file

@ -1,6 +1,7 @@
from extensions.openai.utils import float_list_to_base64
from modules.text_generation import encode, decode
def token_count(prompt):
tokens = encode(prompt)[0]
@ -11,8 +12,8 @@ def token_count(prompt):
}
def token_encode(input, encoding_format = ''):
#if isinstance(input, list):
def token_encode(input, encoding_format=''):
# if isinstance(input, list):
tokens = encode(input)[0]
return {
@ -25,9 +26,9 @@ def token_encode(input, encoding_format = ''):
def token_decode(tokens, encoding_format):
#if isinstance(input, list):
# if encoding_format == "base64":
# tokens = base64_to_float_list(tokens)
# if isinstance(input, list):
# if encoding_format == "base64":
# tokens = base64_to_float_list(tokens)
output = decode(tokens)[0]
return {