mmarch/text-generation-webui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add Support for Static NTK RoPE scaling for exllama/exllama_hf (#2955)

Browse source
This commit is contained in:
Panchovix 2023-07-04 00:13:16 -04:00 • committed by GitHub
parent 1610d5ffb2
commit 10c8c197bf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 18 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/shared.py
View file

@ -150,6 +150,7 @@ parser.add_argument('--desc_act', action='store_true', help='For models that don
parser.add_argument('--gpu-split', type=str, help="Comma-separated list of VRAM (in GB) to use per GPU device for model layers, e.g. 20,7,7")
parser.add_argument('--max_seq_len', type=int, default=2048, help="Maximum sequence length.")
parser.add_argument('--compress_pos_emb', type=int, default=1, help="Positional embeddings compression factor. Should typically be set to max_seq_len / 2048.")
parser.add_argument('--alpha_value', type=int, default=1, help="Positional embeddings alpha factor for NTK RoPE scaling. Same as above. Use either this or compress_pos_emb, not both.")
# FlexGen
parser.add_argument('--flexgen', action='store_true', help='DEPRECATED')