From 6170b5ba31926ee32200a062707269cd8fbd923d Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 17 Aug 2023 21:39:26 -0700 Subject: [PATCH 1/5] Bump llama-cpp-python --- README.md | 1 + modules/llamacpp_hf.py | 7 +++++++ modules/llamacpp_model.py | 7 +++++++ modules/loaders.py | 2 ++ modules/shared.py | 1 + modules/ui.py | 1 + requirements.txt | 4 ++-- 7 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 18f6d73..4607ab5 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,7 @@ Optionally, you can use the following command-line flags: | `--no-mmap` | Prevent mmap from being used. | | `--mlock` | Force the system to keep the model in RAM. | | `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. | +| `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 | | `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). | | `--n_gqa N_GQA` | grouped-query attention. Must be 8 for llama-2 70b. | | `--rms_norm_eps RMS_NORM_EPS` | 5e-6 is a good value for llama-2 models. | diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py index fa0554c..7deae98 100644 --- a/modules/llamacpp_hf.py +++ b/modules/llamacpp_hf.py @@ -102,6 +102,12 @@ class LlamacppHF(PreTrainedModel): model_file = list(path.glob('*ggml*.bin'))[0] logger.info(f"llama.cpp weights detected: {model_file}\n") + + if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '': + tensor_split_list = None + else: + tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")] + params = { 'model_path': str(model_file), 'n_ctx': shared.args.n_ctx, @@ -113,6 +119,7 @@ class LlamacppHF(PreTrainedModel): 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers, 'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.), + 'tensor_split': tensor_split_list, 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'n_gqa': shared.args.n_gqa or None, 'rms_norm_eps': shared.args.rms_norm_eps or None, diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index f7f4cc9..0e635da 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -55,6 +55,12 @@ class LlamaCppModel: cache_capacity = int(shared.args.cache_capacity) logger.info("Cache capacity is " + str(cache_capacity) + " bytes") + + if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '': + tensor_split_list = None + else: + tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")] + params = { 'model_path': str(path), 'n_ctx': shared.args.n_ctx, @@ -66,6 +72,7 @@ class LlamaCppModel: 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers, 'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.), + 'tensor_split': tensor_split_list, 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'n_gqa': shared.args.n_gqa or None, 'rms_norm_eps': shared.args.rms_norm_eps or None, diff --git a/modules/loaders.py b/modules/loaders.py index 08a11ac..d49f7c0 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -67,6 +67,7 @@ loaders_and_params = OrderedDict({ 'n_gqa', 'rms_norm_eps', 'n_gpu_layers', + 'tensor_split', 'n_batch', 'threads', 'no_mmap', @@ -82,6 +83,7 @@ loaders_and_params = OrderedDict({ 'n_gqa', 'rms_norm_eps', 'n_gpu_layers', + 'tensor_split', 'n_batch', 'threads', 'no_mmap', diff --git a/modules/shared.py b/modules/shared.py index 88aa8cf..23eb398 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -121,6 +121,7 @@ parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode') parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.') parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.') +parser.add_argument('--tensor_split', type=str, default=None, help="Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17") parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.') parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default 0 (random)') parser.add_argument('--n_gqa', type=int, default=0, help='grouped-query attention. Must be 8 for llama-2 70b.') diff --git a/modules/ui.py b/modules/ui.py index a99af37..ea1ca74 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -69,6 +69,7 @@ def list_model_elements(): 'low_vram', 'mlock', 'n_gpu_layers', + 'tensor_split', 'n_ctx', 'n_gqa', 'rms_norm_eps', diff --git a/requirements.txt b/requirements.txt index 888dda8..9a62354 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,8 +31,8 @@ https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117 https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" # llama-cpp-python without GPU support -llama-cpp-python==0.1.77; platform_system != "Windows" -https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.77/llama_cpp_python-0.1.77-cp310-cp310-win_amd64.whl; platform_system == "Windows" +llama-cpp-python==0.1.78; platform_system != "Windows" +https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.78/llama_cpp_python-0.1.78-cp310-cp310-win_amd64.whl; platform_system == "Windows" # llama-cpp-python with CUDA support https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" From 28cf5862af3ad66743eaad7a8bf649fbf34f6c51 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 18 Aug 2023 06:26:48 -0700 Subject: [PATCH 2/5] Add UI element for tensor_split --- modules/ui_model_menu.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 3059f61..2150753 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -33,6 +33,7 @@ def create_ui(): default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i))) else: default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)) * 1000) + while len(default_gpu_mem) < len(total_mem): default_gpu_mem.append(0) @@ -109,6 +110,7 @@ def create_ui(): shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap) shared.gradio['low_vram'] = gr.Checkbox(label="low-vram", value=shared.args.low_vram) shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) + shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17') shared.gradio['llama_cpp_seed'] = gr.Number(label='Seed (0 for random)', value=shared.args.llama_cpp_seed) shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Make sure to inspect the .py files inside the model folder before loading it with this option enabled.') shared.gradio['gptq_for_llama_info'] = gr.Markdown('GPTQ-for-LLaMa support is currently only kept for compatibility with older GPUs. AutoGPTQ or ExLlama is preferred when compatible. GPTQ-for-LLaMa is installed by default with the webui on supported systems. Otherwise, it has to be installed manually following the instructions here: [instructions](https://github.com/oobabooga/text-generation-webui/blob/main/docs/GPTQ-models-(4-bit-mode).md#installation-1).') From 4ec42679e31924809d7734ea40da2f836bad4010 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 18 Aug 2023 07:58:20 -0700 Subject: [PATCH 3/5] Add --mul_mat_q param --- README.md | 1 + modules/llamacpp_hf.py | 1 + modules/llamacpp_model.py | 1 + modules/shared.py | 1 + modules/ui.py | 1 + modules/ui_model_menu.py | 1 + 6 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 4607ab5..5b1e95c 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ Optionally, you can use the following command-line flags: |-------------|-------------| | `--no-mmap` | Prevent mmap from being used. | | `--mlock` | Force the system to keep the model in RAM. | +| `--mul_mat_q` | Activate new mulmat kernels. | | `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. | | `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 | | `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). | diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py index 7deae98..10c3011 100644 --- a/modules/llamacpp_hf.py +++ b/modules/llamacpp_hf.py @@ -116,6 +116,7 @@ class LlamacppHF(PreTrainedModel): 'n_batch': shared.args.n_batch, 'use_mmap': not shared.args.no_mmap, 'use_mlock': shared.args.mlock, + 'mul_mat_q': shared.args.mul_mat_q, 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers, 'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.), diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index 0e635da..28a38de 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -69,6 +69,7 @@ class LlamaCppModel: 'n_batch': shared.args.n_batch, 'use_mmap': not shared.args.no_mmap, 'use_mlock': shared.args.mlock, + 'mul_mat_q': shared.args.mul_mat_q, 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers, 'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.), diff --git a/modules/shared.py b/modules/shared.py index 23eb398..385b99d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -119,6 +119,7 @@ parser.add_argument('--n_batch', type=int, default=512, help='Maximum number of parser.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.') parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode') parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') +parser.add_argument('--mul_mat_q', action='store_true', help='Activate new mulmat kernels.') parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.') parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.') parser.add_argument('--tensor_split', type=str, default=None, help="Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17") diff --git a/modules/ui.py b/modules/ui.py index ea1ca74..15f24d8 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -68,6 +68,7 @@ def list_model_elements(): 'no_mmap', 'low_vram', 'mlock', + 'mul_mat_q', 'n_gpu_layers', 'tensor_split', 'n_ctx', diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 2150753..8e24ebd 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -110,6 +110,7 @@ def create_ui(): shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap) shared.gradio['low_vram'] = gr.Checkbox(label="low-vram", value=shared.args.low_vram) shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) + shared.gradio['mul_mat_q'] = gr.Checkbox(label="mul_mat_q", value=shared.args.mul_mat_q) shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17') shared.gradio['llama_cpp_seed'] = gr.Number(label='Seed (0 for random)', value=shared.args.llama_cpp_seed) shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Make sure to inspect the .py files inside the model folder before loading it with this option enabled.') From d8f660e58684e35a0ca44ca67b601e02dfc26de5 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 18 Aug 2023 08:00:22 -0700 Subject: [PATCH 4/5] Add to modules/loaders.py --- modules/loaders.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/loaders.py b/modules/loaders.py index d49f7c0..7444555 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -73,6 +73,7 @@ loaders_and_params = OrderedDict({ 'no_mmap', 'low_vram', 'mlock', + 'mul_mat_q', 'llama_cpp_seed', 'alpha_value', 'compress_pos_emb', @@ -89,6 +90,7 @@ loaders_and_params = OrderedDict({ 'no_mmap', 'low_vram', 'mlock', + 'mul_mat_q', 'alpha_value', 'compress_pos_emb', 'cpu', From 1a71ab58a9302085bcf342986abb50e5f5a25f73 Mon Sep 17 00:00:00 2001 From: jllllll <3887729+jllllll@users.noreply.github.com> Date: Fri, 18 Aug 2023 10:04:01 -0500 Subject: [PATCH 5/5] Bump llama_cpp_python_cuda to 0.1.78 (#3614) --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9a62354..9cac54d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,8 +34,8 @@ https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117 llama-cpp-python==0.1.78; platform_system != "Windows" https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.78/llama_cpp_python-0.1.78-cp310-cp310-win_amd64.whl; platform_system == "Windows" # llama-cpp-python with CUDA support -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.78+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" +https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.78+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" # GPTQ-for-LLaMa https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.0/gptq_for_llama-0.1.0+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"