From 83640d6f43c80906492109af521e0ddc6d43a5cc Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 26 Aug 2023 01:06:59 -0700
Subject: [PATCH 1/7] Replace ggml occurences with gguf

---
 README.md                  |  4 ++--
 download-model.py          | 12 ++++++------
 modules/llamacpp_hf.py     |  2 +-
 modules/models.py          |  2 +-
 modules/models_settings.py |  4 ++--
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 822deba..f36a8b9 100644
--- a/README.md
+++ b/README.md
@@ -156,7 +156,7 @@ text-generation-webui
 
 In the "Model" tab of the UI, those models can be automatically downloaded from Hugging Face. You can also download them via the command-line with `python download-model.py organization/model`.
 
-* GGML models are a single file and should be placed directly into `models`. Example:
+* GGUF models are a single file and should be placed directly into `models`. Example:
 
 ```
 text-generation-webui
@@ -258,7 +258,7 @@ Optionally, you can use the following command-line flags:
 | `--quant_type QUANT_TYPE`                   | quant_type for 4-bit. Valid options: nf4, fp4. |
 | `--use_double_quant`                        | use_double_quant for 4-bit. |
 
-#### GGML (for llama.cpp and ctransformers)
+#### GGUF (for llama.cpp and ctransformers)
 
 | Flag        | Description |
 |-------------|-------------|
diff --git a/download-model.py b/download-model.py
index a65f82c..be8d59f 100644
--- a/download-model.py
+++ b/download-model.py
@@ -57,7 +57,7 @@ class ModelDownloader:
         classifications = []
         has_pytorch = False
         has_pt = False
-        # has_ggml = False
+        # has_gguf = False
         has_safetensors = False
         is_lora = False
         while True:
@@ -78,10 +78,10 @@ class ModelDownloader:
                 is_pytorch = re.match(r"(pytorch|adapter|gptq)_model.*\.bin", fname)
                 is_safetensors = re.match(r".*\.safetensors", fname)
                 is_pt = re.match(r".*\.pt", fname)
-                is_ggml = re.match(r".*ggml.*\.bin", fname)
+                is_gguf = re.match(r'.*\.gguf', fname)
                 is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname)
                 is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer
-                if any((is_pytorch, is_safetensors, is_pt, is_ggml, is_tokenizer, is_text)):
+                if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)):
                     if 'lfs' in dict[i]:
                         sha256.append([fname, dict[i]['lfs']['oid']])
 
@@ -101,9 +101,9 @@ class ModelDownloader:
                         elif is_pt:
                             has_pt = True
                             classifications.append('pt')
-                        elif is_ggml:
-                            # has_ggml = True
-                            classifications.append('ggml')
+                        elif is_gguf:
+                            # has_gguf = True
+                            classifications.append('gguf')
 
             cursor = base64.b64encode(f'{{"file_name":"{dict[-1]["path"]}"}}'.encode()) + b':50'
             cursor = base64.b64encode(cursor)
diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py
index 4d42394..0608cb0 100644
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@@ -165,7 +165,7 @@ class LlamacppHF(PreTrainedModel):
         if path.is_file():
             model_file = path
         else:
-            model_file = list(path.glob('*ggml*.bin'))[0]
+            model_file = list(path.glob('*.gguf*'))[0]
 
         logger.info(f"llama.cpp weights detected: {model_file}\n")
 
diff --git a/modules/models.py b/modules/models.py
index ea9cc52..5268a2f 100644
--- a/modules/models.py
+++ b/modules/models.py
@@ -241,7 +241,7 @@ def llamacpp_loader(model_name):
     if path.is_file():
         model_file = path
     else:
-        model_file = list(Path(f'{shared.args.model_dir}/{model_name}').glob('*ggml*.bin'))[0]
+        model_file = list(Path(f'{shared.args.model_dir}/{model_name}').glob('*.gguf*'))[0]
 
     logger.info(f"llama.cpp weights detected: {model_file}")
     model, tokenizer = LlamaCppModel.from_pretrained(model_file)
diff --git a/modules/models_settings.py b/modules/models_settings.py
index 5efde34..2ed658b 100644
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -24,9 +24,9 @@ def infer_loader(model_name):
         loader = None
     elif Path(f'{shared.args.model_dir}/{model_name}/quantize_config.json').exists() or ('wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0):
         loader = 'AutoGPTQ'
-    elif len(list(path_to_model.glob('*ggml*.bin'))) > 0:
+    elif len(list(path_to_model.glob('*.gguf*'))) > 0:
         loader = 'llama.cpp'
-    elif re.match(r'.*ggml.*\.bin', model_name.lower()):
+    elif re.match(r'.*\.gguf', model_name.lower()):
         loader = 'llama.cpp'
     elif re.match(r'.*rwkv.*\.pth', model_name.lower()):
         loader = 'RWKV'

From 6e6431e73ff006ad272f0ea735e86e4577af1002 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 26 Aug 2023 01:07:28 -0700
Subject: [PATCH 2/7] Update requirements.txt

---
 requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 90c7969..a28d87e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -30,11 +30,11 @@ https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117
 https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
 
 # llama-cpp-python without GPU support
-llama-cpp-python==0.1.78; platform_system != "Windows"
-https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.78/llama_cpp_python-0.1.78-cp310-cp310-win_amd64.whl; platform_system == "Windows"
+llama-cpp-python==0.1.79; platform_system != "Windows"
+https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.79/llama_cpp_python-0.1.79-cp310-cp310-win_amd64.whl; platform_system == "Windows"
 # llama-cpp-python with CUDA support
-https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.78+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
-https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.78+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.79+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
+https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.79+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
 
 # GPTQ-for-LLaMa
 https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.0/gptq_for_llama-0.1.0+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"

From 4a999e3bcd6f44e78eda3d448f4e36884aaf8966 Mon Sep 17 00:00:00 2001
From: jllllll <3887729+jllllll@users.noreply.github.com>
Date: Sat, 26 Aug 2023 09:15:11 -0500
Subject: [PATCH 3/7] Use separate llama-cpp-python packages for GGML support

---
 download-model.py          | 15 +++++++++++++--
 modules/llamacpp_hf.py     | 27 +++++++++++++++++++++------
 modules/llamacpp_model.py  | 29 +++++++++++++++++++++++------
 modules/models.py          |  2 +-
 modules/models_settings.py |  4 ++--
 modules/utils.py           |  9 +++++++++
 requirements.txt           |  5 +++++
 7 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/download-model.py b/download-model.py
index be8d59f..3bb4a39 100644
--- a/download-model.py
+++ b/download-model.py
@@ -57,7 +57,8 @@ class ModelDownloader:
         classifications = []
         has_pytorch = False
         has_pt = False
-        # has_gguf = False
+        has_gguf = False
+        has_ggml = False
         has_safetensors = False
         is_lora = False
         while True:
@@ -79,6 +80,7 @@ class ModelDownloader:
                 is_safetensors = re.match(r".*\.safetensors", fname)
                 is_pt = re.match(r".*\.pt", fname)
                 is_gguf = re.match(r'.*\.gguf', fname)
+                is_ggml = re.match(r".*ggml.*\.bin", fname)
                 is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname)
                 is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer
                 if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)):
@@ -102,8 +104,11 @@ class ModelDownloader:
                             has_pt = True
                             classifications.append('pt')
                         elif is_gguf:
-                            # has_gguf = True
+                            has_gguf = True
                             classifications.append('gguf')
+                        elif is_ggml:
+                            has_ggml = True
+                            classifications.append('ggml')
 
             cursor = base64.b64encode(f'{{"file_name":"{dict[-1]["path"]}"}}'.encode()) + b':50'
             cursor = base64.b64encode(cursor)
@@ -115,6 +120,12 @@ class ModelDownloader:
                 if classifications[i] in ['pytorch', 'pt']:
                     links.pop(i)
 
+        # If both GGML and GGUF are available, download GGUF only
+        if has_ggml and has_gguf:
+            for i in range(len(classifications) - 1, -1, -1):
+                if classifications[i] == 'ggml':
+                    links.pop(i)
+
         return links, sha256, is_lora
 
     def get_output_folder(self, model, branch, is_lora, base_folder=None):
diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py
index 0608cb0..bcb537f 100644
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@@ -9,23 +9,38 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
 
 from modules import RoPE, shared
 from modules.logging_colors import logger
+from modules.utils import is_gguf
 
 import llama_cpp
 
+try:
+    import llama_cpp_ggml
+except:
+    llama_cpp_ggml = llama_cpp
+
 if torch.cuda.is_available() and not torch.version.hip:
     try:
         import llama_cpp_cuda
     except:
         llama_cpp_cuda = None
+    try:
+        import llama_cpp_ggml_cuda
+    except:
+        llama_cpp_ggml_cuda = llama_cpp_cuda
 else:
     llama_cpp_cuda = None
+    llama_cpp_ggml_cuda = None
 
 
-def llama_cpp_lib():
-    if shared.args.cpu or llama_cpp_cuda is None:
-        return llama_cpp
+def llama_cpp_lib(model_file: Union[str, Path] = None):
+    if model_file is not None:
+        gguf_model = is_gguf(model_file)
     else:
-        return llama_cpp_cuda
+        gguf_model = True
+    if shared.args.cpu or llama_cpp_cuda is None:
+        return llama_cpp if gguf_model else llama_cpp_ggml
+    else:
+        return llama_cpp_cuda if gguf_model else llama_cpp_ggml_cuda
 
 
 class LlamacppHF(PreTrainedModel):
@@ -165,7 +180,7 @@ class LlamacppHF(PreTrainedModel):
         if path.is_file():
             model_file = path
         else:
-            model_file = list(path.glob('*.gguf*'))[0]
+            model_file = (list(path.glob('*.gguf*')) + list(path.glob('*ggml*.bin')))[0]
 
         logger.info(f"llama.cpp weights detected: {model_file}\n")
 
@@ -193,7 +208,7 @@ class LlamacppHF(PreTrainedModel):
             'logits_all': True,
         }
 
-        Llama = llama_cpp_lib().Llama
+        Llama = llama_cpp_lib(model_file).Llama
         model = Llama(**params)
 
         return LlamacppHF(model)
diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py
index d2893b0..c3c4154 100644
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@@ -1,5 +1,7 @@
 import re
 from functools import partial
+from pathlib import Path
+from typing import Union
 
 import torch
 
@@ -7,23 +9,38 @@ from modules import RoPE, shared
 from modules.callbacks import Iteratorize
 from modules.logging_colors import logger
 from modules.text_generation import get_max_prompt_length
+from modules.utils import is_gguf
 
 import llama_cpp
 
+try:
+    import llama_cpp_ggml
+except:
+    llama_cpp_ggml = llama_cpp
+
 if torch.cuda.is_available() and not torch.version.hip:
     try:
         import llama_cpp_cuda
     except:
         llama_cpp_cuda = None
+    try:
+        import llama_cpp_ggml_cuda
+    except:
+        llama_cpp_ggml_cuda = llama_cpp_cuda
 else:
     llama_cpp_cuda = None
+    llama_cpp_ggml_cuda = None
 
 
-def llama_cpp_lib():
-    if shared.args.cpu or llama_cpp_cuda is None:
-        return llama_cpp
+def llama_cpp_lib(model_file: Union[str, Path] = None):
+    if model_file is not None:
+        gguf_model = is_gguf(model_file)
     else:
-        return llama_cpp_cuda
+        gguf_model = True
+    if shared.args.cpu or llama_cpp_cuda is None:
+        return llama_cpp if gguf_model else llama_cpp_ggml
+    else:
+        return llama_cpp_cuda if gguf_model else llama_cpp_ggml_cuda
 
 
 def ban_eos_logits_processor(eos_token, input_ids, logits):
@@ -41,8 +58,8 @@ class LlamaCppModel:
     @classmethod
     def from_pretrained(self, path):
 
-        Llama = llama_cpp_lib().Llama
-        LlamaCache = llama_cpp_lib().LlamaCache
+        Llama = llama_cpp_lib(str(path)).Llama
+        LlamaCache = llama_cpp_lib(str(path)).LlamaCache
 
         result = self()
         cache_capacity = 0
diff --git a/modules/models.py b/modules/models.py
index 5268a2f..3025fe3 100644
--- a/modules/models.py
+++ b/modules/models.py
@@ -241,7 +241,7 @@ def llamacpp_loader(model_name):
     if path.is_file():
         model_file = path
     else:
-        model_file = list(Path(f'{shared.args.model_dir}/{model_name}').glob('*.gguf*'))[0]
+        model_file = (list(Path(f'{shared.args.model_dir}/{model_name}').glob('*.gguf*')) + list(Path(f'{shared.args.model_dir}/{model_name}').glob('*ggml*.bin')))[0]
 
     logger.info(f"llama.cpp weights detected: {model_file}")
     model, tokenizer = LlamaCppModel.from_pretrained(model_file)
diff --git a/modules/models_settings.py b/modules/models_settings.py
index 2ed658b..c55b1e8 100644
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -24,9 +24,9 @@ def infer_loader(model_name):
         loader = None
     elif Path(f'{shared.args.model_dir}/{model_name}/quantize_config.json').exists() or ('wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0):
         loader = 'AutoGPTQ'
-    elif len(list(path_to_model.glob('*.gguf*'))) > 0:
+    elif len(list(path_to_model.glob('*.gguf*')) + list(path_to_model.glob('*ggml*.bin'))) > 0:
         loader = 'llama.cpp'
-    elif re.match(r'.*\.gguf', model_name.lower()):
+    elif re.match(r'.*\.gguf|.*ggml.*\.bin', model_name.lower()):
         loader = 'llama.cpp'
     elif re.match(r'.*rwkv.*\.pth', model_name.lower()):
         loader = 'RWKV'
diff --git a/modules/utils.py b/modules/utils.py
index 0a7edff..3862817 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -2,6 +2,7 @@ import os
 import re
 from datetime import datetime
 from pathlib import Path
+from typing import Union
 
 from modules import shared
 from modules.logging_colors import logger
@@ -124,3 +125,11 @@ def get_datasets(path: str, ext: str):
 
 def get_available_chat_styles():
     return sorted(set(('-'.join(k.stem.split('-')[1:]) for k in Path('css').glob('chat_style*.css'))), key=natural_keys)
+
+# Determines if a llama.cpp model is in GGUF format
+# Copied from ctransformers utils.py
+def is_gguf(path: Union[str, Path]) -> bool:
+    path = str(Path(path).resolve())
+    with open(path, "rb") as f:
+        magic = f.read(4)
+    return magic == "GGUF".encode()
diff --git a/requirements.txt b/requirements.txt
index a28d87e..0c6aeb1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -35,6 +35,11 @@ https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.79/llama_cpp_
 # llama-cpp-python with CUDA support
 https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.79+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
 https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.79+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+# llama-cpp-python with GGML support
+https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python_ggml-0.1.78+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows"
+https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python_ggml-0.1.78+cpuavx2-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_ggml_cuda-0.1.78+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
+https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_ggml_cuda-0.1.78+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
 
 # GPTQ-for-LLaMa
 https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.0/gptq_for_llama-0.1.0+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"

From 4d61a7d9daa4aa565f840d0fff31c2b4a0247298 Mon Sep 17 00:00:00 2001
From: jllllll <3887729+jllllll@users.noreply.github.com>
Date: Sat, 26 Aug 2023 14:07:46 -0500
Subject: [PATCH 4/7] Account for deprecated GGML parameters

---
 modules/llamacpp_hf.py    | 9 +++++++--
 modules/llamacpp_model.py | 9 +++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py
index bcb537f..a2dcb34 100644
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@@ -203,10 +203,15 @@ class LlamacppHF(PreTrainedModel):
             'rope_freq_base': RoPE.get_rope_freq_base(shared.args.alpha_value, shared.args.rope_freq_base),
             'tensor_split': tensor_split_list,
             'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
-            'n_gqa': shared.args.n_gqa or None,
-            'rms_norm_eps': shared.args.rms_norm_eps or None,
             'logits_all': True,
         }
+        
+        if not is_gguf(model_file):
+            ggml_params = {
+                'n_gqa': shared.args.n_gqa or None,
+                'rms_norm_eps': shared.args.rms_norm_eps or None,
+            }
+            params = params | ggml_params
 
         Llama = llama_cpp_lib(model_file).Llama
         model = Llama(**params)
diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py
index c3c4154..4908ecb 100644
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@@ -92,9 +92,14 @@ class LlamaCppModel:
             'rope_freq_base': RoPE.get_rope_freq_base(shared.args.alpha_value, shared.args.rope_freq_base),
             'tensor_split': tensor_split_list,
             'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
-            'n_gqa': shared.args.n_gqa or None,
-            'rms_norm_eps': shared.args.rms_norm_eps or None,
         }
+        
+        if not is_gguf(str(path)):
+            ggml_params = {
+                'n_gqa': shared.args.n_gqa or None,
+                'rms_norm_eps': shared.args.rms_norm_eps or None,
+            }
+            params = params | ggml_params
 
         result.model = Llama(**params)
         if cache_capacity > 0:

From 7f5370a2726ef54f4ef9c350b986297f7809c226 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 26 Aug 2023 22:11:07 -0700
Subject: [PATCH 5/7] Minor fixes/cosmetics

---
 README.md                 | 4 ++--
 download-model.py         | 2 +-
 modules/llamacpp_hf.py    | 3 ++-
 modules/llamacpp_model.py | 9 +++++----
 modules/utils.py          | 8 ++++++--
 requirements.txt          | 7 +++++++
 6 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index f36a8b9..f527cd5 100644
--- a/README.md
+++ b/README.md
@@ -156,7 +156,7 @@ text-generation-webui
 
 In the "Model" tab of the UI, those models can be automatically downloaded from Hugging Face. You can also download them via the command-line with `python download-model.py organization/model`.
 
-* GGUF models are a single file and should be placed directly into `models`. Example:
+* GGML/GGUF models are a single file and should be placed directly into `models`. Example:
 
 ```
 text-generation-webui
@@ -258,7 +258,7 @@ Optionally, you can use the following command-line flags:
 | `--quant_type QUANT_TYPE`                   | quant_type for 4-bit. Valid options: nf4, fp4. |
 | `--use_double_quant`                        | use_double_quant for 4-bit. |
 
-#### GGUF (for llama.cpp and ctransformers)
+#### GGML/GGUF (for llama.cpp and ctransformers)
 
 | Flag        | Description |
 |-------------|-------------|
diff --git a/download-model.py b/download-model.py
index 3bb4a39..b36865d 100644
--- a/download-model.py
+++ b/download-model.py
@@ -83,7 +83,7 @@ class ModelDownloader:
                 is_ggml = re.match(r".*ggml.*\.bin", fname)
                 is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname)
                 is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer
-                if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)):
+                if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_ggml, is_tokenizer, is_text)):
                     if 'lfs' in dict[i]:
                         sha256.append([fname, dict[i]['lfs']['oid']])
 
diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py
index a2dcb34..ce8c6d1 100644
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@@ -37,6 +37,7 @@ def llama_cpp_lib(model_file: Union[str, Path] = None):
         gguf_model = is_gguf(model_file)
     else:
         gguf_model = True
+
     if shared.args.cpu or llama_cpp_cuda is None:
         return llama_cpp if gguf_model else llama_cpp_ggml
     else:
@@ -205,7 +206,7 @@ class LlamacppHF(PreTrainedModel):
             'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
             'logits_all': True,
         }
-        
+
         if not is_gguf(model_file):
             ggml_params = {
                 'n_gqa': shared.args.n_gqa or None,
diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py
index 4908ecb..12aa3a4 100644
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@@ -37,6 +37,7 @@ def llama_cpp_lib(model_file: Union[str, Path] = None):
         gguf_model = is_gguf(model_file)
     else:
         gguf_model = True
+
     if shared.args.cpu or llama_cpp_cuda is None:
         return llama_cpp if gguf_model else llama_cpp_ggml
     else:
@@ -58,8 +59,8 @@ class LlamaCppModel:
     @classmethod
     def from_pretrained(self, path):
 
-        Llama = llama_cpp_lib(str(path)).Llama
-        LlamaCache = llama_cpp_lib(str(path)).LlamaCache
+        Llama = llama_cpp_lib(path).Llama
+        LlamaCache = llama_cpp_lib(path).LlamaCache
 
         result = self()
         cache_capacity = 0
@@ -93,8 +94,8 @@ class LlamaCppModel:
             'tensor_split': tensor_split_list,
             'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
         }
-        
-        if not is_gguf(str(path)):
+
+        if not is_gguf(path):
             ggml_params = {
                 'n_gqa': shared.args.n_gqa or None,
                 'rms_norm_eps': shared.args.rms_norm_eps or None,
diff --git a/modules/utils.py b/modules/utils.py
index 3862817..15dbd9d 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -126,10 +126,14 @@ def get_datasets(path: str, ext: str):
 def get_available_chat_styles():
     return sorted(set(('-'.join(k.stem.split('-')[1:]) for k in Path('css').glob('chat_style*.css'))), key=natural_keys)
 
-# Determines if a llama.cpp model is in GGUF format
-# Copied from ctransformers utils.py
+
 def is_gguf(path: Union[str, Path]) -> bool:
+    '''
+    Determines if a llama.cpp model is in GGUF format
+    Copied from ctransformers utils.py
+    '''
     path = str(Path(path).resolve())
     with open(path, "rb") as f:
         magic = f.read(4)
+
     return magic == "GGUF".encode()
diff --git a/requirements.txt b/requirements.txt
index 0c6aeb1..7311370 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,19 +22,26 @@ tensorboard
 tqdm
 wandb
 
+# bitsandbytes
 bitsandbytes==0.41.1; platform_system != "Windows"
 https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows"
+
+# AutoGPTQ
 https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.4.2/auto_gptq-0.4.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
 https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.4.2/auto_gptq-0.4.2+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+
+# ExLlama
 https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
 https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
 
 # llama-cpp-python without GPU support
 llama-cpp-python==0.1.79; platform_system != "Windows"
 https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.79/llama_cpp_python-0.1.79-cp310-cp310-win_amd64.whl; platform_system == "Windows"
+
 # llama-cpp-python with CUDA support
 https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.79+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
 https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.79+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+
 # llama-cpp-python with GGML support
 https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python_ggml-0.1.78+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows"
 https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python_ggml-0.1.78+cpuavx2-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"

From 8aeae3b3f40f293c608283ddb34d33c2b5a4113c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 26 Aug 2023 22:15:06 -0700
Subject: [PATCH 6/7] Fix llamacpp_HF loading

---
 modules/llamacpp_hf.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py
index ce8c6d1..918ce7f 100644
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@@ -45,7 +45,7 @@ def llama_cpp_lib(model_file: Union[str, Path] = None):
 
 
 class LlamacppHF(PreTrainedModel):
-    def __init__(self, model):
+    def __init__(self, model, path):
         super().__init__(PretrainedConfig())
         self.model = model
         self.generation_config = GenerationConfig()
@@ -64,7 +64,7 @@ class LlamacppHF(PreTrainedModel):
                 'n_tokens': self.model.n_tokens,
                 'input_ids': self.model.input_ids.copy(),
                 'scores': self.model.scores.copy(),
-                'ctx': llama_cpp_lib().llama_new_context_with_model(model.model, model.params)
+                'ctx': llama_cpp_lib(path).llama_new_context_with_model(model.model, model.params)
             }
 
     def _validate_model_class(self):
@@ -217,4 +217,4 @@ class LlamacppHF(PreTrainedModel):
         Llama = llama_cpp_lib(model_file).Llama
         model = Llama(**params)
 
-        return LlamacppHF(model)
+        return LlamacppHF(model, model_file)

From 3361728da154ab6ed9b472b529bc06a0a01e633c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 26 Aug 2023 22:24:44 -0700
Subject: [PATCH 7/7] Change some comments

---
 README.md                | 16 ++++++++--------
 modules/ui_model_menu.py |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index f527cd5..77d67fa 100644
--- a/README.md
+++ b/README.md
@@ -269,16 +269,16 @@ Optionally, you can use the following command-line flags:
 
 #### llama.cpp
 
-| Flag        | Description |
-|-------------|-------------|
-| `--no-mmap` | Prevent mmap from being used. |
-| `--mlock`   | Force the system to keep the model in RAM. |
+| Flag          | Description |
+|---------------|---------------|
+| `--no-mmap`   | Prevent mmap from being used. |
+| `--mlock`     | Force the system to keep the model in RAM. |
 | `--mul_mat_q` | Activate new mulmat kernels. |
 | `--cache-capacity CACHE_CAPACITY`   | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. |
-| `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 |
-| `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). |
-| `--n_gqa N_GQA`         | grouped-query attention. Must be 8 for llama-2 70b. |
-| `--rms_norm_eps RMS_NORM_EPS`  | 5e-6 is a good value for llama-2 models. |
+| `--tensor_split TENSOR_SPLIT`  | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 |
+| `--llama_cpp_seed SEED`        | Seed for llama-cpp models. Default 0 (random). |
+| `--n_gqa N_GQA`                | GGML only (not used by GGUF): Grouped-Query Attention. Must be 8 for llama-2 70b. |
+| `--rms_norm_eps RMS_NORM_EPS`  | GGML only (not used by GGUF): 5e-6 is a good value for llama-2 models. |
 | `--cpu`                        | Use the CPU version of llama-cpp-python instead of the GPU-accelerated version. |
 |`--cfg-cache`                   | llamacpp_HF: Create an additional cache for CFG negative prompts. |
 
diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py
index fde2fb3..5c945de 100644
--- a/modules/ui_model_menu.py
+++ b/modules/ui_model_menu.py
@@ -80,8 +80,8 @@ def create_ui():
                             shared.gradio['n_ctx'] = gr.Slider(minimum=0, maximum=16384, step=256, label="n_ctx", value=shared.args.n_ctx)
                             shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=32, value=shared.args.threads)
                             shared.gradio['n_batch'] = gr.Slider(label="n_batch", minimum=1, maximum=2048, value=shared.args.n_batch)
-                            shared.gradio['n_gqa'] = gr.Slider(minimum=0, maximum=16, step=1, label="n_gqa", value=shared.args.n_gqa, info='grouped-query attention. Must be 8 for llama-2 70b.')
-                            shared.gradio['rms_norm_eps'] = gr.Slider(minimum=0, maximum=1e-5, step=1e-6, label="rms_norm_eps", value=shared.args.rms_norm_eps, info='5e-6 is a good value for llama-2 models.')
+                            shared.gradio['n_gqa'] = gr.Slider(minimum=0, maximum=16, step=1, label="n_gqa", value=shared.args.n_gqa, info='GGML only (not used by GGUF): Grouped-Query Attention. Must be 8 for llama-2 70b.')
+                            shared.gradio['rms_norm_eps'] = gr.Slider(minimum=0, maximum=1e-5, step=1e-6, label="rms_norm_eps", value=shared.args.rms_norm_eps, info='GGML only (not used by GGUF): 5e-6 is a good value for llama-2 models.')
 
                             shared.gradio['wbits'] = gr.Dropdown(label="wbits", choices=["None", 1, 2, 3, 4, 8], value=str(shared.args.wbits) if shared.args.wbits > 0 else "None")
                             shared.gradio['groupsize'] = gr.Dropdown(label="groupsize", choices=["None", 32, 64, 128, 1024], value=str(shared.args.groupsize) if shared.args.groupsize > 0 else "None")