Add llama-2-70b GGML support (#3285)

2023-07-24 16:37:03 -03:00 · 2023-07-24 16:37:03 -03:00 · a07d070b6c
commit a07d070b6c
parent 6f4830b4d3
8 changed files with 20 additions and 4 deletions
--- a/modules/llamacpp_hf.py
+++ b/modules/llamacpp_hf.py
@ -106,6 +106,8 @@ class LlamacppHF(PreTrainedModel):
            'n_gpu_layers': shared.args.n_gpu_layers,
            'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.),
            'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
+            'n_gqa': shared.args.n_gqa or None,
+            'rms_norm_eps': shared.args.rms_norm_eps or None,
            'logits_all': True,
        }