Add llama-2-70b GGML support (#3285)

2023-07-24 16:37:03 -03:00 · 2023-07-24 16:37:03 -03:00 · a07d070b6c
commit a07d070b6c
parent 6f4830b4d3
8 changed files with 20 additions and 4 deletions
--- a/modules/llamacpp_model.py
+++ b/modules/llamacpp_model.py
@ -53,6 +53,8 @@ class LlamaCppModel:
            'n_gpu_layers': shared.args.n_gpu_layers,
            'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.),
            'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
+            'n_gqa': shared.args.n_gqa or None,
+            'rms_norm_eps': shared.args.rms_norm_eps or None,
        }

        result.model = Llama(**params)