From 316e07f06a67751d047c2072d8296d05bfb6a1c9 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 10 Feb 2023 16:36:06 -0300
Subject: [PATCH] auto-assign gpu memory with --auto-devices alone

---
 server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server.py b/server.py
index 27b4202..30b5903 100644
--- a/server.py
+++ b/server.py
@@ -130,7 +130,7 @@ def load_model(model_name):
 
             if args.gpu_memory:
                 params.append(f"max_memory={{0: '{args.gpu_memory or '99'}GiB', 'cpu': '{args.cpu_memory or '99'}GiB'}}")
-            elif (args.gpu_memory or args.cpu_memory) and not args.load_in_8bit:
+            elif not args.load_in_8bit:
                 total_mem = (torch.cuda.get_device_properties(0).total_memory/(1024*1024))
                 suggestion = round((total_mem-1000)/1000)*1000
                 if total_mem-suggestion < 800: