8-bitowa kwantyzacja

app.py

@@ -27,6 +27,7 @@ def load_model():
                 torch_dtype=torch.bfloat16,
                 device_map="auto",
                 low_cpu_mem_usage=True
+                load_in_8bit=True
             )
             logger.info("Model loaded successfully")
         model_load_queue.put(model)