Update app.py

app.py

@@ -5,7 +5,7 @@ import gradio as gr
 
 # ## GGUF MOD: Unused environment variables for PyTorch have been removed.
 # ## GGUF MOD: ctransformers handles its own memory and GPU management.
-# os.putenv('PYTORCH_NVML_BASED_CUDA_CHECK','1')
+os.putenv('PYTORCH_NVML_BASED_CUDA_CHECK','1')
 # os.putenv('TORCH_LINALG_PREFER_CUSOLVER','1')
 # alloc_conf_parts = [
 #     'expandable_segments:True',
@@ -19,6 +19,7 @@ os.putenv('HF_HUB_ENABLE_HF_TRANSFER','1')
 # ## GGUF MOD: BitsAndBytesConfig is no longer needed.
 from ctransformers import AutoModelForCausalLM
 from transformers import AutoTokenizer
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
 # ## GGUF MOD: PyTorch backend settings are not used by ctransformers.
 # torch.backends.cuda.matmul.allow_tf32 = True