Remove quantization_config entirely to avoid bitsandbytes dependency2

app.py

@@ -1,24 +1,36 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
 import accelerate
 
 print("Accelerate version:", accelerate.__version__)
 
-from transformers import BitsAndBytesConfig
+# Load the original model with overridden configuration
+original_config = AutoConfig.from_pretrained(
+    "unsloth/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit"
+)
+# Remove quantization-related attributes from the config
+original_config._load_in_4bit = False
+original_config._load_in_8bit = False
+original_config.quant_method = None
 
-# Load the original model
 original_model = AutoModelForCausalLM.from_pretrained(
     "unsloth/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit",
-    load_in_4bit=False,  # Disable 4-bit quantization
-    # Remove quantization_config entirely
+    config=original_config  # Use the overridden configuration
 )
 original_tokenizer = AutoTokenizer.from_pretrained("unsloth/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit")
 
-# Load the fine-tuned model
+# Load the fine-tuned model with overridden configuration
+fine_tuned_config = AutoConfig.from_pretrained(
+    "kas1/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit-John1"
+)
+# Remove quantization-related attributes from the config
+fine_tuned_config._load_in_4bit = False
+fine_tuned_config._load_in_8bit = False
+fine_tuned_config.quant_method = None
+
 fine_tuned_model = AutoModelForCausalLM.from_pretrained(
     "kas1/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit-John1",
-    load_in_4bit=False,  # Disable 4-bit quantization
-    # Remove quantization_config entirely
+    config=fine_tuned_config  # Use the overridden configuration
 )
 fine_tuned_tokenizer = AutoTokenizer.from_pretrained("kas1/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit-John1")
 
@@ -41,21 +53,17 @@ def batch_test(json_file):
     import json
     results = []
     data = json.load(json_file)
-
     for item in data:
         question = item.get("question", "")
         expected_answer = item.get("answer", "")
-
         # Generate responses from both models
         original_response, fine_tuned_response = compare_models(question)
-
         results.append({
             "question": question,
             "expected_answer": expected_answer,
             "original_model_response": original_response,
             "fine_tuned_model_response": fine_tuned_response
         })
-
     return results
 
 # Define the Gradio interface
@@ -68,7 +76,6 @@ with gr.Blocks() as demo:
         compare_button = gr.Button("Compare Responses")
         original_output = gr.Textbox(label="Original Model Response")
         fine_tuned_output = gr.Textbox(label="Fine-Tuned Model Response")
-
         compare_button.click(
             compare_models,
             inputs=prompt_input,
@@ -80,7 +87,6 @@ with gr.Blocks() as demo:
         json_file_input = gr.File(label="Upload JSON File with Questions")
         batch_results = gr.JSON(label="Comparison Results")
         batch_button = gr.Button("Run Batch Test")
-
         batch_button.click(
             batch_test,
             inputs=json_file_input,