Update README.md

README.md

@@ -18,10 +18,10 @@ The backbone is under `LoftQ/Llama-2-7b-hf-4bit-64rank` and LoRA adapters are un
 
 ## Model Info
 ### Backbone
-- Stored format: `torch.bfloat16`
-- Size: ~ 14 GiB
+- Stored format: bitsandbytes nf4
+- Size: ~ 5 GiB
 - Loaded format: bitsandbytes nf4
-- Size loaded on GPU: ~3.5 GiB
+- Size loaded on GPU: ~5 GiB
 
 ### LoRA adapters
 - rank: 64
@@ -39,16 +39,7 @@ from peft import PeftModel
 
 MODEL_ID = "LoftQ/Llama-2-7b-hf-4bit-64rank"
 
-base_model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID, 
-    torch_dtype=torch.bfloat16,  # you may change it with different models
-    quantization_config=BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_compute_dtype=torch.bfloat16,  # bfloat16 is recommended
-        bnb_4bit_use_double_quant=False,
-        bnb_4bit_quant_type='nf4',
-    ),
-)
+base_model = AutoModelForCausalLM.from_pretrained(MODEL_ID)
 peft_model = PeftModel.from_pretrained(
     base_model,
     MODEL_ID,