pytorch
/

Qwen3-8B-AWQ-INT4

Text Generation

text-generation-inference

Model card Files Files and versions

jerryzh168 commited on 5 days ago

Commit

99caebc

·

verified ·

1 Parent(s): 574af31

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -281,7 +281,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig
 # use "Qwen/Qwen3-8B" or "pytorch/Qwen3-8B-AWQ-INT4"
 model_id = "pytorch/Qwen3-8B-AWQ-INT4"
-quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 torch.cuda.reset_peak_memory_stats()

 # use "Qwen/Qwen3-8B" or "pytorch/Qwen3-8B-AWQ-INT4"
 model_id = "pytorch/Qwen3-8B-AWQ-INT4"
+quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda:0", torch_dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 torch.cuda.reset_peak_memory_stats()