Update README.md

README.md

@@ -1,3 +1,71 @@
 ---
 license: mit
 ---
+```python
+model = LlamaForCausalLM.from_pretrained(
+        '/llama_7b_hf',
+        load_in_8bit=True,
+        torch_dtype=torch.float16,
+        device_map='auto',
+    )
+
+lora_weights = 'ashwinram472/alpaca-cleaned-lora-7b'
+
+model = PeftModel.from_pretrained(
+    model,
+    lora_weights,
+    torch_dtype=torch.float16,
+)
+
+tokenizer = LlamaTokenizer.from_pretrained("../models/llama_7b_hf")
+
+def generate_prompt(instruction: str, input_ctxt: str = None) -> str:
+    if input_ctxt:
+        return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+### Instruction:
+{instruction}
+
+### Input:
+{input_ctxt}
+
+### Response:"""
+    else:
+        return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
+
+### Instruction:
+{instruction}
+
+### Response:"""
+
+generation_config = GenerationConfig(
+    temperature=0.1,
+    top_p=0.75,
+    top_k=40,
+    num_beams=4,
+    max_new_tokens=128,
+)
+
+model.eval()
+
+
+instruction = "Count up from 1 to 500."
+
+input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.
+
+prompt = generate_prompt(instruction, input_ctxt)
+input_ids = tokenizer(prompt, return_tensors="pt").input_ids
+input_ids = input_ids.to(model.device)
+
+with torch.no_grad():
+    outputs = model.generate(
+        input_ids=input_ids,
+        generation_config=generation_config,
+        return_dict_in_generate=True,
+        output_scores=True,
+    )
+
+response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
+
+print(response.split("### Response:")[1].strip().split("### Instruction")[0])
+```