GurgenGulay commited on
Commit
87d61d3
·
verified ·
1 Parent(s): 15bf6d8

Update fine_tuning.py

Browse files
Files changed (1) hide show
  1. fine_tuning.py +1 -2
fine_tuning.py CHANGED
@@ -16,7 +16,6 @@ def clean_text(text):
16
  text = re.sub(r'\d+', '', text)
17
  text = text.lower()
18
  text = " ".join([word for word in text.split() if word not in stop_words])
19
- text = " ".join([ps.stem(word) for word in word_tokenize(text)])
20
  return text
21
 
22
  # Prompts okuma
@@ -40,7 +39,7 @@ def prepare_data(input_texts, target_texts):
40
 
41
  # Paraphrasing fonksiyonu
42
  def paraphrase_with_model(text, model, tokenizer):
43
- prompt = "paraphrase: " + text
44
  inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
45
  output_ids = model.generate(
46
  inputs["input_ids"],
 
16
  text = re.sub(r'\d+', '', text)
17
  text = text.lower()
18
  text = " ".join([word for word in text.split() if word not in stop_words])
 
19
  return text
20
 
21
  # Prompts okuma
 
39
 
40
  # Paraphrasing fonksiyonu
41
  def paraphrase_with_model(text, model, tokenizer):
42
+ prompt = "Teach the following content: " + text
43
  inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
44
  output_ids = model.generate(
45
  inputs["input_ids"],