Spaces:
Runtime error
Runtime error
Update fine_tuning.py
Browse files- fine_tuning.py +1 -2
fine_tuning.py
CHANGED
@@ -16,7 +16,6 @@ def clean_text(text):
|
|
16 |
text = re.sub(r'\d+', '', text)
|
17 |
text = text.lower()
|
18 |
text = " ".join([word for word in text.split() if word not in stop_words])
|
19 |
-
text = " ".join([ps.stem(word) for word in word_tokenize(text)])
|
20 |
return text
|
21 |
|
22 |
# Prompts okuma
|
@@ -40,7 +39,7 @@ def prepare_data(input_texts, target_texts):
|
|
40 |
|
41 |
# Paraphrasing fonksiyonu
|
42 |
def paraphrase_with_model(text, model, tokenizer):
|
43 |
-
prompt = "
|
44 |
inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
|
45 |
output_ids = model.generate(
|
46 |
inputs["input_ids"],
|
|
|
16 |
text = re.sub(r'\d+', '', text)
|
17 |
text = text.lower()
|
18 |
text = " ".join([word for word in text.split() if word not in stop_words])
|
|
|
19 |
return text
|
20 |
|
21 |
# Prompts okuma
|
|
|
39 |
|
40 |
# Paraphrasing fonksiyonu
|
41 |
def paraphrase_with_model(text, model, tokenizer):
|
42 |
+
prompt = "Teach the following content: " + text
|
43 |
inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
|
44 |
output_ids = model.generate(
|
45 |
inputs["input_ids"],
|