Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -1,14 +1,12 @@
|
|
1 |
-
from fine_tuning import fine_tune_model # fine_tuning.py
|
2 |
-
|
3 |
|
4 |
def clean_text_with_spacy(text):
|
5 |
-
#
|
6 |
doc = nlp(text)
|
7 |
-
# Stopwords'leri, noktalama işaretlerini dışla ve lemmatize et
|
8 |
tokens = [token.lemma_.lower() for token in doc if not token.is_stop and not token.is_punct]
|
9 |
return " ".join(tokens)
|
10 |
|
11 |
-
#
|
12 |
def read_prompts(file_path):
|
13 |
input_texts = []
|
14 |
target_texts = []
|
@@ -21,17 +19,14 @@ def read_prompts(file_path):
|
|
21 |
target_texts.append(line.replace("target:", "").strip())
|
22 |
return input_texts, target_texts
|
23 |
|
24 |
-
#
|
25 |
def process_input_for_fine_tuning(input_texts, target_texts):
|
26 |
-
# Metni temizle
|
27 |
cleaned_input_texts = [clean_text_with_spacy(text) for text in input_texts]
|
28 |
cleaned_target_texts = [clean_text_with_spacy(text) for text in target_texts]
|
29 |
-
|
30 |
-
# Temizlenmiş veriyi fine-tuning için göndermek
|
31 |
fine_tune_model(cleaned_input_texts, cleaned_target_texts)
|
32 |
|
33 |
-
#
|
34 |
input_texts, target_texts = read_prompts("prompts.txt")
|
35 |
|
36 |
-
#
|
37 |
process_input_for_fine_tuning(input_texts, target_texts)
|
|
|
1 |
+
from fine_tuning import fine_tune_model # Import the function from fine_tuning.py
|
|
|
2 |
|
3 |
def clean_text_with_spacy(text):
|
4 |
+
# Clean the text using spaCy
|
5 |
doc = nlp(text)
|
|
|
6 |
tokens = [token.lemma_.lower() for token in doc if not token.is_stop and not token.is_punct]
|
7 |
return " ".join(tokens)
|
8 |
|
9 |
+
# Function to read prompts.txt file
|
10 |
def read_prompts(file_path):
|
11 |
input_texts = []
|
12 |
target_texts = []
|
|
|
19 |
target_texts.append(line.replace("target:", "").strip())
|
20 |
return input_texts, target_texts
|
21 |
|
22 |
+
# Function to process input texts for fine-tuning
|
23 |
def process_input_for_fine_tuning(input_texts, target_texts):
|
|
|
24 |
cleaned_input_texts = [clean_text_with_spacy(text) for text in input_texts]
|
25 |
cleaned_target_texts = [clean_text_with_spacy(text) for text in target_texts]
|
|
|
|
|
26 |
fine_tune_model(cleaned_input_texts, cleaned_target_texts)
|
27 |
|
28 |
+
# Reading data from prompts.txt
|
29 |
input_texts, target_texts = read_prompts("prompts.txt")
|
30 |
|
31 |
+
# Initiating the fine-tuning process
|
32 |
process_input_for_fine_tuning(input_texts, target_texts)
|