GurgenGulay commited on
Commit
3dc5c3f
·
verified ·
1 Parent(s): df269a0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -11
app.py CHANGED
@@ -1,14 +1,12 @@
1
- from fine_tuning import fine_tune_model # fine_tuning.py'deki fonksiyonu içe aktar
2
-
3
 
4
  def clean_text_with_spacy(text):
5
- # spaCy ile metni işlemeye başla
6
  doc = nlp(text)
7
- # Stopwords'leri, noktalama işaretlerini dışla ve lemmatize et
8
  tokens = [token.lemma_.lower() for token in doc if not token.is_stop and not token.is_punct]
9
  return " ".join(tokens)
10
 
11
- # Prompts.txt dosyasından verileri okuma
12
  def read_prompts(file_path):
13
  input_texts = []
14
  target_texts = []
@@ -21,17 +19,14 @@ def read_prompts(file_path):
21
  target_texts.append(line.replace("target:", "").strip())
22
  return input_texts, target_texts
23
 
24
- # Temizlenmiş metni modelinize göndermek için fonksiyon
25
  def process_input_for_fine_tuning(input_texts, target_texts):
26
- # Metni temizle
27
  cleaned_input_texts = [clean_text_with_spacy(text) for text in input_texts]
28
  cleaned_target_texts = [clean_text_with_spacy(text) for text in target_texts]
29
-
30
- # Temizlenmiş veriyi fine-tuning için göndermek
31
  fine_tune_model(cleaned_input_texts, cleaned_target_texts)
32
 
33
- # **prompts.txt** dosyasından veriyi okuma
34
  input_texts, target_texts = read_prompts("prompts.txt")
35
 
36
- # Temizlenmiş veriyi fine_tuning.py'ye göndermek için işlemi başlat
37
  process_input_for_fine_tuning(input_texts, target_texts)
 
1
+ from fine_tuning import fine_tune_model # Import the function from fine_tuning.py
 
2
 
3
  def clean_text_with_spacy(text):
4
+ # Clean the text using spaCy
5
  doc = nlp(text)
 
6
  tokens = [token.lemma_.lower() for token in doc if not token.is_stop and not token.is_punct]
7
  return " ".join(tokens)
8
 
9
+ # Function to read prompts.txt file
10
  def read_prompts(file_path):
11
  input_texts = []
12
  target_texts = []
 
19
  target_texts.append(line.replace("target:", "").strip())
20
  return input_texts, target_texts
21
 
22
+ # Function to process input texts for fine-tuning
23
  def process_input_for_fine_tuning(input_texts, target_texts):
 
24
  cleaned_input_texts = [clean_text_with_spacy(text) for text in input_texts]
25
  cleaned_target_texts = [clean_text_with_spacy(text) for text in target_texts]
 
 
26
  fine_tune_model(cleaned_input_texts, cleaned_target_texts)
27
 
28
+ # Reading data from prompts.txt
29
  input_texts, target_texts = read_prompts("prompts.txt")
30
 
31
+ # Initiating the fine-tuning process
32
  process_input_for_fine_tuning(input_texts, target_texts)