Spaces:
Runtime error
Runtime error
| import pandas as pd | |
| import random | |
| from pathlib import Path | |
| import re | |
| from transformers import pipeline | |
| # Configuración básica | |
| raw_path = Path("../data/raw") | |
| processed_path = Path("../data/processed") | |
| synthetic_path = Path("../data/synthetic") | |
| # Cargar datos | |
| medqa_file = raw_path / "medqa.csv" | |
| medqa = pd.read_csv(medqa_file, encoding='utf-8') | |
| # Limpieza de datos | |
| def clean_text(text): | |
| if pd.isna(text): | |
| return "" | |
| text = str(text).lower().strip() | |
| text = re.sub(r'\s+', ' ', text) # Elimina espacios extras | |
| text = re.sub(r'[^\w\s]', '', text) # Elimina caracteres especiales (opcional, ajusta según necesidad) | |
| return text | |
| # Aplicar limpieza a las columnas relevantes | |
| medqa['question'] = medqa['question'].apply(clean_text) | |
| medqa['answer'] = medqa['answer'].apply(clean_text) | |
| # Si las opciones están en formato de lista o texto, también puedes limpiarlas | |
| if 'options' in medqa.columns: | |
| medqa['options'] = medqa['options'].apply(lambda x: clean_text(str(x)) if pd.notnull(x) else "") | |
| # Verificación inicial (opcional, para debugging) | |
| print("Dimensiones de MedQA:", medqa.shape) | |
| print("Valores nulos en MedQA:\n", medqa.isnull().sum()) | |
| print("Duplicados en MedQA:", medqa.duplicated().sum()) | |
| # Manejo de nulos (por ejemplo, rellenar con valores predeterminados) | |
| medqa.fillna({"options": "No options provided"}, inplace=True) | |
| # Generación de datos sintéticos para MedQA | |
| def generate_synthetic_medqa(df, n=100): | |
| """ | |
| Genera datos sintéticos para el dataset MedQA. | |
| Toma 'n' muestras y modifica ligeramente el texto en las columnas question y answer. | |
| """ | |
| sampled = df.sample(n, replace=True) | |
| modificaciones = [ | |
| "(consulta sintética)", | |
| "[versión aumentada]", | |
| "(datos generados)", | |
| "[synthetic data]" | |
| ] | |
| questions, answers = [], [] | |
| for _, row in sampled.iterrows(): | |
| mod = random.choice(modificaciones) | |
| question = str(row["question"]) + " " + mod | |
| answer = str(row["answer"]) + " " + mod | |
| questions.append(question) | |
| answers.append(answer) | |
| # Si quieres incluir opciones sintéticas, puedes generarlas de manera similar | |
| if 'options' in df.columns: | |
| options_sinteticas = [str(row["options"]) + " " + mod for _, row in sampled.iterrows()] | |
| df_synthetic = pd.DataFrame({ | |
| "question_sintetica": questions, | |
| "answer_sintetica": answers, | |
| "options_sinteticas": options_sinteticas | |
| }) | |
| else: | |
| df_synthetic = pd.DataFrame({ | |
| "question_sintetica": questions, | |
| "answer_sintetica": answers | |
| }) | |
| return df_synthetic | |
| # Generar y guardar datos sintéticos | |
| medqa_synthetic = generate_synthetic_medqa(medqa, n=100) | |
| medqa_synthetic.to_csv(synthetic_path / "medqa_synthetic.csv", index=False) | |
| # Combinar datos originales y sintéticos | |
| medqa_combined = pd.concat([medqa, medqa_synthetic], ignore_index=True) | |
| medqa_combined.to_csv(processed_path / "medqa_combined.csv", index=False) | |
| print("Datos procesados y sintéticos generados exitosamente para MedQA.") |