final

finetuneandevaluationscores.py

@@ -1,710 +0,0 @@
-# -*- coding: utf-8 -*-
-"""FineTuneAndEvaluationscores.ipynb
-
-Automatically generated by Colab.
-
-Original file is located at
-    https://colab.research.google.com/drive/122o4g9XIObEOsSOo8-ZcfE0tgGRG-QrV
-"""
-
-!pip install torch==2.4.1 transformers==4.44.2 datasets==3.0.1 nltk==3.9.1 pandas==2.2.3 matplotlib==3.8.4 evaluate==0.4.5 rouge_score>=0.1.2 sentence-transformers==2.7.0 -q
-
-# Uninstall conflicting packages
-!pip uninstall -y torch torchvision torchaudio pandas fsspec gcsfs -q
-# Install compatible versions
-!pip install torch torchvision torchaudio pandas transformers datasets nltk matplotlib evaluate rouge_score sentence-transformers -q
-
-!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json -O train-v1.1.json
-
-import json
-
-with open('train-v1.1.json', 'r', encoding='utf-8') as f:
-    squad_data = json.load(f)
-
-# Print the first paragraph to inspect
-print("Sample data:", squad_data['data'][0]['paragraphs'][0])
-
-import pandas as pd
-from datasets import Dataset, Features, Value
-
-data = []
-for article in squad_data['data']:
-    for paragraph in article['paragraphs']:
-        context = paragraph['context'].strip()
-        for qa in paragraph['qas']:
-            question = qa['question'].strip()
-            answer = qa['answers'][0]['text'].strip() if qa['answers'] else ""
-            if context and question and answer:  # Basic cleaning
-                data.append({"context": context, "question": question, "answer": answer})
-
-# Limit to 100 samples for quick testing
-data = data[:100]
-
-# Create DataFrame and Dataset
-df = pd.DataFrame(data)
-features = Features({
-    "context": Value("string"),
-    "question": Value("string"),
-    "answer": Value("string")
-})
-dataset = Dataset.from_pandas(df, features=features)
-train_test_split = dataset.train_test_split(test_size=0.2, seed=42)
-train_dataset = train_test_split["train"]
-eval_dataset = train_test_split["test"]
-
-print(f"Train size: {len(train_dataset)} | Eval size: {len(eval_dataset)}")
-print("First train example:", train_dataset[0])
-
-
-
-# Install dependencies
-!pip uninstall -y torch torchvision torchaudio pandas fsspec gcsfs -q
-!pip install torch torchvision torchaudio pandas transformers datasets nltk matplotlib evaluate rouge_score sentence-transformers -q
-# Restart runtime after installation
-
-import json
-import pandas as pd
-from datasets import Dataset, Features, Value
-from transformers import T5Tokenizer, T5ForConditionalGeneration, TrainingArguments, Trainer
-import evaluate
-import matplotlib.pyplot as plt
-import torch
-import nltk
-import numpy as np  # Added missing import
-nltk.download('punkt')
-
-# Verify setup
-print(f"Torch version: {torch.__version__}")
-print(f"GPU available: {torch.cuda.is_available()}")
-
-# Step 2: Download and load dataset
-!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json -O train-v1.1.json
-with open('train-v1.1.json', 'r', encoding='utf-8') as f:
-    squad_data = json.load(f)
-print("Sample data:", squad_data['data'][0]['paragraphs'][0])
-
-# Step 3: Clean and prepare dataset
-data = []
-for article in squad_data['data']:
-    for paragraph in article['paragraphs']:
-        context = paragraph['context'].strip()
-        for qa in paragraph['qas']:
-            question = qa['question'].strip()
-            answer = qa['answers'][0]['text'].strip() if qa['answers'] else ""
-            if context and question and answer:
-                data.append({"context": context, "question": question, "answer": answer})
-
-data = data[:100]
-df = pd.DataFrame(data)
-features = Features({
-    "context": Value("string"),
-    "question": Value("string"),
-    "answer": Value("string")
-})
-dataset = Dataset.from_pandas(df, features=features)
-train_test_split = dataset.train_test_split(test_size=0.2, seed=42)
-train_dataset = train_test_split["train"]
-eval_dataset = train_test_split["test"]
-print(f"Train size: {len(train_dataset)} | Eval size: {len(eval_dataset)}")
-print("First train example:", train_dataset[0])
-
-# Step 4: Fine-tune the model
-model_name = "valhalla/t5-small-qg-hl"
-tokenizer = T5Tokenizer.from_pretrained(model_name)
-model = T5ForConditionalGeneration.from_pretrained(model_name)
-
-def preprocess(examples):
-    inputs = [f"generate question: {ctx} {ans}" for ctx, ans in zip(examples['context'], examples['answer'])]
-    targets = examples['question']
-    model_inputs = tokenizer(inputs, max_length=256, truncation=True, padding="max_length", return_tensors=None)
-    labels = tokenizer(targets, max_length=32, truncation=True, padding="max_length")["input_ids"]
-    model_inputs["labels"] = labels
-    return model_inputs
-
-tokenized_train_dataset = train_dataset.map(preprocess, remove_columns=train_dataset.column_names, batched=True)
-tokenized_eval_dataset = eval_dataset.map(preprocess, remove_columns=eval_dataset.column_names, batched=True)
-
-tokenized_train_dataset = tokenized_train_dataset.with_format("torch")
-tokenized_eval_dataset = tokenized_eval_dataset.with_format("torch")
-
-training_args = TrainingArguments(
-    output_dir="./qg-finetuned",
-    per_device_train_batch_size=2,
-    per_device_eval_batch_size=2,
-    num_train_epochs=3,  # Increased to 3
-    eval_strategy="epoch",
-    learning_rate=2e-5,
-    logging_dir="./logs",
-    logging_steps=10,
-    save_strategy="epoch",
-    save_total_limit=1,
-    fp16=True,
-    report_to="none",
-    load_best_model_at_end=True,
-    metric_for_best_model="eval_loss",
-    greater_is_better=False
-)
-
-
-def compute_metrics(eval_pred):
-    predictions, labels = eval_pred
-    predictions = predictions[0] if isinstance(predictions, tuple) else predictions
-    predictions = np.argmax(predictions, axis=-1) if predictions.ndim == 3 else predictions
-    labels = np.argmax(labels, axis=-1) if labels.ndim == 3 else labels
-
-    def decode_sequences(sequences):
-        return [tokenizer.decode(seq, skip_special_tokens=True) for seq in sequences]
-
-    decoded_preds = decode_sequences(predictions)
-    decoded_labels = decode_sequences(labels)
-
-    rouge = evaluate.load("rouge")
-    rouge_score = rouge.compute(predictions=decoded_preds, references=decoded_labels)
-
-    return {
-        "rouge1": rouge_score["rouge1"],
-        "rougeL": rouge_score["rougeL"]
-    }
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized_train_dataset,
-    eval_dataset=tokenized_eval_dataset,
-    compute_metrics=compute_metrics
-)
-
-print("Fine-tuning started...")
-trainer.train()
-print("Running final evaluation...")
-results = trainer.evaluate()
-print("Final Evaluation Results:")
-for metric, score in results.items():
-    print(f"  {metric}: {score}")
-
-# Step 5: Generate and evaluate sample questions
-from transformers import GenerationConfig
-model.eval()
-sample = eval_dataset[0]
-inputs = tokenizer(f"generate question: {sample['context']} {sample['answer']}", max_length=256, truncation=True, padding="max_length", return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
-
-generation_config = GenerationConfig(early_stopping=True, num_beams=5, max_length=128)  # Adjusted
-outputs = model.generate(**inputs, generation_config=generation_config)
-generated_question = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-print(f"Context: {sample['context'][:100]}...")
-print(f"Answer: {sample['answer']}")
-print(f"Generated Question: {generated_question}")
-print(f"Reference Question: {sample['question']}")
-
-# Step 6: Plot evaluation scores
-log_history = trainer.state.log_history
-epochs = [entry['epoch'] for entry in log_history if 'eval_rouge1' in entry]
-rouge1_scores = [entry['eval_rouge1'] for entry in log_history if 'eval_rouge1' in entry]
-rougeL_scores = [entry['eval_rougeL'] for entry in log_history if 'eval_rougeL' in entry]
-
-plt.figure(figsize=(10, 5))
-plt.plot(epochs, rouge1_scores, label='ROUGE-1')
-plt.plot(epochs, rougeL_scores, label='ROUGE-L')
-plt.xlabel('Epoch')
-plt.ylabel('Score')
-plt.title('Evaluation Scores Over Epochs')
-plt.legend()
-plt.grid(True)
-plt.show()
-
-# Step 7: Save the model
-model.save_pretrained("./qg-finetuned/final")
-tokenizer.save_pretrained("./qg-finetuned/final")
-print("Model and tokenizer saved!")
-
-
-
-# Install dependencies
-!pip uninstall -y torch torchvision torchaudio pandas fsspec gcsfs -q
-!pip install torch torchvision torchaudio pandas transformers datasets nltk matplotlib evaluate rouge_score sentence-transformers -q
-# Restart runtime after installation
-
-import json
-import pandas as pd
-from datasets import Dataset, Features, Value
-from transformers import T5Tokenizer, T5ForConditionalGeneration, TrainingArguments, Trainer
-import evaluate
-import matplotlib.pyplot as plt
-import torch
-import nltk
-import numpy as np  # Added missing import
-nltk.download('punkt')
-
-# Verify setup
-print(f"Torch version: {torch.__version__}")
-print(f"GPU available: {torch.cuda.is_available()}")
-
-# Step 2: Download and load dataset
-!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json -O train-v1.1.json
-with open('train-v1.1.json', 'r', encoding='utf-8') as f:
-    squad_data = json.load(f)
-print("Sample data:", squad_data['data'][0]['paragraphs'][0])
-
-# Step 3: Clean and prepare dataset
-data = []
-for article in squad_data['data']:
-    for paragraph in article['paragraphs']:
-        context = paragraph['context'].strip()
-        for qa in paragraph['qas']:
-            question = qa['question'].strip()
-            answer = qa['answers'][0]['text'].strip() if qa['answers'] else ""
-            if context and question and answer:
-                data.append({"context": context, "question": question, "answer": answer})
-
-data = data[:800]
-df = pd.DataFrame(data)
-features = Features({
-    "context": Value("string"),
-    "question": Value("string"),
-    "answer": Value("string")
-})
-dataset = Dataset.from_pandas(df, features=features)
-train_test_split = dataset.train_test_split(test_size=0.2, seed=42)
-train_dataset = train_test_split["train"]
-eval_dataset = train_test_split["test"]
-print(f"Train size: {len(train_dataset)} | Eval size: {len(eval_dataset)}")
-print("First train example:", train_dataset[0])
-
-# Step 4: Fine-tune the model
-model_name = "valhalla/t5-small-qg-hl"
-tokenizer = T5Tokenizer.from_pretrained(model_name)
-model = T5ForConditionalGeneration.from_pretrained(model_name)
-
-def preprocess(examples):
-    inputs = [f"generate question: {ctx} {ans}" for ctx, ans in zip(examples['context'], examples['answer'])]
-    targets = examples['question']
-    model_inputs = tokenizer(inputs, max_length=256, truncation=True, padding="max_length", return_tensors=None)
-    labels = tokenizer(targets, max_length=32, truncation=True, padding="max_length")["input_ids"]
-    model_inputs["labels"] = labels
-    return model_inputs
-
-tokenized_train_dataset = train_dataset.map(preprocess, remove_columns=train_dataset.column_names, batched=True)
-tokenized_eval_dataset = eval_dataset.map(preprocess, remove_columns=eval_dataset.column_names, batched=True)
-
-tokenized_train_dataset = tokenized_train_dataset.with_format("torch")
-tokenized_eval_dataset = tokenized_eval_dataset.with_format("torch")
-
-training_args = TrainingArguments(
-    output_dir="./qg-finetuned",
-    per_device_train_batch_size=4,
-    per_device_eval_batch_size=4,
-    num_train_epochs=2,
-    eval_strategy="epoch",
-    learning_rate=2e-5,
-    logging_dir="./logs",
-    logging_steps=10,
-    save_strategy="epoch",
-    save_total_limit=1,
-    fp16=True,
-    report_to="none",
-    load_best_model_at_end=True,
-    metric_for_best_model="eval_loss",
-    greater_is_better=False
-)
-
-
-def compute_metrics(eval_pred):
-    predictions, labels = eval_pred
-    predictions = predictions[0] if isinstance(predictions, tuple) else predictions
-    predictions = np.argmax(predictions, axis=-1) if predictions.ndim == 3 else predictions
-    labels = np.argmax(labels, axis=-1) if labels.ndim == 3 else labels
-
-    def decode_sequences(sequences):
-        return [tokenizer.decode(seq, skip_special_tokens=True) for seq in sequences]
-
-    decoded_preds = decode_sequences(predictions)
-    decoded_labels = decode_sequences(labels)
-
-    rouge = evaluate.load("rouge")
-    rouge_score = rouge.compute(predictions=decoded_preds, references=decoded_labels)
-
-    return {
-        "rouge1": rouge_score["rouge1"],
-        "rougeL": rouge_score["rougeL"]
-    }
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized_train_dataset,
-    eval_dataset=tokenized_eval_dataset,
-    compute_metrics=compute_metrics
-)
-
-print("Fine-tuning started...")
-trainer.train()
-print("Running final evaluation...")
-results = trainer.evaluate()
-print("Final Evaluation Results:")
-for metric, score in results.items():
-    print(f"  {metric}: {score}")
-
-# Step 5: Generate and evaluate sample questions
-from transformers import GenerationConfig
-model.eval()
-sample = eval_dataset[0]
-inputs = tokenizer(f"generate question: {sample['context']} {sample['answer']}", max_length=256, truncation=True, padding="max_length", return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
-
-generation_config = GenerationConfig(early_stopping=True, num_beams=5, max_length=128)  # Adjusted
-outputs = model.generate(**inputs, generation_config=generation_config)
-generated_question = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-print(f"Context: {sample['context'][:100]}...")
-print(f"Answer: {sample['answer']}")
-print(f"Generated Question: {generated_question}")
-print(f"Reference Question: {sample['question']}")
-
-# Step 6: Plot evaluation scores
-log_history = trainer.state.log_history
-epochs = [entry['epoch'] for entry in log_history if 'eval_rouge1' in entry]
-rouge1_scores = [entry['eval_rouge1'] for entry in log_history if 'eval_rouge1' in entry]
-rougeL_scores = [entry['eval_rougeL'] for entry in log_history if 'eval_rougeL' in entry]
-
-plt.figure(figsize=(10, 5))
-plt.plot(epochs, rouge1_scores, label='ROUGE-1')
-plt.plot(epochs, rougeL_scores, label='ROUGE-L')
-plt.xlabel('Epoch')
-plt.ylabel('Score')
-plt.title('Evaluation Scores Over Epochs')
-plt.legend()
-plt.grid(True)
-plt.show()
-
-# Step 7: Save the model
-model.save_pretrained("./qg-finetuned/final")
-tokenizer.save_pretrained("./qg-finetuned/final")
-print("Model and tokenizer saved!")
-
-
-
-# Install dependencies
-!pip uninstall -y torch torchvision torchaudio pandas fsspec gcsfs -q
-!pip install torch torchvision torchaudio pandas transformers datasets nltk matplotlib evaluate rouge_score sentence-transformers -q
-# Restart runtime after installation
-
-import json
-import pandas as pd
-from datasets import Dataset, Features, Value
-from transformers import T5Tokenizer, T5ForConditionalGeneration, TrainingArguments, Trainer
-import evaluate
-import matplotlib.pyplot as plt
-import torch
-import nltk
-import numpy as np  # Added missing import
-nltk.download('punkt')
-
-# Verify setup
-print(f"Torch version: {torch.__version__}")
-print(f"GPU available: {torch.cuda.is_available()}")
-
-# Step 2: Download and load dataset
-!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json -O train-v1.1.json
-with open('train-v1.1.json', 'r', encoding='utf-8') as f:
-    squad_data = json.load(f)
-print("Sample data:", squad_data['data'][0]['paragraphs'][0])
-
-# Step 3: Clean and prepare dataset
-data = []
-for article in squad_data['data']:
-    for paragraph in article['paragraphs']:
-        context = paragraph['context'].strip()
-        for qa in paragraph['qas']:
-            question = qa['question'].strip()
-            answer = qa['answers'][0]['text'].strip() if qa['answers'] else ""
-            if context and question and answer:
-                data.append({"context": context, "question": question, "answer": answer})
-
-data = data[:800]
-df = pd.DataFrame(data)
-features = Features({
-    "context": Value("string"),
-    "question": Value("string"),
-    "answer": Value("string")
-})
-dataset = Dataset.from_pandas(df, features=features)
-train_test_split = dataset.train_test_split(test_size=0.2, seed=42)
-train_dataset = train_test_split["train"]
-eval_dataset = train_test_split["test"]
-print(f"Train size: {len(train_dataset)} | Eval size: {len(eval_dataset)}")
-print("First train example:", train_dataset[0])
-
-# Step 4: Fine-tune the model
-model_name = "valhalla/t5-small-qg-hl"
-tokenizer = T5Tokenizer.from_pretrained(model_name)
-model = T5ForConditionalGeneration.from_pretrained(model_name)
-
-def preprocess(examples):
-    inputs = []
-    for ctx, ans in zip(examples['context'], examples['answer']):
-        if ans in ctx:
-            highlighted = ctx.replace(ans, f"<hl> {ans} <hl>")
-            inputs.append(f"generate question: {highlighted}")
-        else:
-            inputs.append(f"generate question: {ctx} <hl> {ans} <hl>")
-    targets = examples['question']
-    model_inputs = tokenizer(inputs, max_length=256, truncation=True, padding="max_length", return_tensors=None)
-    labels = tokenizer(targets, max_length=32, truncation=True, padding="max_length")["input_ids"]
-    model_inputs["labels"] = labels
-    return model_inputs
-
-tokenized_train_dataset = train_dataset.map(preprocess, remove_columns=train_dataset.column_names, batched=True)
-tokenized_eval_dataset = eval_dataset.map(preprocess, remove_columns=eval_dataset.column_names, batched=True)
-
-tokenized_train_dataset = tokenized_train_dataset.with_format("torch")
-tokenized_eval_dataset = tokenized_eval_dataset.with_format("torch")
-
-training_args = TrainingArguments(
-    output_dir="./qg-finetuned",
-    per_device_train_batch_size=4,
-    per_device_eval_batch_size=4,
-    num_train_epochs=2,
-    eval_strategy="epoch",
-    learning_rate=2e-5,
-    logging_dir="./logs",
-    logging_steps=10,
-    save_strategy="epoch",
-    save_total_limit=1,
-    fp16=True,
-    report_to="none",
-    load_best_model_at_end=True,
-    metric_for_best_model="eval_loss",
-    greater_is_better=False
-)
-
-
-def compute_metrics(eval_pred):
-    predictions, labels = eval_pred
-    predictions = predictions[0] if isinstance(predictions, tuple) else predictions
-    predictions = np.argmax(predictions, axis=-1) if predictions.ndim == 3 else predictions
-    labels = np.argmax(labels, axis=-1) if labels.ndim == 3 else labels
-
-    def decode_sequences(sequences):
-        return [tokenizer.decode(seq, skip_special_tokens=True) for seq in sequences]
-
-    decoded_preds = decode_sequences(predictions)
-    decoded_labels = decode_sequences(labels)
-
-    rouge = evaluate.load("rouge")
-    rouge_score = rouge.compute(predictions=decoded_preds, references=decoded_labels)
-
-    return {
-        "rouge1": rouge_score["rouge1"],
-        "rougeL": rouge_score["rougeL"]
-    }
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized_train_dataset,
-    eval_dataset=tokenized_eval_dataset,
-    compute_metrics=compute_metrics
-)
-
-print("Fine-tuning started...")
-trainer.train()
-print("Running final evaluation...")
-results = trainer.evaluate()
-print("Final Evaluation Results:")
-for metric, score in results.items():
-    print(f"  {metric}: {score}")
-
-# Step 5: Generate and evaluate sample questions
-from transformers import GenerationConfig
-model.eval()
-sample = eval_dataset[0]
-inputs = tokenizer(f"generate question: {sample['context']} {sample['answer']}", max_length=256, truncation=True, padding="max_length", return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
-
-generation_config = GenerationConfig(early_stopping=True, num_beams=5, max_length=128)  # Adjusted
-outputs = model.generate(**inputs, generation_config=generation_config)
-generated_question = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-print(f"Context: {sample['context'][:100]}...")
-print(f"Answer: {sample['answer']}")
-print(f"Generated Question: {generated_question}")
-print(f"Reference Question: {sample['question']}")
-
-# Step 6: Plot evaluation scores
-log_history = trainer.state.log_history
-epochs = [entry['epoch'] for entry in log_history if 'eval_rouge1' in entry]
-rouge1_scores = [entry['eval_rouge1'] for entry in log_history if 'eval_rouge1' in entry]
-rougeL_scores = [entry['eval_rougeL'] for entry in log_history if 'eval_rougeL' in entry]
-
-plt.figure(figsize=(10, 5))
-plt.plot(epochs, rouge1_scores, label='ROUGE-1')
-plt.plot(epochs, rougeL_scores, label='ROUGE-L')
-plt.xlabel('Epoch')
-plt.ylabel('Score')
-plt.title('Evaluation Scores Over Epochs')
-plt.legend()
-plt.grid(True)
-plt.show()
-
-# Step 7: Save the model
-model.save_pretrained("./qg-finetuned/final")
-tokenizer.save_pretrained("./qg-finetuned/final")
-print("Model and tokenizer saved!")
-
-from tqdm import tqdm
-
-decoded_preds = []
-decoded_refs = []
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-model.to(device)
-model.eval()
-
-for i, sample in enumerate(tqdm(eval_dataset)):
-    if sample["answer"] in sample["context"]:
-        highlighted_context = sample["context"].replace(sample["answer"], f"<hl> {sample['answer']} <hl>")
-    else:
-        highlighted_context = sample["context"] + f" <hl> {sample['answer']} <hl>"
-
-    input_text = f"generate question: {highlighted_context}"
-    inputs = tokenizer(
-        input_text,
-        return_tensors="pt",
-        truncation=True,
-        padding="max_length",
-        max_length=256
-    ).to(device)
-
-    output_ids = model.generate(
-        **inputs,
-        max_length=64,
-        num_beams=4,
-        early_stopping=False,  # <— loosen this up for now
-        no_repeat_ngram_size=2
-    )
-
-    # 🪵 Debug print
-    print(f"\n--- Sample {i + 1} ---")
-    print("Raw token IDs:", output_ids[0].tolist())
-
-    decoded_pred = tokenizer.decode(output_ids[0], skip_special_tokens=True)
-    print("Decoded Prediction:", decoded_pred)
-
-    decoded_preds.append(decoded_pred)
-    decoded_refs.append(sample["question"])
-
-from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
-
-# Use smoothing to avoid zero score for short outputs
-smoothie = SmoothingFunction().method1
-
-bleu_scores = []
-print("\nSample Predictions vs References with BLEU-1:")
-print("-" * 50)
-
-for i in range(min(5, len(decoded_preds))):
-    pred = decoded_preds[i]
-    ref = decoded_refs[i]
-    bleu = sentence_bleu([ref.split()], pred.split(), weights=(1, 0, 0, 0), smoothing_function=smoothie)
-
-    print(f"\nSample {i + 1}")
-    print(f"Prediction : {pred}")
-    print(f"Reference  : {ref}")
-    print(f"BLEU-1     : {bleu:.4f}")
-    bleu_scores.append(bleu)
-
-# Compute average BLEU-1 score across all examples
-avg_bleu = sum(bleu_scores) / len(bleu_scores) if bleu_scores else 0
-print(f"\nAverage BLEU-1 Score on Eval Set: {avg_bleu:.4f}")
-
-from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
-
-for i, (pred, ref) in enumerate(zip(decoded_preds, decoded_refs)):
-    bleu2 = sentence_bleu([ref.split()], pred.split(), weights=(0.5, 0.5), smoothing_function=smoothie)
-    bleu4 = sentence_bleu([ref.split()], pred.split(), weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smoothie)
-    print(f"Sample {i+1}\nBLEU-2: {bleu2:.4f}, BLEU-4: {bleu4:.4f}")
-
-print("Length of decoded_preds:", len(decoded_preds))
-print("Length of decoded_refs:", len(decoded_refs))
-print("Length of bleu_scores:", len(bleu_scores))
-
-from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
-
-smoothing = SmoothingFunction().method1
-bleu_scores = [
-    sentence_bleu([ref.split()], pred.split(), weights=(1, 0, 0, 0), smoothing_function=smoothing)
-    for pred, ref in zip(decoded_preds, decoded_refs)
-]
-
-df = pd.DataFrame({
-    "Prediction": decoded_preds,
-    "Reference": decoded_refs,
-    "BLEU-1": bleu_scores
-})
-df.to_csv("question_generation_bleu_scores.csv", index=False)
-
-#preview of the file
-import pandas as pd
-
-df_check = pd.read_csv("question_generation_bleu_scores.csv")
-print(df_check.head())
-
-# Plot ROUGE-1 and ROUGE-L scores over epochs
-plt.figure(figsize=(10, 5))
-plt.plot(epochs, rouge1_scores, marker='o', label='ROUGE-1')
-plt.plot(epochs, rougeL_scores, marker='o', label='ROUGE-L')
-plt.xlabel('Epoch')
-plt.ylabel('Score')
-plt.title('ROUGE Scores over Epochs')
-plt.legend()
-plt.grid(True)
-plt.tight_layout()
-plt.show()
-
-#ADD TO YOUR REPORT :
-#INTERPRETATION : The line plot shows a steady increase in both ROUGE-1 and ROUGE-L scores over training epochs, indicating that the model's ability to generate relevant and coherent questions improved progressively. ROUGE-1 evaluates unigram overlap, while ROUGE-L captures longest common subsequence similarity, so their combined trend confirms enhanced syntactic and semantic alignment with reference questions.
-
-#Histogram: BLEU-1 Score Distribution
-import matplotlib.pyplot as plt
-
-# BLEU score histogram
-plt.figure(figsize=(8, 4))
-plt.hist(bleu_scores, bins=10, color='skyblue', edgecolor='black')
-plt.title('BLEU-1 Score Distribution')
-plt.xlabel('BLEU-1 Score')
-plt.ylabel('Frequency')
-plt.grid(True)
-plt.tight_layout()
-plt.show()
-
-#INTERPRETATION : The BLEU-1 histogram reveals that most generated questions received lower unigram overlap scores, with only a few predictions achieving high similarity with the reference. This is expected in generative tasks, especially when multiple valid phrasings exist for a single question.
-
-print("Length of BLEU-1 scores:", len(bleu_scores))
-print("Length of ROUGE-1 scores:", len(rouge1_scores))
-
-import evaluate
-rouge = evaluate.load("rouge")
-
-rouge1_scores = []
-rougeL_scores = []
-
-for pred, ref in zip(decoded_preds, decoded_refs):
-    result = rouge.compute(predictions=[pred], references=[ref])
-    rouge1_scores.append(result["rouge1"])
-    rougeL_scores.append(result["rougeL"])
-
-print("Length of BLEU-1 scores:", len(bleu_scores))
-print("Length of ROUGE-1 scores:", len(rouge1_scores))
-
-#Scatter Plot Between BLEU-1 and ROUGE-1
-import matplotlib.pyplot as plt
-
-plt.figure(figsize=(8, 6))
-plt.scatter(bleu_scores, rouge1_scores, alpha=0.6, color='purple')
-plt.title('BLEU-1 vs ROUGE-1 Scores')
-plt.xlabel('BLEU-1 Score')
-plt.ylabel('ROUGE-1 Score')
-plt.grid(True)
-plt.show()
-
-#Interpretation : To assess the quality of the generated questions, we computed BLEU-1, ROUGE-1, and ROUGE-L scores across the evaluation set. While BLEU-1 captures exact n-gram overlap, ROUGE measures both lexical and semantic similarity more flexibly. A scatter plot comparing BLEU-1 and ROUGE-1 scores showed moderate variation, with some samples scoring high on ROUGE despite lower BLEU, suggesting semantic validity despite lexical mismatch. This highlights the limitation of using a single metric and motivates multi-metric evaluation for generative tasks.
-