Upload 2 files

eval/test_set.json

@@ -0,0 +1,32 @@
+[
+  {"text": "People like you are a disease on this country.", "expected": "Direct Hate Speech"},
+  {"text": "Immigrants are just parasites who ruin everything.", "expected": "Direct Hate Speech"},
+  {"text": "Go back to your own country, you don't belong here.", "expected": "Direct Hate Speech"},
+  {"text": "You people are all criminals and always will be.", "expected": "Direct Hate Speech"},
+  {"text": "Women are too stupid to work in engineering.", "expected": "Direct Hate Speech"},
+  {"text": "Oh sure, because people from your country are always SO trustworthy.", "expected": "Sarcastic Hate Speech"},
+  {"text": "Wow, another brilliant contribution from someone who clearly finished school.", "expected": "Sarcastic Hate Speech"},
+  {"text": "Of course you'd say that, your kind is always so reasonable.", "expected": "Sarcastic Hate Speech"},
+  {"text": "Sure, because everyone from that religion is SO peaceful, right?", "expected": "Sarcastic Hate Speech"},
+  {"text": "Oh yes, women are definitely 'too emotional' to lead, how original.", "expected": "Sarcastic Hate Speech"},
+  {"text": "Great, my flight got cancelled again. Best day ever.", "expected": "Sarcastic/Ironic (Non-Hateful)"},
+  {"text": "Oh sure, because waiting three hours in line is exactly how I wanted to spend my Saturday.", "expected": "Sarcastic/Ironic (Non-Hateful)"},
+  {"text": "Wonderful, the WiFi is down again during my presentation.", "expected": "Sarcastic/Ironic (Non-Hateful)"},
+  {"text": "Sure, because losing my keys twice in one day is exactly what I needed.", "expected": "Sarcastic/Ironic (Non-Hateful)"},
+  {"text": "Oh fantastic, another meeting that could have been an email.", "expected": "Sarcastic/Ironic (Non-Hateful)"},
+  {"text": "The meeting is scheduled for 3pm tomorrow.", "expected": "Neutral"},
+  {"text": "I really enjoyed the concert last night.", "expected": "Neutral"},
+  {"text": "She works as a software engineer at a tech company.", "expected": "Neutral"},
+  {"text": "The recipe calls for two cups of flour and one egg.", "expected": "Neutral"},
+  {"text": "He moved to Barcelona last year for a new job.", "expected": "Neutral"},
+  {"text": "Those who don't eat pork tend to keep to themselves around here.", "expected": "Coded/Dogwhistle Hate — Needs Review"},
+  {"text": "The Austrian painter really had some interesting ideas about unity.", "expected": "Coded/Dogwhistle Hate — Needs Review"},
+  {"text": "You know how 'those people' are with money.", "expected": "Coded/Dogwhistle Hate — Needs Review"},
+  {"text": "The new engineers are really taking over the department, aren't they.", "expected": "Coded/Dogwhistle Hate — Needs Review"},
+  {"text": "Funny how certain neighborhoods have changed so much lately.", "expected": "Coded/Dogwhistle Hate — Needs Review"},
+  {"text": "I don't eat pork because of a health condition, not religion.", "expected": "Neutral"},
+  {"text": "He studied painting in Austria before switching careers.", "expected": "Neutral"},
+  {"text": "Our engineering team grew by ten people this quarter.", "expected": "Neutral"},
+  {"text": "Sure, blame the weather again for the delay.", "expected": "Sarcastic/Ironic (Non-Hateful)"},
+  {"text": "Some people just don't know how to use a turn signal, unbelievable.", "expected": "Neutral"}
+]

run_eval.py

@@ -0,0 +1,112 @@
+"""
+Evaluation script for DeepPragma Classifier.
+
+Runs the labeled test set against GroqClient.classify() and reports:
+- overall accuracy
+- per-category accuracy
+- a confusion list of misclassified examples (for manual inspection)
+
+Usage:
+    python run_eval.py
+
+Requires GROQ_API_KEY to be set in the environment.
+Place this file at the project root (same level as app.py),
+and test_set.json inside an eval/ folder (or adjust TEST_SET_PATH below).
+"""
+
+import json
+import os
+import time
+from collections import defaultdict
+
+from src.llm_client import GroqClient
+
+TEST_SET_PATH = os.path.join("eval", "test_set.json")
+
+
+def load_test_set(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def run_evaluation():
+    if not os.environ.get("GROQ_API_KEY"):
+        print("ERROR: GROQ_API_KEY not set in environment.")
+        return
+
+    test_cases = load_test_set(TEST_SET_PATH)
+    llm = GroqClient()
+
+    total = len(test_cases)
+    correct = 0
+    per_category_total = defaultdict(int)
+    per_category_correct = defaultdict(int)
+    errors = []
+
+    for i, case in enumerate(test_cases, start=1):
+        text = case["text"]
+        expected = case["expected"]
+
+        result = llm.classify(text)
+        predicted = result["label"]
+
+        per_category_total[expected] += 1
+        is_correct = predicted == expected
+        if is_correct:
+            correct += 1
+            per_category_correct[expected] += 1
+        else:
+            errors.append({
+                "text": text,
+                "expected": expected,
+                "predicted": predicted,
+                "reasoning": result.get("reasoning", ""),
+            })
+
+        print(f"[{i}/{total}] {'OK ' if is_correct else 'FAIL'} "
+              f"expected={expected!r} predicted={predicted!r}")
+
+        # small delay to be gentle with API rate limits
+        time.sleep(0.5)
+
+    print("\n" + "=" * 60)
+    print(f"OVERALL ACCURACY: {correct}/{total} ({100 * correct / total:.1f}%)")
+    print("=" * 60)
+
+    print("\nPER-CATEGORY ACCURACY:")
+    for category in per_category_total:
+        cat_total = per_category_total[category]
+        cat_correct = per_category_correct[category]
+        print(f"  {category}: {cat_correct}/{cat_total} "
+              f"({100 * cat_correct / cat_total:.1f}%)")
+
+    if errors:
+        print(f"\nMISCLASSIFIED EXAMPLES ({len(errors)}):")
+        for err in errors:
+            print(f"\n  Text: {err['text']}")
+            print(f"  Expected:  {err['expected']}")
+            print(f"  Predicted: {err['predicted']}")
+            print(f"  Reasoning: {err['reasoning']}")
+
+    # Save results to a JSON report for later reference (e.g. in your CV/interview prep)
+    report = {
+        "total": total,
+        "correct": correct,
+        "accuracy": round(100 * correct / total, 1),
+        "per_category": {
+            cat: {
+                "total": per_category_total[cat],
+                "correct": per_category_correct[cat],
+                "accuracy": round(100 * per_category_correct[cat] / per_category_total[cat], 1),
+            }
+            for cat in per_category_total
+        },
+        "errors": errors,
+    }
+    with open("eval_report.json", "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2, ensure_ascii=False)
+    print("\nFull report saved to eval_report.json")
+
+
+if __name__ == "__main__":
+    run_evaluation()