Files

pert.py

@@ -0,0 +1,158 @@
+import csv
+import numpy as np
+from collections import defaultdict
+from scipy.optimize import linear_sum_assignment
+import os
+
+def load_clusters(path):
+    cluster_to_tokens = defaultdict(set)
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            parts = line.strip().split("|||")
+            if len(parts) < 2:
+                continue
+            token = parts[0]
+            cluster_id = parts[-1]
+            cluster_to_tokens[cluster_id].add(token)
+    return cluster_to_tokens
+
+def compute_jaccard_matrix(clusters_a, clusters_b):
+    a_keys = list(clusters_a.keys())
+    b_keys = list(clusters_b.keys())
+    matrix = np.zeros((len(a_keys), len(b_keys)))
+
+    for i, ca in enumerate(a_keys):
+        for j, cb in enumerate(b_keys):
+            set_a = clusters_a[ca]
+            set_b = clusters_b[cb]
+            intersection = len(set_a & set_b)
+            union = len(set_a | set_b)
+            matrix[i, j] = intersection / union if union > 0 else 0.0
+
+    return matrix, a_keys, b_keys
+
+# Dictionary mapping perturbation names to their descriptions
+perturbation_descriptions = {
+    "Scope Modification": "Identifies variables in complex scopes and moves them to unrelated blocks.",
+    "Lexical Similarity Modification": "Generates lexical variations of class and variable names with different casing.",
+    "Log Modification": "Adds logging statements to blocks of code for tracking execution flow.",
+    "Operator Modification": "Modifies boolean expressions by negating them in various contexts.",
+    "Pointer Modification": "Add C style pointer to the code.",
+    "POS finetuned": "Clusters based on finetuned POS codebert model",
+    "Random Modification": "Permutes statements within basic blocks, allowing different execution orders.",
+    "Try Catch Modification": "Converts switch statements into equivalent if statements.",
+    "Unused Statement Modification": "Inserts unused statements into blocks of code for testing/debugging.",
+    "Exact Name Class Variable Modification": "Renames classes and variables to a specific randomly generated name.",
+    "Casing Class Variable Modification": "Generates lexical variations of class and variable names with different casing."
+}
+
+def compute_and_log_csi(file_orig, file_pert, perturbation_name, output_csv="results/csi_summary.csv"):
+    clusters_orig = load_clusters(file_orig)
+    clusters_pert = load_clusters(file_pert)
+
+    if len(clusters_orig) != len(clusters_pert):
+        raise ValueError(f"Cluster count mismatch: {len(clusters_orig)} (original) vs {len(clusters_pert)} (perturbed)")
+
+    jaccard_matrix, orig_ids, pert_ids = compute_jaccard_matrix(clusters_orig, clusters_pert)
+
+    row_ind, col_ind = linear_sum_assignment(-jaccard_matrix)
+
+    matched_similarities = [jaccard_matrix[i, j] for i, j in zip(row_ind, col_ind)]
+    avg_jaccard = np.mean(matched_similarities)
+    csi = 1.0 - avg_jaccard
+
+    print(f"Perturbation: {perturbation_name}")
+    print(f"  Average Jaccard Similarity: {avg_jaccard:.4f}")
+    print(f"  Cluster Sensitivity Index (CSI): {csi:.4f}")
+
+    # Append to CSV
+    os.makedirs(os.path.dirname(output_csv), exist_ok=True)
+    file_exists = os.path.isfile(output_csv)
+
+    with open(output_csv, mode="a", newline='', encoding="utf-8") as file:
+        writer = csv.writer(file)
+        if not file_exists:
+            writer.writerow(["Perturbation", "Average Jaccard", "CSI", "Description"])
+        writer.writerow([perturbation_name, avg_jaccard, csi, perturbation_descriptions.get(perturbation_name, "No description available")])
+
+    return avg_jaccard, csi
+
+# Example usage
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_scope_error/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Scope Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_lexical_similar/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Lexical Similarity Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_log/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Log Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_operator/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Operator Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_pointer/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Pointer Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_POS/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="POS Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_random/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Random Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_trycatch/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Try Catch Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_unusedStatement/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Unused Statement Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_exactNameClassVariable/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Exact Name Class Variable Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+compute_and_log_csi(
+    "codenet_4000_del_15000/Java/layer12/kmeans/clusters-kmeans-350.txt",
+    "codenet_4000_CasingClassVariable/java/layer12/kmeans/clusters-kmeans-350.txt",
+    perturbation_name="Casing Class Variable Modification",
+    output_csv="results/csi_summary.csv"
+)
+
+# You can now call compute_and_log_csi again and again for other perturbations!

results/csi_summary.csv

@@ -0,0 +1,12 @@
+Perturbation,Average Jaccard,CSI,Description
+Scope Modification,0.6788942354152336,0.32110576458476636,Identifies variables in complex scopes and moves them to unrelated blocks.
+Lexical Similarity Modification,0.5546761391746684,0.4453238608253316,Generates lexical variations of class and variable names with different casing.
+Log Modification,0.5597545985057552,0.44024540149424485,Adds logging statements to blocks of code for tracking execution flow.
+Operator Modification,0.7675911973340813,0.23240880266591868,Modifies boolean expressions by negating them in various contexts.
+Pointer Modification,0.7341816285924795,0.2658183714075205,Add C style pointer to the code.
+POS Modification,0.39399085068850775,0.6060091493114923,No description available
+Random Modification,0.5314837325594708,0.4685162674405292,"Permutes statements within basic blocks, allowing different execution orders."
+Try Catch Modification,0.6985673658171294,0.3014326341828706,Converts switch statements into equivalent if statements.
+Unused Statement Modification,0.5844954343120634,0.4155045656879366,Inserts unused statements into blocks of code for testing/debugging.
+Exact Name Class Variable Modification,0.675121649837896,0.324878350162104,Renames classes and variables to a specific randomly generated name.
+Casing Class Variable Modification,0.6722713965133429,0.3277286034866571,Generates lexical variations of class and variable names with different casing.