updates

app.py

@@ -0,0 +1,112 @@
+import streamlit as st
+from pathlib import Path
+import pandas as pd
+import json
+import metric
+from sklearn.metrics import roc_auc_score, roc_curve
+import numpy as np
+import altair as alt
+
+
+
+st.set_page_config(
+    page_title="Public Leaderboard",
+    initial_sidebar_state="collapsed",
+    layout="wide",  # This makes the app use the full width of the screen
+)
+
+@st.cache_data
+def load_results(task):
+    return pd.read_csv(task).set_index("team")
+
+
+split = "public"
+
+def show_leaderboad(results):
+    cols = [
+            "generated_accuracy",
+            "pristine_accuracy",
+            "balanced_accuracy",
+            "fail_rate",
+            "total_time",
+        ]
+
+    # st.dataframe(results[f"{split}_score"])
+
+    column_config = {
+        "balanced_accuracy": st.column_config.ProgressColumn(
+            "Balanced Acc", format="compact", min_value=0, pinned=True, max_value=1.0, width = "large"
+        ),
+        "generated_accuracy": st.column_config.ProgressColumn(
+            "🤖 Acc", format="compact", min_value=0, pinned=True, max_value=1.0, width = "large"
+        ),
+        "pristine_accuracy": st.column_config.ProgressColumn(
+            "🧑‍🎤 Acc", format="compact", min_value=0, pinned=True, max_value=1.0, width = "large"
+        ),
+        "fail_rate": st.column_config.NumberColumn(
+            "❌ Fail Rate",
+            format="compact",
+            width = "small",
+        ),
+        "fail_total_timerate": st.column_config.NumberColumn(
+            "🕒 Inference Time",
+            format="compact",
+            width = "small",
+        ),
+    }
+
+    labels = {"pristine": "🧑‍🎤", "generated": "🤖"}
+
+    for c in results[f"{split}_score"].columns:
+        if "accuracy" in c:
+            continue
+        if any(p in c for p in ["generated", "pristine"]):
+            s = c.split("_")
+            pred = s[0]
+            source = " ".join(s[1:])
+            column_config[c] = st.column_config.ProgressColumn(
+                labels[pred] + " " + source,
+                help=c,
+                format="compact",
+                min_value=0,
+                max_value=1.0,
+            )
+
+    "#### Summary"
+    st.dataframe(results[f"{split}_score"].loc[:, cols], column_config=column_config)
+
+    "#### Accuracy on 🤖 Generated by Source"
+
+    cols = [
+        c
+        for c in results[f"{split}_score"].columns
+        if "generated" in c and "accuracy" not in c
+    ]
+    st.dataframe(results[f"{split}_score"].loc[:, cols], column_config=column_config)
+
+    "#### Accuracy on 🧑‍🎤 Pristine by Source"
+
+    cols = [
+        c
+        for c in results[f"{split}_score"].columns
+        if "pristine" in c and "accuracy" not in c
+    ]
+    st.dataframe(results[f"{split}_score"].loc[:, cols], column_config=column_config)
+
+
+split = "public"
+
+
+st.markdown("#### Detailed Public Leaderboard")
+st.markdown("[SAFE: Synthetic Audio Forensics Evaluation Challenge](https://stresearch.github.io/SAFE/)")
+
+t1,t2 = st.tabs(["**Task 1**","**Task 2**"])
+with t1:
+    results1 = {f"{split}_score": load_results("task1.csv")}
+    show_leaderboad(results1)
+    
+with t2:
+    results2 = {f"{split}_score": load_results("task2.csv")}
+    show_leaderboad(results2)
+
+

metric.py

@@ -0,0 +1,125 @@
+import pandas as pd
+from huggingface_hub import hf_hub_download
+import json
+
+
+def _metric(solution_df,submission_df, mode = "top_level", admin = False):
+    """
+    This function calculates the accuracy of the generated predictions.
+
+    Parameters
+    ----------
+    solution_df : pandas.DataFrame
+        The dataframe containing the solution data.
+    submission_df : pandas.DataFrame
+        The dataframe containing the submission data.
+    mode : str, optional
+        The mode of evaluation. Can be "top_level" or "bottom_level". The default is "top_level".
+    
+    Returns
+    -------
+    None.
+    """
+
+
+    solution_df["submission_pred"] = submission_df["pred"]
+   
+    if admin:
+        source_col = "source_og"
+    else:
+        source_col = "source"
+
+    
+    cols = ["split","pred", source_col]
+
+
+    solution_df["correct"] = solution_df["pred"] == solution_df["submission_pred"]
+    accuracy = solution_df.groupby(cols)["correct"].mean().to_frame("accuracy").reset_index()
+    accuracy["score_name"] = accuracy["pred"] +"_"+ accuracy[source_col]
+    
+    evaluation = {}
+    
+    split = "public"
+    
+    temp = accuracy.query(f"split=='{split}'")
+    scores_by_source = temp.set_index("score_name")["accuracy"].sort_index()
+    scores_by_source["generated_accuracy"] = temp.query("pred=='generated'")["accuracy"].mean()
+    scores_by_source["pristine_accuracy"] = temp.query("pred=='pristine'")["accuracy"].mean()
+    scores_by_source["balanced_accuracy"] = (scores_by_source["generated_accuracy"] + scores_by_source["pristine_accuracy"])/2.
+    
+    
+    if mode == "top_level":
+        scores_to_save = ["generated_accuracy", "pristine_accuracy", "balanced_accuracy"]
+        evaluation[f"{split}_score"] = scores_by_source.loc[scores_to_save].to_dict()
+    else:
+        evaluation[f"{split}_score"] = scores_by_source.to_dict()
+
+    split = "private"
+    # private has everything
+
+    temp = accuracy
+    scores_by_source = temp.set_index("score_name")["accuracy"].sort_index()
+    scores_by_source["generated_accuracy"] = temp.query("pred=='generated'")["accuracy"].mean()
+    scores_by_source["pristine_accuracy"] = temp.query("pred=='pristine'")["accuracy"].mean()
+    scores_by_source["balanced_accuracy"] = (scores_by_source["generated_accuracy"] + scores_by_source["pristine_accuracy"])/2.
+    
+    if mode == "top_level":
+        scores_to_save = ["generated_accuracy", "pristine_accuracy", "balanced_accuracy"]
+        evaluation[f"{split}_score"] = scores_by_source.loc[scores_to_save].to_dict()
+    else:
+        evaluation[f"{split}_score"] = scores_by_source.to_dict()
+    
+
+    if "time" in submission_df.columns:
+        solution_df["submission_time"] = submission_df["time"]
+        
+        split = "public"
+        evaluation[f"{split}_score"]["total_time"] = float(solution_df.query(f"split=='{split}'")["submission_time"].sum())
+
+        split = "private"
+        evaluation[f"{split}_score"]["total_time"] = float(solution_df["submission_time"].sum())
+    else:
+        for split in ["public","private"]:
+            evaluation[f"{split}_score"]["total_time"] = -1
+
+
+    if "score" in submission_df.columns:
+        solution_df["submission_score"] = submission_df["score"]
+    
+        split = "public"
+        evaluation[f"{split}_score"]["fail_rate"] = float(solution_df.query(f"split=='{split}'")["submission_score"].isna().mean())
+
+        split = "private"
+        evaluation[f"{split}_score"]["fail_rate"] = float(solution_df["submission_score"].isna().mean())
+
+    else:
+        for split in ["public","private"]:
+            evaluation[f"{split}_score"]["fail_rate"] = -1
+
+
+
+    return evaluation
+
+
+
+def compute(params):
+    solution_file = hf_hub_download(
+        repo_id=params.competition_id,
+        filename="solution.csv",
+        token=params.token,
+        repo_type="dataset",
+    )
+
+    solution_df = pd.read_csv(solution_file).set_index(params.submission_id_col)
+
+    submission_filename = f"submissions/{params.team_id}-{params.submission_id}.csv"
+    submission_file = hf_hub_download(
+        repo_id=params.competition_id,
+        filename=submission_filename,
+        token=params.token,
+        repo_type="dataset",
+    )
+    
+    submission_df = pd.read_csv(submission_file).set_index(params.submission_id_col)
+
+    return _metric(solution_df,submission_df)

process_data.py

@@ -0,0 +1,140 @@
+import pandas as pd
+from pathlib import Path
+from pathlib import Path
+import pandas as pd
+import json
+import metric
+from sklearn.metrics import roc_auc_score, roc_curve
+import numpy as np
+import altair as alt
+
+_metric = metric._metric
+
+
+def get_submission(f):
+    submission_info = json.load(open(f))
+    submissions = pd.DataFrame(submission_info["submissions"])
+    submissions["team_id"] = submission_info["id"]
+
+    return submissions
+
+def get_submissions_file(f):
+    submission_df = pd.read_csv(f).set_index("id")
+    if isinstance(submission_df.iloc[0]["score"],str):
+        submission_df.loc[:, "score"] = submission_df.loc[:, "score"].apply(lambda a: json.loads(a)[0])
+    return submission_df
+
+def load_results(local_dir):
+    team_file_name = "teams.json"
+    team_info = pd.read_json(Path(local_dir) / team_file_name).T
+    team_info.loc["baselines", "name"] = "baselines"
+    submission_info_dir = "submission_info"
+    submission_info_files = list((Path(local_dir) / submission_info_dir).glob("*.json"))
+    # submission_info_files += ["baselines/baselines.json"]
+    submissions = pd.concat(
+        [get_submission(f) for f in submission_info_files], ignore_index=True
+    )
+    submissions.loc[:, "team"] = team_info.loc[
+        submissions["team_id"].values, "name"
+    ].values
+
+    submissions["submission_files"] = submissions.apply(
+        lambda a: (
+            str(
+                Path(local_dir)
+                / "submissions"
+                / (a["team_id"] + "-" + a["submission_id"] + ".csv")
+            )
+            if a["team_id"] != "baselines"
+            else str(
+                Path("baselines") / (a["team_id"] + "-" + a["submission_id"] + ".csv")
+            )
+        ),
+        axis=1,
+    )
+    submissions = submissions.drop(columns=["public_score", "private_score"])
+    submissions["submission"] = (
+        submissions["team"] + " - " + submissions["submission_repo"]
+    )
+    return submissions
+
+
+def compute_metrics(submissions, local_dir, admin=True):
+
+    submissions=submissions.query("status==3.0")
+
+    if not admin:
+        selected_by_team = submissions.groupby("team")["selected"].sum()
+        teams_no_selected = selected_by_team.index[selected_by_team==0]
+        submissions.loc[submissions.team.isin(teams_no_selected),"selected"] = True
+        submissions = submissions.query("selected")
+
+    solution_df = pd.read_csv(Path(local_dir) / "solution.csv").set_index("id")
+
+    results = {"private_score": [], "public_score": []}
+
+    fields = ["team_id", "team", "submission_id", "submission_repo"]
+    for i, row in submissions.T.items():
+        # r = pd.read_csv(row["submission_files"]).set_index("id")
+        r = get_submissions_file(row["submission_files"])
+        eval = _metric(solution_df, r, mode="detailed", admin=admin)
+        for m in ["private_score", "public_score"]:
+            for f in fields:
+                eval[m][f] = row[f]
+            eval[m]["submission"] = f"{row.team} - {row.submission_repo}"
+
+            eval[m] = pd.Series(eval[m]).to_frame().T
+            results[m].append(eval[m])
+
+    for m in ["private_score", "public_score"]:
+        temp = pd.concat(results[m], ignore_index=True).T
+        temp.index.name = "metric"
+        temp = temp.reset_index()
+
+        # def parse(s):
+        #     if any(p in s for p in ["generated","pristine"]):
+        #         s = s.split("_")
+        #         return pd.Series(dict(pred = s[0], source = "_".join(s[1:])))
+        #     else:
+        #         return pd.Series(dict(pred = s, source = None))
+
+        # temp = pd.concat([temp, temp["metric"].apply(parse)], axis = 1)
+        # results[m] = temp.set_index(["pred","source"])
+        # results[m] = results[m].drop(columns = ["metric"]).T
+        results[m] = (
+            temp.set_index("metric")
+            .T.sort_values("balanced_accuracy", ascending=False)
+            .drop_duplicates(subset=["team", "submission_repo"])
+        )
+
+        if not admin:
+            # only show top selected
+            results[m] = (
+                results[m]
+                .sort_values(["team", "balanced_accuracy"], ascending=False)
+                .drop_duplicates(subset=["team"])
+                .sort_values("balanced_accuracy", ascending=False)
+            )
+
+        results[m] = results[m].set_index("submission" if admin else "team")
+
+
+    fields_to_merge = ['generated_accuracy', 'pristine_accuracy', 'balanced_accuracy', 'total_time', 'fail_rate']
+
+    submissions = pd.concat([submissions.set_index("submission_id"),
+                             results["private_score"].reset_index().set_index("submission_id").loc[:,fields_to_merge]],axis = 1).reset_index()
+    
+    return results, submissions
+
+
+def process_data(path,save_path):
+    submissions = load_results(path)
+    results,submissions = compute_metrics(submissions, path, admin=False)
+    cols_to_drop = ["team_id","submission_id","submission_repo","submission"]
+    results["public_score"].drop(columns =cols_to_drop).to_csv(save_path)
+
+
+if __name__=="__main__":
+    process_data("comp_data_task1","task1.csv")
+    process_data("comp_data_task2","task2.csv")
+

requirements.txt

@@ -0,0 +1,2 @@
+scikit-learn
+numpy

run.sh

@@ -0,0 +1 @@
+streamlit run app.py

task1.csv

@@ -0,0 +1,10 @@
+team,generated_g_02,generated_g_04,generated_g_05,generated_g_06,generated_g_09,generated_g_10,generated_g_11,pristine_p_00,pristine_p_01,pristine_p_02,pristine_p_05,pristine_p_09,pristine_p_10,pristine_p_11,pristine_p_16,pristine_p_18,pristine_p_20,generated_accuracy,pristine_accuracy,balanced_accuracy,total_time,fail_rate
+baseline-2,0.82,0.985,0.715,1.0,0.875,1.0,0.68,0.9,0.66,0.895,0.915,0.935,0.68,0.815,0.85,0.88,0.945,0.8678571428571428,0.8474999999999999,0.8576785714285713,1095.5729746818542,0.0
+ISPL,0.965,0.985,0.445,1.0,1.0,1.0,0.29,1.0,0.08,0.985,0.98,0.97,0.835,0.925,0.965,0.86,0.995,0.812142857142857,0.8595,0.8358214285714285,86.06722044944746,0.0
+baseline-1,0.825,0.995,0.465,0.97,1.0,1.0,0.925,0.96,0.37,0.955,0.815,1.0,0.69,0.285,0.635,0.885,0.955,0.8828571428571428,0.755,0.8189285714285715,2053.3383333683014,0.002058823529411765
+DMF,0.925,0.74,0.76,0.965,0.835,1.0,1.0,0.37,0.0,0.11,0.105,0.81,0.175,0.99,0.53,0.43,0.45,0.8892857142857142,0.397,0.6431428571428571,73.15069174766523,0.0
+Anon_Peking,0.985,0.92,0.91,0.98,0.89,0.97,0.95,0.28,0.105,0.125,0.23,0.2,0.355,0.09,0.15,0.165,0.125,0.9435714285714285,0.1825,0.5630357142857143,366.5311744213103,0.0
+gylin,0.115,0.36,0.03,0.035,0.535,0.0,0.005,0.98,0.985,0.96,0.9,0.99,0.8,1.0,0.82,0.93,0.945,0.1542857142857143,0.931,0.5426428571428572,21.466523647308183,0.0
+UCASRikki,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.5,116.53438615798935,0.0
+safe-test,0.435,0.5,0.475,0.55,0.495,0.53,0.52,0.51,0.485,0.48,0.58,0.435,0.53,0.43,0.545,0.48,0.445,0.5007142857142858,0.492,0.4963571428571429,142.9702701568602,0.0
+JAIST-HIS,0.895,0.94,0.985,0.525,0.89,0.98,0.455,0.01,0.035,0.025,0.03,0.13,0.065,0.0,0.04,0.035,0.07,0.8099999999999999,0.044,0.427,61.43132781982406,0.0

task2.csv

@@ -0,0 +1,5 @@
+team,generated_g_02,generated_g_04,generated_g_05,generated_g_06,generated_g_09,generated_g_10,generated_g_11,pristine_p_00,pristine_p_01,pristine_p_02,pristine_p_05,pristine_p_09,pristine_p_10,pristine_p_11,pristine_p_16,pristine_p_18,pristine_p_20,generated_accuracy,pristine_accuracy,balanced_accuracy,total_time,fail_rate
+baseline-2,0.8789473684210526,0.9394736842105263,0.6868421052631579,0.9394736842105263,0.9078947368421053,0.9263157894736842,0.8236842105263158,0.9,0.66,0.895,0.915,0.935,0.68,0.815,0.85,0.88,0.945,0.8718045112781956,0.8474999999999999,0.8596522556390978,1558.1854865550995,0.0
+baseline-1,0.9210526315789473,0.9868421052631579,0.7684210526315789,0.9289473684210526,0.9763157894736842,0.9815789473684211,0.9157894736842105,0.96,0.36,0.955,0.815,1.0,0.69,0.29,0.635,0.885,0.955,0.925563909774436,0.7545,0.8400319548872179,2615.167044878006,0.008583690987124463
+ISPL,0.9105263157894737,0.9131578947368421,0.7,0.9710526315789474,0.8710526315789474,0.95,0.5921052631578947,0.985,0.015,0.925,0.8,0.74,0.83,0.805,0.935,0.8,0.975,0.8439849624060151,0.7809999999999999,0.8124924812030074,4660.0,0.0
+safe-test,0.49736842105263157,0.4710526315789474,0.4789473684210526,0.4842105263157895,0.5236842105263158,0.5026315789473684,0.49473684210526314,0.5,0.455,0.495,0.495,0.505,0.515,0.505,0.535,0.51,0.51,0.4932330827067669,0.5025,0.49786654135338343,200.45346903800942,0.0

update_data.sh

@@ -0,0 +1,3 @@
+huggingface-cli download safe-challenge/SAFEChallengeTask1 --local-dir ./comp_data_task1 --repo-type dataset
+huggingface-cli download safe-challenge/SAFEChallengeTask2 --local-dir ./comp_data_task2 --repo-type dataset
+python process_data.py