File size: 8,832 Bytes
b694cb3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 |
from utils import (
update_leaderboard_multilingual,
update_leaderboard_one_vs_all,
handle_evaluation,
process_results_file,
create_html_image,
)
import os
import gradio as gr
from constants import *
if __name__ == "__main__":
with gr.Blocks() as app:
base_path = os.path.dirname(__file__)
local_image_path = os.path.join(base_path, 'open_arabic_lid_arena.png')
gr.HTML(create_html_image(local_image_path))
gr.Markdown("# 🏅 Open Arabic Dialect Identification Leaderboard")
# Multi-dialects leaderboard
with gr.Tab("Multi-dialects model leaderboard"):
gr.Markdown("""
Complete leaderboard across multiple arabic dialects.
Compare the performance of different models across various metrics such as FNR, FPR, and other clasical metrics.
"""
)
with gr.Row():
with gr.Column(scale=1):
gr.Markdown("### Select country to display")
country_selector = gr.Dropdown(
choices=supported_dialects,
value='Morocco', # Default to Morocco of course
label="Country"
)
with gr.Column(scale=2):
gr.Markdown("### Select metrics to display")
metric_checkboxes = gr.CheckboxGroup(
choices=metrics,
value=default_metrics,
label="Metrics"
)
with gr.Row():
leaderboard_table = gr.DataFrame(
interactive=False
)
gr.Markdown("</br>")
gr.Markdown("## Contribute to the Leaderboard")
gr.Markdown("""
We welcome contributions from the community!
If you have a model that you would like to see on the leaderboard, please use the 'Evaluate a model' or 'Upload your results' tabs to submit your model's performance.
Let's work together to improve Arabic dialect identification! 🚀
""")
# Dialect confusion leaderboard
with gr.Tab("Dialect confusion leaderboard"): # use to be "One-vs-All leaderboard"
gr.Markdown("""
Detailed analysis of how well models distinguish specific dialects from others.
For each target dialect, see how often models incorrectly classify other dialects as the target.
Lower `false_positive_rate` indicate better ability to identify the true dialect by
showing **how often it misclassifies other dialects as the target dialect**.
"""
)
with gr.Row():
with gr.Column(scale=1):
gr.Markdown("### Select your target language")
target_language_selector = gr.Dropdown(
choices=languages_to_display_one_vs_all,
value='Morocco', # Default to Morocco of course
label="Target Language"
)
with gr.Column(scale=2):
gr.Markdown("### Select languages to compare to")
languages_checkboxes = gr.CheckboxGroup(
choices=languages_to_display_one_vs_all,
value=default_languages,
label="Languages"
)
with gr.Row():
binary_leaderboard_table = gr.DataFrame(
interactive=False
)
with gr.Tab("Evaluate a model"):
gr.Markdown("Suggest a model to evaluate 🤗 (Supports only **Fasttext** models as SfayaLID, GlotLID, OpenLID, etc.)")
gr.Markdown("For other models, you are welcome to **submit your results** through the upload section.")
model_path = gr.Textbox(label="Model Path", placeholder='path/to/model')
model_path_bin = gr.Textbox(label=".bin filename", placeholder='model.bin')
gr.Markdown("### **⚠️ To ensure correct results, tick this when the model's labels are the iso_codes**")
use_mapping = gr.Checkbox(label="Does not map to country", value=True) # Initially enabled
eval_button = gr.Button("Evaluate", value=False) # Initially disabled
# Status message area
status_message = gr.Markdown(value="")
def update_status_message():
return "### **⚠️Evaluating... Please wait...**"
eval_button.click(update_status_message, outputs=[status_message])
eval_button.click(handle_evaluation, inputs=[model_path, model_path_bin, use_mapping], outputs=[leaderboard_table, status_message])
with gr.Tab("Upload your results"):
# Define a code block to display
code_snippet = """
```python
# Load your model
model = ... # Load your model here
# Load evaluation benchmark
eval_dataset = load_dataset("atlasia/Arabic-LID-Leaderboard", split='test').to_pandas() # do not change this line :)
# Predict labels using your model
eval_dataset['preds'] = eval_dataset['text'].apply(lambda text: predict_label(text, model)) # predict_label is a function that you need to define for your model
# now drop the columns that are not needed, i.e. 'text', 'metadata' and 'dataset_source'
df_eval = df_eval.drop(columns=['text', 'metadata', 'dataset_source'])
df_eval.to_csv('your_model_name.csv')
# submit your results: 'your_model_name.csv' to the leaderboard
```
"""
gr.Markdown("## Upload your results to the leaderboard 🚀")
gr.Markdown("### Submission guidelines: Run the test dataset on your model and save the results in a CSV file. Bellow a code snippet to help you with that.")
gr.Markdown("### Nota Bene: The One-vs-All leaderboard evaluation is currently unavailable with the csv upload but will be implemented soon. Stay tuned!")
gr.Markdown(code_snippet)
uploaded_model_name = gr.Textbox(label="Model name", placeholder='Your model/team name')
file = gr.File(label="Upload your results")
upload_button = gr.Button("Upload")
# Status message area
status_message = gr.Markdown(value="")
def update_status_message():
return "### **⚠️Evaluating... Please wait...**"
upload_button.click(update_status_message, outputs=[status_message])
upload_button.click(process_results_file, inputs=[file, uploaded_model_name], outputs=[leaderboard_table, status_message])
# Update multilangual table when any input changes
country_selector.change(
update_leaderboard_multilingual,
inputs=[country_selector, metric_checkboxes],
outputs=leaderboard_table
)
metric_checkboxes.change(
update_leaderboard_multilingual,
inputs=[country_selector, metric_checkboxes],
outputs=leaderboard_table
)
# Update binary table when any input changes
target_language_selector.change(
update_leaderboard_one_vs_all,
inputs=[target_language_selector, languages_checkboxes],
outputs=[binary_leaderboard_table, languages_checkboxes]
)
languages_checkboxes.change(
update_leaderboard_one_vs_all,
inputs=[target_language_selector, languages_checkboxes],
outputs=[binary_leaderboard_table, languages_checkboxes]
)
# Define load event to run at startup
app.load(
update_leaderboard_one_vs_all,
inputs=[target_language_selector, languages_checkboxes],
outputs=[binary_leaderboard_table, languages_checkboxes]
)
app.load(
update_leaderboard_multilingual,
inputs=[country_selector, metric_checkboxes],
outputs=leaderboard_table
)
app.launch(allowed_paths=[base_path])
|