Hugging Face's logo

Spaces:
atlasia
/
Open-Arabic-Dialect-Identification-Leaderboard
Running

App Files Community
BounharAbdelaziz commited on
Commit
be25a4c
·
verified ·
1 Parent(s): 3cfe38b

implemented multilingual, eval from link and csv

Browse files
Files changed (6) hide show
  1. .gitattributes +1 -0
  2. app.py +205 -0
  3. darija_leaderboard_binary.json +3 -0
  4. darija_leaderboard_multilingual.json +1378 -0
  5. open_arabic_lid_arena.png +3 -0
  6. utils.py +488 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ open_arabic_lid_arena.png filter=lfs diff=lfs merge=lfs -text
app.py ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import pandas as pd
3
+ from utils import (
4
+ update_leaderboard_multilingual,
5
+ handle_evaluation,
6
+ process_results_file,
7
+ create_html_image,
8
+ )
9
+ from datasets import load_dataset
10
+ import gradio as gr
11
+
12
+ if __name__ == "__main__":
13
+ # Evaluation dataset path
14
+ DATA_PATH = "atlasia/No-Arabic-Dialect-Left-Behind-Filtered-Balanced"
15
+ # All Metrics
16
+ metrics = [
17
+ 'f1_score',
18
+ 'precision',
19
+ 'recall',
20
+ 'specificity',
21
+ 'false_positive_rate',
22
+ 'false_negative_rate',
23
+ 'negative_predictive_value',
24
+ 'n_test_samples',
25
+ ]
26
+ # Default metrics to display
27
+ default_metrics = [
28
+ 'f1_score',
29
+ 'precision',
30
+ 'recall',
31
+ 'false_positive_rate',
32
+ 'false_negative_rate'
33
+ ]
34
+ # default language to display
35
+ default_languages = [
36
+ 'Morocco',
37
+ 'MSA',
38
+ 'Egypt',
39
+ 'Algeria',
40
+ 'Tunisia',
41
+ 'Levantine',
42
+ ]
43
+
44
+ # Load test dataset
45
+ test_dataset = load_dataset(DATA_PATH, split='test')
46
+ # Supported dialects
47
+ supported_dialects = list(test_dataset.unique("dialect")) + ['All']
48
+
49
+ with gr.Blocks() as app:
50
+ base_path = os.path.dirname(__file__)
51
+ local_image_path = os.path.join(base_path, 'open_arabic_lid_arena.png')
52
+
53
+ gr.HTML(create_html_image(local_image_path))
54
+ gr.Markdown("# 🏅 Open Arabic Dialect Identification Leaderboard")
55
+
56
+ # Multilingual model leaderboard
57
+ with gr.Tab("Multilingual model leaderboard"):
58
+ gr.Markdown("""
59
+ Complete leaderboard across multiple arabic dialects.
60
+ Compare the performance of different models across various metrics such as FNR, FPR, and other clasical metrics.
61
+ """
62
+ )
63
+
64
+ with gr.Row():
65
+ with gr.Column(scale=1):
66
+ gr.Markdown("### Select country to display")
67
+ country_selector = gr.Dropdown(
68
+ choices=supported_dialects,
69
+ value='Morocco', # Default to Morocco of course
70
+ label="Country"
71
+ )
72
+
73
+ with gr.Column(scale=2):
74
+ gr.Markdown("### Select metrics to display")
75
+ metric_checkboxes = gr.CheckboxGroup(
76
+ choices=metrics,
77
+ value=default_metrics,
78
+ label="Metrics"
79
+ )
80
+
81
+ with gr.Row():
82
+ leaderboard_table = gr.DataFrame(
83
+ interactive=False
84
+ )
85
+
86
+ gr.Markdown("</br>")
87
+
88
+ gr.Markdown("## Contribute to the Leaderboard")
89
+ gr.Markdown("""
90
+ We welcome contributions from the community!
91
+ If you have a model that you would like to see on the leaderboard, please use the 'Evaluate a model' or 'Upload your results' tabs to submit your model's performance.
92
+ Let's work together to improve Arabic dialect identification! 🚀
93
+ """)
94
+
95
+ # Binary model leaderboard
96
+ with gr.Tab("One-vs-All leaderboard"):
97
+
98
+ gr.Markdown("""
99
+ A kind of one-vs-all approach for evaluating LID models across multiple arabic dialects.
100
+ Computes the `false_positive_rate` of different models for a given target language.
101
+ This should help you understand how well a model can identify a specific dialect by
102
+ showing how often it misclassifies other dialects as the target dialect.
103
+ """
104
+ )
105
+
106
+ with gr.Column(scale=1):
107
+ gr.Markdown("### Select target language")
108
+ target_language_selector = gr.Dropdown(
109
+ choices=supported_dialects,
110
+ value='Morocco', # Default to Morocco of course
111
+ label="Target Language"
112
+ )
113
+
114
+ with gr.Column(scale=2):
115
+ gr.Markdown("### Select Languages to display")
116
+ languages_checkboxes = gr.CheckboxGroup(
117
+ choices=supported_dialects,
118
+ value=default_languages,
119
+ label="Languages"
120
+ )
121
+
122
+ with gr.Row():
123
+ binary_leaderboard_table = gr.DataFrame(
124
+ interactive=False
125
+ )
126
+
127
+
128
+ with gr.Tab("Evaluate a model"):
129
+ gr.Markdown("Suggest a model to evaluate 🤗 (Supports only **Fasttext** models as SfayaLID, GlotLID, OpenLID, etc.)")
130
+ gr.Markdown("For other models, you are welcome to **submit your results** through the upload section.")
131
+
132
+ model_path = gr.Textbox(label="Model Path", placeholder='path/to/model')
133
+ model_path_bin = gr.Textbox(label=".bin filename", placeholder='model.bin')
134
+ gr.Markdown("### **⚠️ To ensure correct results, tick this when the model's labels are the iso_codes**")
135
+ use_mapping = gr.Checkbox(label="Does not map to country")
136
+ eval_button = gr.Button("Evaluate", value=False) # Initially disabled
137
+
138
+ eval_button.click(handle_evaluation, inputs=[model_path, model_path_bin, use_mapping], outputs=[leaderboard_table])
139
+
140
+ with gr.Tab("Upload your results"):
141
+
142
+ # Define a code block to display
143
+ code_snippet = """
144
+ ```python
145
+
146
+ # Load your model
147
+ model = ... # Load your model here
148
+
149
+ # Load evaluation benchmark
150
+ eval_dataset = load_dataset("atlasia/No-Arabic-Dialect-Left-Behind-Filtered-Balanced", split='test').to_pandas() # do not change this line :)
151
+
152
+ # Predict labels using your model
153
+ eval_dataset['preds'] = eval_dataset['text'].apply(lambda text: predict_label(text, model)) # predict_label is a function that you need to define for your model
154
+
155
+ # now drop the columns that are not needed, i.e. 'text', 'metadata' and 'dataset_source'
156
+ df_eval = df_eval.drop(columns=['text', 'metadata', 'dataset_source'])
157
+ df_eval.to_csv('your_model_name.csv')
158
+
159
+ # submit your results: 'your_model_name.csv' to the leaderboard
160
+
161
+ ```
162
+ """
163
+ gr.Markdown("## Upload your results to the leaderboard 🚀")
164
+ gr.Markdown("### Submission guidelines: Run the test dataset on your model and save the results in a CSV file. Bellow a code snippet to help you with that.")
165
+ gr.Markdown(code_snippet)
166
+
167
+ uploaded_model_name = gr.Textbox(label="Model name", placeholder='Your model/team name')
168
+ file = gr.File(label="Upload your results")
169
+ upload_button = gr.Button("Upload")
170
+ upload_button.click(process_results_file, inputs=[file, uploaded_model_name], outputs=[leaderboard_table])
171
+
172
+ # Update multilangual table when any input changes
173
+ country_selector.change(
174
+ update_leaderboard_multilingual,
175
+ inputs=[country_selector, metric_checkboxes],
176
+ outputs=leaderboard_table
177
+ )
178
+
179
+ metric_checkboxes.change(
180
+ update_leaderboard_multilingual,
181
+ inputs=[country_selector, metric_checkboxes],
182
+ outputs=leaderboard_table
183
+ )
184
+
185
+ # Update binary table when any input changes
186
+ target_language_selector.change(
187
+ update_leaderboard_multilingual,
188
+ inputs=[country_selector, metric_checkboxes],
189
+ outputs=leaderboard_table
190
+ )
191
+
192
+ languages_checkboxes.change(
193
+ update_leaderboard_multilingual,
194
+ inputs=[country_selector, metric_checkboxes],
195
+ outputs=leaderboard_table
196
+ )
197
+
198
+ # Define load event to run at startup
199
+ app.load(
200
+ update_leaderboard_multilingual,
201
+ inputs=[country_selector, metric_checkboxes],
202
+ outputs=leaderboard_table
203
+ )
204
+
205
+ app.launch(allowed_paths=[base_path])
darija_leaderboard_binary.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ [
2
+
3
+ ]
darija_leaderboard_multilingual.json ADDED
@@ -0,0 +1,1378 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "MSA": {
4
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
5
+ "f1_score": 0.9749,
6
+ "precision": 0.9908,
7
+ "recall": 0.9594,
8
+ "specificity": 0.9942,
9
+ "false_positive_rate": 0.0058,
10
+ "false_negative_rate": 0.0406,
11
+ "negative_predictive_value": 0.974,
12
+ "n_test_samples": 54390
13
+ },
14
+ "cis-lmu/glotlid/model.bin": {
15
+ "f1_score": 0.9554,
16
+ "precision": 0.9252,
17
+ "recall": 0.9876,
18
+ "specificity": 0.9478,
19
+ "false_positive_rate": 0.0522,
20
+ "false_negative_rate": 0.0124,
21
+ "negative_predictive_value": 0.9915,
22
+ "n_test_samples": 54390
23
+ },
24
+ "laurievb/OpenLID/model.bin": {
25
+ "f1_score": 0.9264,
26
+ "precision": 0.9359,
27
+ "recall": 0.9172,
28
+ "specificity": 0.9589,
29
+ "false_positive_rate": 0.0411,
30
+ "false_negative_rate": 0.0828,
31
+ "negative_predictive_value": 0.9465,
32
+ "n_test_samples": 54390
33
+ },
34
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
35
+ "f1_score": 0.0,
36
+ "precision": 0.0,
37
+ "recall": 0.0,
38
+ "specificity": 1.0,
39
+ "false_positive_rate": 0.0,
40
+ "false_negative_rate": 1.0,
41
+ "negative_predictive_value": 0.6047,
42
+ "n_test_samples": 54390
43
+ }
44
+ }
45
+ },
46
+ {
47
+ "Pakistan": {
48
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
49
+ "f1_score": 0.9963,
50
+ "precision": 0.9999,
51
+ "recall": 0.9927,
52
+ "specificity": 0.9999,
53
+ "false_positive_rate": 0.0001,
54
+ "false_negative_rate": 0.0073,
55
+ "negative_predictive_value": 0.9958,
56
+ "n_test_samples": 50000
57
+ },
58
+ "cis-lmu/glotlid/model.bin": {
59
+ "f1_score": 0.999,
60
+ "precision": 0.9989,
61
+ "recall": 0.9991,
62
+ "specificity": 0.9994,
63
+ "false_positive_rate": 0.0006,
64
+ "false_negative_rate": 0.0009,
65
+ "negative_predictive_value": 0.9995,
66
+ "n_test_samples": 50000
67
+ },
68
+ "laurievb/OpenLID/model.bin": {
69
+ "f1_score": 0.9927,
70
+ "precision": 0.9928,
71
+ "recall": 0.9925,
72
+ "specificity": 0.9959,
73
+ "false_positive_rate": 0.0041,
74
+ "false_negative_rate": 0.0075,
75
+ "negative_predictive_value": 0.9957,
76
+ "n_test_samples": 50000
77
+ },
78
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
79
+ "f1_score": 0.0,
80
+ "precision": 0.0,
81
+ "recall": 0.0,
82
+ "specificity": 1.0,
83
+ "false_positive_rate": 0.0,
84
+ "false_negative_rate": 1.0,
85
+ "negative_predictive_value": 0.6366,
86
+ "n_test_samples": 50000
87
+ }
88
+ }
89
+ },
90
+ {
91
+ "Morocco": {
92
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
93
+ "f1_score": 0.8789,
94
+ "precision": 0.8624,
95
+ "recall": 0.8961,
96
+ "specificity": 0.9771,
97
+ "false_positive_rate": 0.0229,
98
+ "false_negative_rate": 0.1039,
99
+ "negative_predictive_value": 0.9832,
100
+ "n_test_samples": 19005
101
+ },
102
+ "cis-lmu/glotlid/model.bin": {
103
+ "f1_score": 0.7172,
104
+ "precision": 0.9038,
105
+ "recall": 0.5945,
106
+ "specificity": 0.9899,
107
+ "false_positive_rate": 0.0101,
108
+ "false_negative_rate": 0.4055,
109
+ "negative_predictive_value": 0.9384,
110
+ "n_test_samples": 19005
111
+ },
112
+ "laurievb/OpenLID/model.bin": {
113
+ "f1_score": 0.6146,
114
+ "precision": 0.7279,
115
+ "recall": 0.5318,
116
+ "specificity": 0.9681,
117
+ "false_positive_rate": 0.0319,
118
+ "false_negative_rate": 0.4682,
119
+ "negative_predictive_value": 0.9281,
120
+ "n_test_samples": 19005
121
+ },
122
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
123
+ "f1_score": 0.8986,
124
+ "precision": 0.9166,
125
+ "recall": 0.8812,
126
+ "specificity": 0.9871,
127
+ "false_positive_rate": 0.0129,
128
+ "false_negative_rate": 0.1188,
129
+ "negative_predictive_value": 0.9811,
130
+ "n_test_samples": 19005
131
+ }
132
+ }
133
+ },
134
+ {
135
+ "Egypt": {
136
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
137
+ "f1_score": 0.5929,
138
+ "precision": 0.5835,
139
+ "recall": 0.6025,
140
+ "specificity": 0.993,
141
+ "false_positive_rate": 0.007,
142
+ "false_negative_rate": 0.3975,
143
+ "negative_predictive_value": 0.9935,
144
+ "n_test_samples": 2204
145
+ },
146
+ "cis-lmu/glotlid/model.bin": {
147
+ "f1_score": 0.6028,
148
+ "precision": 0.4837,
149
+ "recall": 0.7999,
150
+ "specificity": 0.9861,
151
+ "false_positive_rate": 0.0139,
152
+ "false_negative_rate": 0.2001,
153
+ "negative_predictive_value": 0.9967,
154
+ "n_test_samples": 2204
155
+ },
156
+ "laurievb/OpenLID/model.bin": {
157
+ "f1_score": 0.4094,
158
+ "precision": 0.2663,
159
+ "recall": 0.8843,
160
+ "specificity": 0.9603,
161
+ "false_positive_rate": 0.0397,
162
+ "false_negative_rate": 0.1157,
163
+ "negative_predictive_value": 0.998,
164
+ "n_test_samples": 2204
165
+ },
166
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
167
+ "f1_score": 0.0,
168
+ "precision": 0.0,
169
+ "recall": 0.0,
170
+ "specificity": 1.0,
171
+ "false_positive_rate": 0.0,
172
+ "false_negative_rate": 1.0,
173
+ "negative_predictive_value": 0.984,
174
+ "n_test_samples": 2204
175
+ }
176
+ }
177
+ },
178
+ {
179
+ "Palestine": {
180
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
181
+ "f1_score": 0.2511,
182
+ "precision": 0.3352,
183
+ "recall": 0.2007,
184
+ "specificity": 0.9957,
185
+ "false_positive_rate": 0.0043,
186
+ "false_negative_rate": 0.7993,
187
+ "negative_predictive_value": 0.9914,
188
+ "n_test_samples": 1465
189
+ },
190
+ "cis-lmu/glotlid/model.bin": {
191
+ "f1_score": 0.0,
192
+ "precision": 0.0,
193
+ "recall": 0.0,
194
+ "specificity": 1.0,
195
+ "false_positive_rate": 0.0,
196
+ "false_negative_rate": 1.0,
197
+ "negative_predictive_value": 0.9894,
198
+ "n_test_samples": 1465
199
+ },
200
+ "laurievb/OpenLID/model.bin": {
201
+ "f1_score": 0.0,
202
+ "precision": 0.0,
203
+ "recall": 0.0,
204
+ "specificity": 1.0,
205
+ "false_positive_rate": 0.0,
206
+ "false_negative_rate": 1.0,
207
+ "negative_predictive_value": 0.9894,
208
+ "n_test_samples": 1465
209
+ },
210
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
211
+ "f1_score": 0.0,
212
+ "precision": 0.0,
213
+ "recall": 0.0,
214
+ "specificity": 1.0,
215
+ "false_positive_rate": 0.0,
216
+ "false_negative_rate": 1.0,
217
+ "negative_predictive_value": 0.9894,
218
+ "n_test_samples": 1465
219
+ }
220
+ }
221
+ },
222
+ {
223
+ "Levantine": {
224
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
225
+ "f1_score": 0.1723,
226
+ "precision": 0.1386,
227
+ "recall": 0.2275,
228
+ "specificity": 0.9854,
229
+ "false_positive_rate": 0.0146,
230
+ "false_negative_rate": 0.7725,
231
+ "negative_predictive_value": 0.992,
232
+ "n_test_samples": 1402
233
+ },
234
+ "cis-lmu/glotlid/model.bin": {
235
+ "f1_score": 0.1171,
236
+ "precision": 0.073,
237
+ "recall": 0.2953,
238
+ "specificity": 0.9614,
239
+ "false_positive_rate": 0.0386,
240
+ "false_negative_rate": 0.7047,
241
+ "negative_predictive_value": 0.9925,
242
+ "n_test_samples": 1402
243
+ },
244
+ "laurievb/OpenLID/model.bin": {
245
+ "f1_score": 0.1029,
246
+ "precision": 0.0645,
247
+ "recall": 0.2532,
248
+ "specificity": 0.9622,
249
+ "false_positive_rate": 0.0378,
250
+ "false_negative_rate": 0.7468,
251
+ "negative_predictive_value": 0.9921,
252
+ "n_test_samples": 1402
253
+ },
254
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
255
+ "f1_score": 0.0,
256
+ "precision": 0.0,
257
+ "recall": 0.0,
258
+ "specificity": 1.0,
259
+ "false_positive_rate": 0.0,
260
+ "false_negative_rate": 1.0,
261
+ "negative_predictive_value": 0.9898,
262
+ "n_test_samples": 1402
263
+ }
264
+ }
265
+ },
266
+ {
267
+ "Saudi": {
268
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
269
+ "f1_score": 0.4382,
270
+ "precision": 0.3474,
271
+ "recall": 0.5932,
272
+ "specificity": 0.9887,
273
+ "false_positive_rate": 0.0113,
274
+ "false_negative_rate": 0.4068,
275
+ "negative_predictive_value": 0.9958,
276
+ "n_test_samples": 1384
277
+ },
278
+ "cis-lmu/glotlid/model.bin": {
279
+ "f1_score": 0.3893,
280
+ "precision": 0.2692,
281
+ "recall": 0.703,
282
+ "specificity": 0.9806,
283
+ "false_positive_rate": 0.0194,
284
+ "false_negative_rate": 0.297,
285
+ "negative_predictive_value": 0.9969,
286
+ "n_test_samples": 1384
287
+ },
288
+ "laurievb/OpenLID/model.bin": {
289
+ "f1_score": 0.3436,
290
+ "precision": 0.2381,
291
+ "recall": 0.6171,
292
+ "specificity": 0.9799,
293
+ "false_positive_rate": 0.0201,
294
+ "false_negative_rate": 0.3829,
295
+ "negative_predictive_value": 0.996,
296
+ "n_test_samples": 1384
297
+ },
298
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
299
+ "f1_score": 0.0,
300
+ "precision": 0.0,
301
+ "recall": 0.0,
302
+ "specificity": 1.0,
303
+ "false_positive_rate": 0.0,
304
+ "false_negative_rate": 1.0,
305
+ "negative_predictive_value": 0.9899,
306
+ "n_test_samples": 1384
307
+ }
308
+ }
309
+ },
310
+ {
311
+ "Jordan": {
312
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
313
+ "f1_score": 0.2726,
314
+ "precision": 0.4203,
315
+ "recall": 0.2017,
316
+ "specificity": 0.9972,
317
+ "false_positive_rate": 0.0028,
318
+ "false_negative_rate": 0.7983,
319
+ "negative_predictive_value": 0.992,
320
+ "n_test_samples": 1373
321
+ },
322
+ "cis-lmu/glotlid/model.bin": {
323
+ "f1_score": 0.0,
324
+ "precision": 0.0,
325
+ "recall": 0.0,
326
+ "specificity": 1.0,
327
+ "false_positive_rate": 0.0,
328
+ "false_negative_rate": 1.0,
329
+ "negative_predictive_value": 0.99,
330
+ "n_test_samples": 1373
331
+ },
332
+ "laurievb/OpenLID/model.bin": {
333
+ "f1_score": 0.0,
334
+ "precision": 0.0,
335
+ "recall": 0.0,
336
+ "specificity": 1.0,
337
+ "false_positive_rate": 0.0,
338
+ "false_negative_rate": 1.0,
339
+ "negative_predictive_value": 0.99,
340
+ "n_test_samples": 1373
341
+ },
342
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
343
+ "f1_score": 0.0,
344
+ "precision": 0.0,
345
+ "recall": 0.0,
346
+ "specificity": 1.0,
347
+ "false_positive_rate": 0.0,
348
+ "false_negative_rate": 1.0,
349
+ "negative_predictive_value": 0.99,
350
+ "n_test_samples": 1373
351
+ }
352
+ }
353
+ },
354
+ {
355
+ "Algeria": {
356
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
357
+ "f1_score": 0.5221,
358
+ "precision": 0.5849,
359
+ "recall": 0.4714,
360
+ "specificity": 0.9974,
361
+ "false_positive_rate": 0.0026,
362
+ "false_negative_rate": 0.5286,
363
+ "negative_predictive_value": 0.9959,
364
+ "n_test_samples": 1067
365
+ },
366
+ "cis-lmu/glotlid/model.bin": {
367
+ "f1_score": 0.1235,
368
+ "precision": 0.2751,
369
+ "recall": 0.0797,
370
+ "specificity": 0.9984,
371
+ "false_positive_rate": 0.0016,
372
+ "false_negative_rate": 0.9203,
373
+ "negative_predictive_value": 0.9928,
374
+ "n_test_samples": 1067
375
+ },
376
+ "laurievb/OpenLID/model.bin": {
377
+ "f1_score": 0.0,
378
+ "precision": 0.0,
379
+ "recall": 0.0,
380
+ "specificity": 1.0,
381
+ "false_positive_rate": 0.0,
382
+ "false_negative_rate": 1.0,
383
+ "negative_predictive_value": 0.9922,
384
+ "n_test_samples": 1067
385
+ },
386
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
387
+ "f1_score": 0.0,
388
+ "precision": 0.0,
389
+ "recall": 0.0,
390
+ "specificity": 1.0,
391
+ "false_positive_rate": 0.0,
392
+ "false_negative_rate": 1.0,
393
+ "negative_predictive_value": 0.9922,
394
+ "n_test_samples": 1067
395
+ }
396
+ }
397
+ },
398
+ {
399
+ "UAE": {
400
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
401
+ "f1_score": 0.3452,
402
+ "precision": 0.4207,
403
+ "recall": 0.2926,
404
+ "specificity": 0.9971,
405
+ "false_positive_rate": 0.0029,
406
+ "false_negative_rate": 0.7074,
407
+ "negative_predictive_value": 0.9948,
408
+ "n_test_samples": 998
409
+ },
410
+ "cis-lmu/glotlid/model.bin": {
411
+ "f1_score": 0.0,
412
+ "precision": 0.0,
413
+ "recall": 0.0,
414
+ "specificity": 1.0,
415
+ "false_positive_rate": 0.0,
416
+ "false_negative_rate": 1.0,
417
+ "negative_predictive_value": 0.9927,
418
+ "n_test_samples": 998
419
+ },
420
+ "laurievb/OpenLID/model.bin": {
421
+ "f1_score": 0.0,
422
+ "precision": 0.0,
423
+ "recall": 0.0,
424
+ "specificity": 1.0,
425
+ "false_positive_rate": 0.0,
426
+ "false_negative_rate": 1.0,
427
+ "negative_predictive_value": 0.9927,
428
+ "n_test_samples": 998
429
+ },
430
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
431
+ "f1_score": 0.0,
432
+ "precision": 0.0,
433
+ "recall": 0.0,
434
+ "specificity": 1.0,
435
+ "false_positive_rate": 0.0,
436
+ "false_negative_rate": 1.0,
437
+ "negative_predictive_value": 0.9927,
438
+ "n_test_samples": 998
439
+ }
440
+ }
441
+ },
442
+ {
443
+ "Mauritania": {
444
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
445
+ "f1_score": 0.5343,
446
+ "precision": 0.944,
447
+ "recall": 0.3726,
448
+ "specificity": 0.9998,
449
+ "false_positive_rate": 0.0002,
450
+ "false_negative_rate": 0.6274,
451
+ "negative_predictive_value": 0.9957,
452
+ "n_test_samples": 950
453
+ },
454
+ "cis-lmu/glotlid/model.bin": {
455
+ "f1_score": 0.0,
456
+ "precision": 0.0,
457
+ "recall": 0.0,
458
+ "specificity": 1.0,
459
+ "false_positive_rate": 0.0,
460
+ "false_negative_rate": 1.0,
461
+ "negative_predictive_value": 0.9931,
462
+ "n_test_samples": 950
463
+ },
464
+ "laurievb/OpenLID/model.bin": {
465
+ "f1_score": 0.0,
466
+ "precision": 0.0,
467
+ "recall": 0.0,
468
+ "specificity": 1.0,
469
+ "false_positive_rate": 0.0,
470
+ "false_negative_rate": 1.0,
471
+ "negative_predictive_value": 0.9931,
472
+ "n_test_samples": 950
473
+ },
474
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
475
+ "f1_score": 0.0,
476
+ "precision": 0.0,
477
+ "recall": 0.0,
478
+ "specificity": 1.0,
479
+ "false_positive_rate": 0.0,
480
+ "false_negative_rate": 1.0,
481
+ "negative_predictive_value": 0.9931,
482
+ "n_test_samples": 950
483
+ }
484
+ }
485
+ },
486
+ {
487
+ "Yemen": {
488
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
489
+ "f1_score": 0.223,
490
+ "precision": 0.5619,
491
+ "recall": 0.1391,
492
+ "specificity": 0.9993,
493
+ "false_positive_rate": 0.0007,
494
+ "false_negative_rate": 0.8609,
495
+ "negative_predictive_value": 0.9943,
496
+ "n_test_samples": 913
497
+ },
498
+ "cis-lmu/glotlid/model.bin": {
499
+ "f1_score": 0.0,
500
+ "precision": 0.0,
501
+ "recall": 0.0,
502
+ "specificity": 1.0,
503
+ "false_positive_rate": 0.0,
504
+ "false_negative_rate": 1.0,
505
+ "negative_predictive_value": 0.9934,
506
+ "n_test_samples": 913
507
+ },
508
+ "laurievb/OpenLID/model.bin": {
509
+ "f1_score": 0.0,
510
+ "precision": 0.0,
511
+ "recall": 0.0,
512
+ "specificity": 1.0,
513
+ "false_positive_rate": 0.0,
514
+ "false_negative_rate": 1.0,
515
+ "negative_predictive_value": 0.9934,
516
+ "n_test_samples": 913
517
+ },
518
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
519
+ "f1_score": 0.0,
520
+ "precision": 0.0,
521
+ "recall": 0.0,
522
+ "specificity": 1.0,
523
+ "false_positive_rate": 0.0,
524
+ "false_negative_rate": 1.0,
525
+ "negative_predictive_value": 0.9934,
526
+ "n_test_samples": 913
527
+ }
528
+ }
529
+ },
530
+ {
531
+ "Syria": {
532
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
533
+ "f1_score": 0.1965,
534
+ "precision": 0.2126,
535
+ "recall": 0.1827,
536
+ "specificity": 0.9971,
537
+ "false_positive_rate": 0.0029,
538
+ "false_negative_rate": 0.8173,
539
+ "negative_predictive_value": 0.9965,
540
+ "n_test_samples": 591
541
+ },
542
+ "cis-lmu/glotlid/model.bin": {
543
+ "f1_score": 0.0,
544
+ "precision": 0.0,
545
+ "recall": 0.0,
546
+ "specificity": 1.0,
547
+ "false_positive_rate": 0.0,
548
+ "false_negative_rate": 1.0,
549
+ "negative_predictive_value": 0.9957,
550
+ "n_test_samples": 591
551
+ },
552
+ "laurievb/OpenLID/model.bin": {
553
+ "f1_score": 0.0,
554
+ "precision": 0.0,
555
+ "recall": 0.0,
556
+ "specificity": 1.0,
557
+ "false_positive_rate": 0.0,
558
+ "false_negative_rate": 1.0,
559
+ "negative_predictive_value": 0.9957,
560
+ "n_test_samples": 591
561
+ },
562
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
563
+ "f1_score": 0.0,
564
+ "precision": 0.0,
565
+ "recall": 0.0,
566
+ "specificity": 1.0,
567
+ "false_positive_rate": 0.0,
568
+ "false_negative_rate": 1.0,
569
+ "negative_predictive_value": 0.9957,
570
+ "n_test_samples": 591
571
+ }
572
+ }
573
+ },
574
+ {
575
+ "Lebanon": {
576
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
577
+ "f1_score": 0.2699,
578
+ "precision": 0.2133,
579
+ "recall": 0.3675,
580
+ "specificity": 0.9967,
581
+ "false_positive_rate": 0.0033,
582
+ "false_negative_rate": 0.6325,
583
+ "negative_predictive_value": 0.9985,
584
+ "n_test_samples": 332
585
+ },
586
+ "cis-lmu/glotlid/model.bin": {
587
+ "f1_score": 0.0,
588
+ "precision": 0.0,
589
+ "recall": 0.0,
590
+ "specificity": 1.0,
591
+ "false_positive_rate": 0.0,
592
+ "false_negative_rate": 1.0,
593
+ "negative_predictive_value": 0.9976,
594
+ "n_test_samples": 332
595
+ },
596
+ "laurievb/OpenLID/model.bin": {
597
+ "f1_score": 0.0,
598
+ "precision": 0.0,
599
+ "recall": 0.0,
600
+ "specificity": 1.0,
601
+ "false_positive_rate": 0.0,
602
+ "false_negative_rate": 1.0,
603
+ "negative_predictive_value": 0.9976,
604
+ "n_test_samples": 332
605
+ },
606
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
607
+ "f1_score": 0.0,
608
+ "precision": 0.0,
609
+ "recall": 0.0,
610
+ "specificity": 1.0,
611
+ "false_positive_rate": 0.0,
612
+ "false_negative_rate": 1.0,
613
+ "negative_predictive_value": 0.9976,
614
+ "n_test_samples": 332
615
+ }
616
+ }
617
+ },
618
+ {
619
+ "Qatar": {
620
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
621
+ "f1_score": 0.3145,
622
+ "precision": 0.2315,
623
+ "recall": 0.4905,
624
+ "specificity": 0.9975,
625
+ "false_positive_rate": 0.0025,
626
+ "false_negative_rate": 0.5095,
627
+ "negative_predictive_value": 0.9992,
628
+ "n_test_samples": 210
629
+ },
630
+ "cis-lmu/glotlid/model.bin": {
631
+ "f1_score": 0.0,
632
+ "precision": 0.0,
633
+ "recall": 0.0,
634
+ "specificity": 1.0,
635
+ "false_positive_rate": 0.0,
636
+ "false_negative_rate": 1.0,
637
+ "negative_predictive_value": 0.9985,
638
+ "n_test_samples": 210
639
+ },
640
+ "laurievb/OpenLID/model.bin": {
641
+ "f1_score": 0.0,
642
+ "precision": 0.0,
643
+ "recall": 0.0,
644
+ "specificity": 1.0,
645
+ "false_positive_rate": 0.0,
646
+ "false_negative_rate": 1.0,
647
+ "negative_predictive_value": 0.9985,
648
+ "n_test_samples": 210
649
+ },
650
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
651
+ "f1_score": 0.0,
652
+ "precision": 0.0,
653
+ "recall": 0.0,
654
+ "specificity": 1.0,
655
+ "false_positive_rate": 0.0,
656
+ "false_negative_rate": 1.0,
657
+ "negative_predictive_value": 0.9985,
658
+ "n_test_samples": 210
659
+ }
660
+ }
661
+ },
662
+ {
663
+ "Iraq": {
664
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
665
+ "f1_score": 0.4075,
666
+ "precision": 0.3884,
667
+ "recall": 0.4286,
668
+ "specificity": 0.999,
669
+ "false_positive_rate": 0.001,
670
+ "false_negative_rate": 0.5714,
671
+ "negative_predictive_value": 0.9992,
672
+ "n_test_samples": 203
673
+ },
674
+ "cis-lmu/glotlid/model.bin": {
675
+ "f1_score": 0.0098,
676
+ "precision": 1.0,
677
+ "recall": 0.0049,
678
+ "specificity": 1.0,
679
+ "false_positive_rate": 0.0,
680
+ "false_negative_rate": 0.9951,
681
+ "negative_predictive_value": 0.9985,
682
+ "n_test_samples": 203
683
+ },
684
+ "laurievb/OpenLID/model.bin": {
685
+ "f1_score": 0.0,
686
+ "precision": 0.0,
687
+ "recall": 0.0,
688
+ "specificity": 1.0,
689
+ "false_positive_rate": 0.0,
690
+ "false_negative_rate": 1.0,
691
+ "negative_predictive_value": 0.9985,
692
+ "n_test_samples": 203
693
+ },
694
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
695
+ "f1_score": 0.0,
696
+ "precision": 0.0,
697
+ "recall": 0.0,
698
+ "specificity": 1.0,
699
+ "false_positive_rate": 0.0,
700
+ "false_negative_rate": 1.0,
701
+ "negative_predictive_value": 0.9985,
702
+ "n_test_samples": 203
703
+ }
704
+ }
705
+ },
706
+ {
707
+ "Libya": {
708
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
709
+ "f1_score": 0.357,
710
+ "precision": 0.2621,
711
+ "recall": 0.5596,
712
+ "specificity": 0.9978,
713
+ "false_positive_rate": 0.0022,
714
+ "false_negative_rate": 0.4404,
715
+ "negative_predictive_value": 0.9994,
716
+ "n_test_samples": 193
717
+ },
718
+ "cis-lmu/glotlid/model.bin": {
719
+ "f1_score": 0.0,
720
+ "precision": 0.0,
721
+ "recall": 0.0,
722
+ "specificity": 1.0,
723
+ "false_positive_rate": 0.0,
724
+ "false_negative_rate": 1.0,
725
+ "negative_predictive_value": 0.9986,
726
+ "n_test_samples": 193
727
+ },
728
+ "laurievb/OpenLID/model.bin": {
729
+ "f1_score": 0.0,
730
+ "precision": 0.0,
731
+ "recall": 0.0,
732
+ "specificity": 1.0,
733
+ "false_positive_rate": 0.0,
734
+ "false_negative_rate": 1.0,
735
+ "negative_predictive_value": 0.9986,
736
+ "n_test_samples": 193
737
+ },
738
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
739
+ "f1_score": 0.0,
740
+ "precision": 0.0,
741
+ "recall": 0.0,
742
+ "specificity": 1.0,
743
+ "false_positive_rate": 0.0,
744
+ "false_negative_rate": 1.0,
745
+ "negative_predictive_value": 0.9986,
746
+ "n_test_samples": 193
747
+ }
748
+ }
749
+ },
750
+ {
751
+ "Tunisia": {
752
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
753
+ "f1_score": 0.1851,
754
+ "precision": 0.1089,
755
+ "recall": 0.6158,
756
+ "specificity": 0.993,
757
+ "false_positive_rate": 0.007,
758
+ "false_negative_rate": 0.3842,
759
+ "negative_predictive_value": 0.9995,
760
+ "n_test_samples": 190
761
+ },
762
+ "cis-lmu/glotlid/model.bin": {
763
+ "f1_score": 0.1143,
764
+ "precision": 0.0624,
765
+ "recall": 0.6737,
766
+ "specificity": 0.986,
767
+ "false_positive_rate": 0.014,
768
+ "false_negative_rate": 0.3263,
769
+ "negative_predictive_value": 0.9995,
770
+ "n_test_samples": 190
771
+ },
772
+ "laurievb/OpenLID/model.bin": {
773
+ "f1_score": 0.1045,
774
+ "precision": 0.0564,
775
+ "recall": 0.7053,
776
+ "specificity": 0.9837,
777
+ "false_positive_rate": 0.0163,
778
+ "false_negative_rate": 0.2947,
779
+ "negative_predictive_value": 0.9996,
780
+ "n_test_samples": 190
781
+ },
782
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
783
+ "f1_score": 0.0,
784
+ "precision": 0.0,
785
+ "recall": 0.0,
786
+ "specificity": 1.0,
787
+ "false_positive_rate": 0.0,
788
+ "false_negative_rate": 1.0,
789
+ "negative_predictive_value": 0.9986,
790
+ "n_test_samples": 190
791
+ }
792
+ }
793
+ },
794
+ {
795
+ "Oman": {
796
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
797
+ "f1_score": 0.1969,
798
+ "precision": 0.1391,
799
+ "recall": 0.3368,
800
+ "specificity": 0.9971,
801
+ "false_positive_rate": 0.0029,
802
+ "false_negative_rate": 0.6632,
803
+ "negative_predictive_value": 0.9991,
804
+ "n_test_samples": 190
805
+ },
806
+ "cis-lmu/glotlid/model.bin": {
807
+ "f1_score": 0.0,
808
+ "precision": 0.0,
809
+ "recall": 0.0,
810
+ "specificity": 1.0,
811
+ "false_positive_rate": 0.0,
812
+ "false_negative_rate": 1.0,
813
+ "negative_predictive_value": 0.9986,
814
+ "n_test_samples": 190
815
+ },
816
+ "laurievb/OpenLID/model.bin": {
817
+ "f1_score": 0.0,
818
+ "precision": 0.0,
819
+ "recall": 0.0,
820
+ "specificity": 1.0,
821
+ "false_positive_rate": 0.0,
822
+ "false_negative_rate": 1.0,
823
+ "negative_predictive_value": 0.9986,
824
+ "n_test_samples": 190
825
+ },
826
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
827
+ "f1_score": 0.0,
828
+ "precision": 0.0,
829
+ "recall": 0.0,
830
+ "specificity": 1.0,
831
+ "false_positive_rate": 0.0,
832
+ "false_negative_rate": 1.0,
833
+ "negative_predictive_value": 0.9986,
834
+ "n_test_samples": 190
835
+ }
836
+ }
837
+ },
838
+ {
839
+ "Kuwait": {
840
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
841
+ "f1_score": 0.29,
842
+ "precision": 0.2091,
843
+ "recall": 0.4728,
844
+ "specificity": 0.9976,
845
+ "false_positive_rate": 0.0024,
846
+ "false_negative_rate": 0.5272,
847
+ "negative_predictive_value": 0.9993,
848
+ "n_test_samples": 184
849
+ },
850
+ "cis-lmu/glotlid/model.bin": {
851
+ "f1_score": 0.0,
852
+ "precision": 0.0,
853
+ "recall": 0.0,
854
+ "specificity": 1.0,
855
+ "false_positive_rate": 0.0,
856
+ "false_negative_rate": 1.0,
857
+ "negative_predictive_value": 0.9987,
858
+ "n_test_samples": 184
859
+ },
860
+ "laurievb/OpenLID/model.bin": {
861
+ "f1_score": 0.0,
862
+ "precision": 0.0,
863
+ "recall": 0.0,
864
+ "specificity": 1.0,
865
+ "false_positive_rate": 0.0,
866
+ "false_negative_rate": 1.0,
867
+ "negative_predictive_value": 0.9987,
868
+ "n_test_samples": 184
869
+ },
870
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
871
+ "f1_score": 0.0,
872
+ "precision": 0.0,
873
+ "recall": 0.0,
874
+ "specificity": 1.0,
875
+ "false_positive_rate": 0.0,
876
+ "false_negative_rate": 1.0,
877
+ "negative_predictive_value": 0.9987,
878
+ "n_test_samples": 184
879
+ }
880
+ }
881
+ },
882
+ {
883
+ "Bahrain": {
884
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
885
+ "f1_score": 0.2045,
886
+ "precision": 0.2069,
887
+ "recall": 0.2022,
888
+ "specificity": 0.999,
889
+ "false_positive_rate": 0.001,
890
+ "false_negative_rate": 0.7978,
891
+ "negative_predictive_value": 0.999,
892
+ "n_test_samples": 178
893
+ },
894
+ "cis-lmu/glotlid/model.bin": {
895
+ "f1_score": 0.0,
896
+ "precision": 0.0,
897
+ "recall": 0.0,
898
+ "specificity": 1.0,
899
+ "false_positive_rate": 0.0,
900
+ "false_negative_rate": 1.0,
901
+ "negative_predictive_value": 0.9987,
902
+ "n_test_samples": 178
903
+ },
904
+ "laurievb/OpenLID/model.bin": {
905
+ "f1_score": 0.0,
906
+ "precision": 0.0,
907
+ "recall": 0.0,
908
+ "specificity": 1.0,
909
+ "false_positive_rate": 0.0,
910
+ "false_negative_rate": 1.0,
911
+ "negative_predictive_value": 0.9987,
912
+ "n_test_samples": 178
913
+ },
914
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
915
+ "f1_score": 0.0,
916
+ "precision": 0.0,
917
+ "recall": 0.0,
918
+ "specificity": 1.0,
919
+ "false_positive_rate": 0.0,
920
+ "false_negative_rate": 1.0,
921
+ "negative_predictive_value": 0.9987,
922
+ "n_test_samples": 178
923
+ }
924
+ }
925
+ },
926
+ {
927
+ "Sudan": {
928
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
929
+ "f1_score": 0.395,
930
+ "precision": 0.3198,
931
+ "recall": 0.5163,
932
+ "specificity": 0.9988,
933
+ "false_positive_rate": 0.0012,
934
+ "false_negative_rate": 0.4837,
935
+ "negative_predictive_value": 0.9995,
936
+ "n_test_samples": 153
937
+ },
938
+ "cis-lmu/glotlid/model.bin": {
939
+ "f1_score": 0.0,
940
+ "precision": 0.0,
941
+ "recall": 0.0,
942
+ "specificity": 1.0,
943
+ "false_positive_rate": 0.0,
944
+ "false_negative_rate": 1.0,
945
+ "negative_predictive_value": 0.9989,
946
+ "n_test_samples": 153
947
+ },
948
+ "laurievb/OpenLID/model.bin": {
949
+ "f1_score": 0.0,
950
+ "precision": 0.0,
951
+ "recall": 0.0,
952
+ "specificity": 1.0,
953
+ "false_positive_rate": 0.0,
954
+ "false_negative_rate": 1.0,
955
+ "negative_predictive_value": 0.9989,
956
+ "n_test_samples": 153
957
+ },
958
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_binary_v3_1fpr.bin": {
959
+ "f1_score": 0.0,
960
+ "precision": 0.0,
961
+ "recall": 0.0,
962
+ "specificity": 1.0,
963
+ "false_positive_rate": 0.0,
964
+ "false_negative_rate": 1.0,
965
+ "negative_predictive_value": 0.9989,
966
+ "n_test_samples": 153
967
+ }
968
+ }
969
+ },
970
+ {
971
+ "Turkey": {
972
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
973
+ "f1_score": 0.0,
974
+ "precision": 0.0,
975
+ "recall": 0.0,
976
+ "specificity": 0.9999,
977
+ "false_positive_rate": 0.0001,
978
+ "false_negative_rate": 0.0,
979
+ "negative_predictive_value": 1.0,
980
+ "n_test_samples": 0
981
+ },
982
+ "cis-lmu/glotlid/model.bin": {
983
+ "f1_score": 0.0,
984
+ "precision": 0.0,
985
+ "recall": 0.0,
986
+ "specificity": 1.0,
987
+ "false_positive_rate": 0.0,
988
+ "false_negative_rate": 0.0,
989
+ "negative_predictive_value": 1.0,
990
+ "n_test_samples": 0
991
+ }
992
+ }
993
+ },
994
+ {
995
+ "Turkmenistan": {
996
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
997
+ "f1_score": 0.0,
998
+ "precision": 0.0,
999
+ "recall": 0.0,
1000
+ "specificity": 1.0,
1001
+ "false_positive_rate": 0.0,
1002
+ "false_negative_rate": 0.0,
1003
+ "negative_predictive_value": 1.0,
1004
+ "n_test_samples": 0
1005
+ }
1006
+ }
1007
+ },
1008
+ {
1009
+ "Uzbekistan": {
1010
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1011
+ "f1_score": 0.0,
1012
+ "precision": 0.0,
1013
+ "recall": 0.0,
1014
+ "specificity": 0.9999,
1015
+ "false_positive_rate": 0.0001,
1016
+ "false_negative_rate": 0.0,
1017
+ "negative_predictive_value": 1.0,
1018
+ "n_test_samples": 0
1019
+ }
1020
+ }
1021
+ },
1022
+ {
1023
+ "Acehnese": {
1024
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1025
+ "f1_score": 0.0,
1026
+ "precision": 0.0,
1027
+ "recall": 0.0,
1028
+ "specificity": 1.0,
1029
+ "false_positive_rate": 0.0,
1030
+ "false_negative_rate": 0.0,
1031
+ "negative_predictive_value": 1.0,
1032
+ "n_test_samples": 0
1033
+ },
1034
+ "laurievb/OpenLID/model.bin": {
1035
+ "f1_score": 0.0,
1036
+ "precision": 0.0,
1037
+ "recall": 0.0,
1038
+ "specificity": 1.0,
1039
+ "false_positive_rate": 0.0,
1040
+ "false_negative_rate": 0.0,
1041
+ "negative_predictive_value": 1.0,
1042
+ "n_test_samples": 0
1043
+ }
1044
+ }
1045
+ },
1046
+ {
1047
+ "Nigeria": {
1048
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1049
+ "f1_score": 0.0,
1050
+ "precision": 0.0,
1051
+ "recall": 0.0,
1052
+ "specificity": 1.0,
1053
+ "false_positive_rate": 0.0,
1054
+ "false_negative_rate": 0.0,
1055
+ "negative_predictive_value": 1.0,
1056
+ "n_test_samples": 0
1057
+ },
1058
+ "cis-lmu/glotlid/model.bin": {
1059
+ "f1_score": 0.0,
1060
+ "precision": 0.0,
1061
+ "recall": 0.0,
1062
+ "specificity": 1.0,
1063
+ "false_positive_rate": 0.0,
1064
+ "false_negative_rate": 0.0,
1065
+ "negative_predictive_value": 1.0,
1066
+ "n_test_samples": 0
1067
+ },
1068
+ "laurievb/OpenLID/model.bin": {
1069
+ "f1_score": 0.0,
1070
+ "precision": 0.0,
1071
+ "recall": 0.0,
1072
+ "specificity": 1.0,
1073
+ "false_positive_rate": 0.0,
1074
+ "false_negative_rate": 0.0,
1075
+ "negative_predictive_value": 1.0,
1076
+ "n_test_samples": 0
1077
+ }
1078
+ }
1079
+ },
1080
+ {
1081
+ "Mesopotamia": {
1082
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1083
+ "f1_score": 0.0,
1084
+ "precision": 0.0,
1085
+ "recall": 0.0,
1086
+ "specificity": 0.9993,
1087
+ "false_positive_rate": 0.0007,
1088
+ "false_negative_rate": 0.0,
1089
+ "negative_predictive_value": 1.0,
1090
+ "n_test_samples": 0
1091
+ },
1092
+ "cis-lmu/glotlid/model.bin": {
1093
+ "f1_score": 0.0,
1094
+ "precision": 0.0,
1095
+ "recall": 0.0,
1096
+ "specificity": 0.9983,
1097
+ "false_positive_rate": 0.0017,
1098
+ "false_negative_rate": 0.0,
1099
+ "negative_predictive_value": 1.0,
1100
+ "n_test_samples": 0
1101
+ },
1102
+ "laurievb/OpenLID/model.bin": {
1103
+ "f1_score": 0.0,
1104
+ "precision": 0.0,
1105
+ "recall": 0.0,
1106
+ "specificity": 0.9989,
1107
+ "false_positive_rate": 0.0011,
1108
+ "false_negative_rate": 0.0,
1109
+ "negative_predictive_value": 1.0,
1110
+ "n_test_samples": 0
1111
+ }
1112
+ }
1113
+ },
1114
+ {
1115
+ "Afghanistan": {
1116
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1117
+ "f1_score": 0.0,
1118
+ "precision": 0.0,
1119
+ "recall": 0.0,
1120
+ "specificity": 0.9998,
1121
+ "false_positive_rate": 0.0002,
1122
+ "false_negative_rate": 0.0,
1123
+ "negative_predictive_value": 1.0,
1124
+ "n_test_samples": 0
1125
+ },
1126
+ "cis-lmu/glotlid/model.bin": {
1127
+ "f1_score": 0.0,
1128
+ "precision": 0.0,
1129
+ "recall": 0.0,
1130
+ "specificity": 0.9999,
1131
+ "false_positive_rate": 0.0001,
1132
+ "false_negative_rate": 0.0,
1133
+ "negative_predictive_value": 1.0,
1134
+ "n_test_samples": 0
1135
+ },
1136
+ "laurievb/OpenLID/model.bin": {
1137
+ "f1_score": 0.0,
1138
+ "precision": 0.0,
1139
+ "recall": 0.0,
1140
+ "specificity": 0.9995,
1141
+ "false_positive_rate": 0.0005,
1142
+ "false_negative_rate": 0.0,
1143
+ "negative_predictive_value": 1.0,
1144
+ "n_test_samples": 0
1145
+ }
1146
+ }
1147
+ },
1148
+ {
1149
+ "Kurdistan": {
1150
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1151
+ "f1_score": 0.0,
1152
+ "precision": 0.0,
1153
+ "recall": 0.0,
1154
+ "specificity": 0.9997,
1155
+ "false_positive_rate": 0.0003,
1156
+ "false_negative_rate": 0.0,
1157
+ "negative_predictive_value": 1.0,
1158
+ "n_test_samples": 0
1159
+ },
1160
+ "cis-lmu/glotlid/model.bin": {
1161
+ "f1_score": 0.0,
1162
+ "precision": 0.0,
1163
+ "recall": 0.0,
1164
+ "specificity": 1.0,
1165
+ "false_positive_rate": 0.0,
1166
+ "false_negative_rate": 0.0,
1167
+ "negative_predictive_value": 1.0,
1168
+ "n_test_samples": 0
1169
+ },
1170
+ "laurievb/OpenLID/model.bin": {
1171
+ "f1_score": 0.0,
1172
+ "precision": 0.0,
1173
+ "recall": 0.0,
1174
+ "specificity": 0.9999,
1175
+ "false_positive_rate": 0.0001,
1176
+ "false_negative_rate": 0.0,
1177
+ "negative_predictive_value": 1.0,
1178
+ "n_test_samples": 0
1179
+ }
1180
+ }
1181
+ },
1182
+ {
1183
+ "Kashmir": {
1184
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1185
+ "f1_score": 0.0,
1186
+ "precision": 0.0,
1187
+ "recall": 0.0,
1188
+ "specificity": 0.9999,
1189
+ "false_positive_rate": 0.0001,
1190
+ "false_negative_rate": 0.0,
1191
+ "negative_predictive_value": 1.0,
1192
+ "n_test_samples": 0
1193
+ },
1194
+ "laurievb/OpenLID/model.bin": {
1195
+ "f1_score": 0.0,
1196
+ "precision": 0.0,
1197
+ "recall": 0.0,
1198
+ "specificity": 0.9993,
1199
+ "false_positive_rate": 0.0007,
1200
+ "false_negative_rate": 0.0,
1201
+ "negative_predictive_value": 1.0,
1202
+ "n_test_samples": 0
1203
+ }
1204
+ }
1205
+ },
1206
+ {
1207
+ "Iran": {
1208
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1209
+ "f1_score": 0.0,
1210
+ "precision": 0.0,
1211
+ "recall": 0.0,
1212
+ "specificity": 0.9994,
1213
+ "false_positive_rate": 0.0006,
1214
+ "false_negative_rate": 0.0,
1215
+ "negative_predictive_value": 1.0,
1216
+ "n_test_samples": 0
1217
+ },
1218
+ "cis-lmu/glotlid/model.bin": {
1219
+ "f1_score": 0.0,
1220
+ "precision": 0.0,
1221
+ "recall": 0.0,
1222
+ "specificity": 0.9999,
1223
+ "false_positive_rate": 0.0001,
1224
+ "false_negative_rate": 0.0,
1225
+ "negative_predictive_value": 1.0,
1226
+ "n_test_samples": 0
1227
+ }
1228
+ }
1229
+ },
1230
+ {
1231
+ "Indonesia": {
1232
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1233
+ "f1_score": 0.0,
1234
+ "precision": 0.0,
1235
+ "recall": 0.0,
1236
+ "specificity": 1.0,
1237
+ "false_positive_rate": 0.0,
1238
+ "false_negative_rate": 0.0,
1239
+ "negative_predictive_value": 1.0,
1240
+ "n_test_samples": 0
1241
+ },
1242
+ "cis-lmu/glotlid/model.bin": {
1243
+ "f1_score": 0.0,
1244
+ "precision": 0.0,
1245
+ "recall": 0.0,
1246
+ "specificity": 1.0,
1247
+ "false_positive_rate": 0.0,
1248
+ "false_negative_rate": 0.0,
1249
+ "negative_predictive_value": 1.0,
1250
+ "n_test_samples": 0
1251
+ },
1252
+ "laurievb/OpenLID/model.bin": {
1253
+ "f1_score": 0.0,
1254
+ "precision": 0.0,
1255
+ "recall": 0.0,
1256
+ "specificity": 0.9999,
1257
+ "false_positive_rate": 0.0001,
1258
+ "false_negative_rate": 0.0,
1259
+ "negative_predictive_value": 1.0,
1260
+ "n_test_samples": 0
1261
+ }
1262
+ }
1263
+ },
1264
+ {
1265
+ "Guinea": {
1266
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1267
+ "f1_score": 0.0,
1268
+ "precision": 0.0,
1269
+ "recall": 0.0,
1270
+ "specificity": 1.0,
1271
+ "false_positive_rate": 0.0,
1272
+ "false_negative_rate": 0.0,
1273
+ "negative_predictive_value": 1.0,
1274
+ "n_test_samples": 0
1275
+ }
1276
+ }
1277
+ },
1278
+ {
1279
+ "Chad": {
1280
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1281
+ "f1_score": 0.0,
1282
+ "precision": 0.0,
1283
+ "recall": 0.0,
1284
+ "specificity": 1.0,
1285
+ "false_positive_rate": 0.0,
1286
+ "false_negative_rate": 0.0,
1287
+ "negative_predictive_value": 1.0,
1288
+ "n_test_samples": 0
1289
+ }
1290
+ }
1291
+ },
1292
+ {
1293
+ "Azerbaijan": {
1294
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1295
+ "f1_score": 0.0,
1296
+ "precision": 0.0,
1297
+ "recall": 0.0,
1298
+ "specificity": 0.9997,
1299
+ "false_positive_rate": 0.0003,
1300
+ "false_negative_rate": 0.0,
1301
+ "negative_predictive_value": 1.0,
1302
+ "n_test_samples": 0
1303
+ },
1304
+ "cis-lmu/glotlid/model.bin": {
1305
+ "f1_score": 0.0,
1306
+ "precision": 0.0,
1307
+ "recall": 0.0,
1308
+ "specificity": 0.9999,
1309
+ "false_positive_rate": 0.0001,
1310
+ "false_negative_rate": 0.0,
1311
+ "negative_predictive_value": 1.0,
1312
+ "n_test_samples": 0
1313
+ },
1314
+ "laurievb/OpenLID/model.bin": {
1315
+ "f1_score": 0.0,
1316
+ "precision": 0.0,
1317
+ "recall": 0.0,
1318
+ "specificity": 1.0,
1319
+ "false_positive_rate": 0.0,
1320
+ "false_negative_rate": 0.0,
1321
+ "negative_predictive_value": 1.0,
1322
+ "n_test_samples": 0
1323
+ }
1324
+ }
1325
+ },
1326
+ {
1327
+ "Malaysia": {
1328
+ "atlasia/Sfaya-Moroccan-Darija-vs-All/model_multi_v3_2fpr.bin": {
1329
+ "f1_score": 0.0,
1330
+ "precision": 0.0,
1331
+ "recall": 0.0,
1332
+ "specificity": 1.0,
1333
+ "false_positive_rate": 0.0,
1334
+ "false_negative_rate": 0.0,
1335
+ "negative_predictive_value": 1.0,
1336
+ "n_test_samples": 0
1337
+ }
1338
+ }
1339
+ },
1340
+ {
1341
+ "Uighur (China)": {
1342
+ "cis-lmu/glotlid/model.bin": {
1343
+ "f1_score": 0.0,
1344
+ "precision": 0.0,
1345
+ "recall": 0.0,
1346
+ "specificity": 1.0,
1347
+ "false_positive_rate": 0.0,
1348
+ "false_negative_rate": 0.0,
1349
+ "negative_predictive_value": 1.0,
1350
+ "n_test_samples": 0
1351
+ },
1352
+ "laurievb/OpenLID/model.bin": {
1353
+ "f1_score": 0.0,
1354
+ "precision": 0.0,
1355
+ "recall": 0.0,
1356
+ "specificity": 1.0,
1357
+ "false_positive_rate": 0.0,
1358
+ "false_negative_rate": 0.0,
1359
+ "negative_predictive_value": 1.0,
1360
+ "n_test_samples": 0
1361
+ }
1362
+ }
1363
+ },
1364
+ {
1365
+ "Balochistan": {
1366
+ "cis-lmu/glotlid/model.bin": {
1367
+ "f1_score": 0.0,
1368
+ "precision": 0.0,
1369
+ "recall": 0.0,
1370
+ "specificity": 1.0,
1371
+ "false_positive_rate": 0.0,
1372
+ "false_negative_rate": 0.0,
1373
+ "negative_predictive_value": 1.0,
1374
+ "n_test_samples": 0
1375
+ }
1376
+ }
1377
+ }
1378
+ ]
open_arabic_lid_arena.png ADDED

Git LFS Details

  • SHA256: ac613af0af6725fc8f1170b075c354ca0a032ef4e0eeca62b9a4f512b32e4697
  • Pointer size: 132 Bytes
  • Size of remote file: 1.75 MB
utils.py ADDED
@@ -0,0 +1,488 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ from fasttext import load_model
3
+ from huggingface_hub import hf_hub_download
4
+ import os
5
+ import json
6
+ import pandas as pd
7
+ from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, balanced_accuracy_score, matthews_corrcoef
8
+ import numpy as np
9
+ from datasets import load_dataset
10
+ import fasttext
11
+
12
+ # Constants
13
+ MODEL_REPO = "atlasia/Sfaya-Moroccan-Darija-vs-All"
14
+ BIN_FILENAME = "model_multi_v3_2fpr.bin"
15
+ BINARY_LEADERBOARD_FILE = "darija_leaderboard_binary.json"
16
+ MULTILINGUAL_LEADERBOARD_FILE = "darija_leaderboard_multilingual.json"
17
+ DATA_PATH = "atlasia/No-Arabic-Dialect-Left-Behind-Filtered-Balanced"
18
+
19
+ target_label = "Morocco"
20
+ is_binary = False
21
+
22
+ metrics = [
23
+ 'f1_score',
24
+ 'precision',
25
+ 'recall',
26
+ 'specificity',
27
+ 'false_positive_rate',
28
+ 'false_negative_rate',
29
+ 'negative_predictive_value',
30
+ 'n_test_samples',
31
+ ]
32
+
33
+ default_metrics = [
34
+ 'f1_score',
35
+ 'precision',
36
+ 'recall',
37
+ 'false_positive_rate',
38
+ 'false_negative_rate'
39
+ ]
40
+
41
+ language_mapping_dict = {
42
+ 'ace_Arab': 'Acehnese',
43
+ 'acm_Arab': 'Mesopotamia', # 'Gilit Mesopotamian'
44
+ 'aeb_Arab': 'Tunisia',
45
+ 'ajp_Arab': 'Levantine', # 'South Levantine'
46
+ 'apc_Arab': 'Levantine',
47
+ 'arb_Arab': 'MSA',
48
+ 'arq_Arab': 'Algeria',
49
+ 'ars_Arab': 'Saudi', # Najdi is primarily Saudi Arabian
50
+ 'ary_Arab': 'Morocco',
51
+ 'arz_Arab': 'Egypt',
52
+ 'ayp_Arab': 'Mesopotamia', # 'North Mesopotamian'
53
+ 'azb_Arab': 'Azerbaijan', # South Azerbaijani pertains to this region
54
+ 'bcc_Arab': 'Balochistan', # Southern Balochi is from Balochistan
55
+ 'bjn_Arab': 'Indonesia', # Banjar is spoken in Indonesia
56
+ 'brh_Arab': 'Pakistan', # Brahui is spoken in Pakistan
57
+ 'ckb_Arab': 'Kurdistan', # Central Kurdish is mainly in Iraq
58
+ 'fuv_Arab': 'Nigeria', # Hausa States Fulfulde
59
+ 'glk_Arab': 'Iran', # Gilaki is spoken in Iran
60
+ 'hac_Arab': 'Iran', # Gurani is also primarily spoken in Iran
61
+ 'kas_Arab': 'Kashmir',
62
+ 'knc_Arab': 'Nigeria', # Central Kanuri is in Nigeria
63
+ 'lki_Arab': 'Iran', # Laki is from Iran
64
+ 'lrc_Arab': 'Iran', # Northern Luri is from Iran
65
+ 'min_Arab': 'Indonesia', # Minangkabau is spoken in Indonesia
66
+ 'mzn_Arab': 'Iran', # Mazanderani is spoken in Iran
67
+ 'ota_Arab': 'Turkey', # Ottoman Turkish
68
+ 'pbt_Arab': 'Afghanistan', # Southern Pashto
69
+ 'pnb_Arab': 'Pakistan', # Western Panjabi
70
+ 'sdh_Arab': 'Iraq', # Southern Kurdish
71
+ 'shu_Arab': 'Chad', # Chadian Arabic
72
+ 'skr_Arab': 'Pakistan', # Saraiki
73
+ 'snd_Arab': 'Pakistan', # Sindhi
74
+ 'sus_Arab': 'Guinea', # Susu
75
+ 'tuk_Arab': 'Turkmenistan', # Turkmen
76
+ 'uig_Arab': 'Uighur (China)', # Uighur
77
+ 'urd_Arab': 'Pakistan', # Urdu
78
+ 'uzs_Arab': 'Uzbekistan', # Southern Uzbek
79
+ 'zsm_Arab': 'Malaysia' # Standard Malay
80
+ }
81
+
82
+ def predict_label(text, model, language_mapping_dict, use_mapping=False):
83
+ # Remove any newline characters and strip whitespace
84
+ text = str(text).strip().replace('\n', ' ')
85
+
86
+ if text == '':
87
+ return 'Other'
88
+
89
+ try:
90
+ # Get top prediction
91
+ prediction = model.predict(text, 1)
92
+
93
+ # Extract label and remove __label__ prefix
94
+ label = prediction[0][0].replace('__label__', '')
95
+
96
+ # Extract confidence score
97
+ confidence = prediction[1][0]
98
+
99
+ # map label to language using language_mapping_dict
100
+ if use_mapping:
101
+ label = language_mapping_dict.get(label, 'Other')
102
+ return label
103
+
104
+ except Exception as e:
105
+ print(f"Error processing text: {text}")
106
+ print(f"Exception: {e}")
107
+ return {'prediction_label': 'Error', 'prediction_confidence': 0.0}
108
+
109
+ def compute_classification_metrics(test_dataset):
110
+ """
111
+ Compute comprehensive classification metrics for each class.
112
+
113
+ Args:
114
+ data (pd.DataFrame): DataFrame containing 'dialect' as true labels and 'preds' as predicted labels.
115
+
116
+ Returns:
117
+ pd.DataFrame: DataFrame with detailed metrics for each class.
118
+ """
119
+ # transform the dataset into a DataFrame
120
+ data = pd.DataFrame(test_dataset)
121
+ # Extract true labels and predictions
122
+ true_labels = list(data['dialect'])
123
+ predicted_labels = list(data['preds'])
124
+
125
+ # Handle all unique labels
126
+ labels = sorted(list(set(true_labels + predicted_labels)))
127
+ label_to_index = {label: index for index, label in enumerate(labels)}
128
+
129
+ # Convert labels to indices
130
+ true_indices = [label_to_index[label] for label in true_labels]
131
+ pred_indices = [label_to_index[label] for label in predicted_labels]
132
+
133
+ # Compute basic metrics
134
+ f1_scores = f1_score(true_indices, pred_indices, average=None, labels=range(len(labels)))
135
+ precision_scores = precision_score(true_indices, pred_indices, average=None, labels=range(len(labels)))
136
+ recall_scores = recall_score(true_indices, pred_indices, average=None, labels=range(len(labels)))
137
+
138
+ # Compute confusion matrix
139
+ conf_mat = confusion_matrix(true_indices, pred_indices, labels=range(len(labels)))
140
+
141
+ # Calculate various metrics per class
142
+ FP = conf_mat.sum(axis=0) - np.diag(conf_mat) # False Positives
143
+ FN = conf_mat.sum(axis=1) - np.diag(conf_mat) # False Negatives
144
+ TP = np.diag(conf_mat) # True Positives
145
+ TN = conf_mat.sum() - (FP + FN + TP) # True Negatives
146
+
147
+ # Calculate sample counts per class
148
+ samples_per_class = np.bincount(true_indices, minlength=len(labels))
149
+
150
+ # Calculate additional metrics
151
+ with np.errstate(divide='ignore', invalid='ignore'):
152
+ fp_rate = FP / (FP + TN) # False Positive Rate
153
+ fn_rate = FN / (FN + TP) # False Negative Rate
154
+ specificity = TN / (TN + FP) # True Negative Rate
155
+ npv = TN / (TN + FN) # Negative Predictive Value
156
+
157
+ # Replace NaN/inf with 0
158
+ metrics = [fp_rate, fn_rate, specificity, npv]
159
+ metrics = [np.nan_to_num(m, nan=0.0, posinf=0.0, neginf=0.0) for m in metrics]
160
+ fp_rate, fn_rate, specificity, npv = metrics
161
+
162
+ # Calculate overall metrics
163
+ balanced_acc = balanced_accuracy_score(true_indices, pred_indices)
164
+ mcc = matthews_corrcoef(true_indices, pred_indices)
165
+
166
+ # Compile results into a DataFrame
167
+ result_df = pd.DataFrame({
168
+ 'country': labels,
169
+ 'samples': samples_per_class,
170
+ 'f1_score': f1_scores,
171
+ 'precision': precision_scores,
172
+ 'recall': recall_scores,
173
+ 'specificity': specificity,
174
+ 'false_positive_rate': fp_rate,
175
+ 'false_negative_rate': fn_rate,
176
+ 'true_positives': TP,
177
+ 'false_positives': FP,
178
+ 'true_negatives': TN,
179
+ 'false_negatives': FN,
180
+ 'negative_predictive_value': npv
181
+ })
182
+
183
+ # Sort by number of samples (descending)
184
+ result_df = result_df.sort_values('samples', ascending=False)
185
+
186
+ # Calculate and add summary metrics
187
+ summary_metrics = {
188
+ 'macro_f1': f1_score(true_indices, pred_indices, average='macro'),
189
+ 'weighted_f1': f1_score(true_indices, pred_indices, average='weighted'),
190
+ 'micro_f1': f1_score(true_indices, pred_indices, average='micro'),
191
+ 'balanced_accuracy': balanced_acc,
192
+ 'matthews_correlation': mcc
193
+ }
194
+
195
+ # Format all numeric columns to 4 decimal places
196
+ numeric_cols = result_df.select_dtypes(include=[np.number]).columns
197
+ result_df[numeric_cols] = result_df[numeric_cols].round(4)
198
+
199
+ print(f'result_df: {result_df}')
200
+
201
+ return result_df, summary_metrics
202
+
203
+ def make_binary(dialect, target):
204
+ if dialect != target:
205
+ return 'Other'
206
+ return target
207
+
208
+ def run_eval_one_vs_all(model, data_test, TARGET_LANG='Morocco', language_mapping_dict=None, use_mapping=False):
209
+
210
+ # Predict labels using the model
211
+ print(f"[INFO] Running predictions...")
212
+ data_test['preds'] = data_test['text'].apply(lambda text: predict_label(text, model, language_mapping_dict, use_mapping=use_mapping))
213
+
214
+ # map to binary
215
+ df_test_preds = data_test.copy()
216
+ df_test_preds.loc[df_test_preds['dialect'] == TARGET_LANG, 'dialect'] = TARGET_LANG
217
+ df_test_preds.loc[df_test_preds['dialect'] != TARGET_LANG, 'dialect'] = 'Other'
218
+
219
+ # compute the fpr per dialect
220
+ dialect_counts = data_test.groupby('dialect')['dialect'].count().reset_index(name='size')
221
+ result_df = pd.merge(dialect_counts, data_test, on='dialect')
222
+ result_df = result_df.groupby(['dialect', 'size', 'preds'])['preds'].count()/result_df.groupby(['dialect', 'size'])['preds'].count()
223
+ result_df.sort_index(ascending=False, level='size', inplace=True)
224
+
225
+ # group by dialect and get the false positive rate
226
+ out = result_df.copy()
227
+ out.name = 'false_positive_rate'
228
+ out = out.reset_index()
229
+ out = out[out['preds']==TARGET_LANG].drop(columns=['preds', 'size'])
230
+
231
+ return out
232
+
233
+ def update_darija_binary_leaderboard(result_df, model_name, BINARY_LEADERBOARD_FILE="darija_leaderboard_binary.json"):
234
+ try:
235
+ with open(BINARY_LEADERBOARD_FILE, "r") as f:
236
+ data = json.load(f)
237
+ except FileNotFoundError:
238
+ data = []
239
+
240
+ # Process the results for each dialect/country
241
+ for _, row in result_df.iterrows():
242
+ country = row['dialect']
243
+ # skip 'Other' class, it is considered as the null space
244
+ if country == 'Other':
245
+ continue
246
+
247
+ # Find existing country entry or create new one
248
+ country_entry = next((item for item in data if country in item), None)
249
+ if country_entry is None:
250
+ country_entry = {country: {}}
251
+ data.append(country_entry)
252
+
253
+ # Update the model metrics directly under the model name
254
+ if country not in country_entry:
255
+ country_entry[country] = {}
256
+ country_entry[country][model_name] = float(row['false_positive_rate'])
257
+
258
+ if country_entry[country].get("n_test_samples") is None:
259
+ country_entry[country]["n_test_samples"] = int(row['size'])
260
+
261
+ # Save updated leaderboard data
262
+ with open(MULTILINGUAL_LEADERBOARD_FILE, "w") as f:
263
+ json.dump(data, f, indent=4)
264
+
265
+ def handle_evaluation(model_path, model_path_bin, use_mapping=False):
266
+ # run the evaluation
267
+ result_df, _ = run_eval(model_path, model_path_bin, language_mapping_dict, use_mapping=use_mapping)
268
+ # set the model name
269
+ model_name = model_path + '/' + model_path_bin
270
+ # update the leaderboard
271
+ update_darija_multilingual_leaderboard(result_df, model_name, MULTILINGUAL_LEADERBOARD_FILE)
272
+ # update the leaderboard table
273
+ df = load_leaderboard_multilingual()
274
+
275
+ return create_leaderboard_display_multilingual(df, 'Morocco', default_metrics)
276
+
277
+ def run_eval(model_path, model_path_bin, language_mapping_dict=None, use_mapping=False):
278
+ """Run evaluation on a dataset and compute metrics.
279
+
280
+ Args:
281
+ model: The model to evaluate.
282
+ DATA_PATH (str): Path to the dataset.
283
+ is_binary (bool): If True, evaluate as binary classification.
284
+ If False, evaluate as multi-class classification.
285
+ target_label (str): The target class label in binary mode.
286
+
287
+ Returns:
288
+ pd.DataFrame: A DataFrame containing evaluation metrics.
289
+ """
290
+
291
+ # download model and get the model path
292
+ model_path = hf_hub_download(repo_id=model_path, filename=model_path_bin, cache_dir=None)
293
+
294
+ # Load the trained model
295
+ print(f"[INFO] Loading model from Path: {model_path}, using version {model_path_bin}...")
296
+ model = fasttext.load_model(model_path)
297
+
298
+ # Load the evaluation dataset
299
+ print(f"[INFO] Loading evaluation dataset from Path: atlasia/No-Arabic-Dialect-Left-Behind-Filtered-Balanced...")
300
+ eval_dataset = load_dataset("atlasia/No-Arabic-Dialect-Left-Behind-Filtered-Balanced", split='test')
301
+
302
+ # Transform to pandas DataFrame
303
+ print(f"[INFO] Converting evaluation dataset to Pandas DataFrame...")
304
+ df_eval = pd.DataFrame(eval_dataset)
305
+
306
+ # Predict labels using the model
307
+ print(f"[INFO] Running predictions...")
308
+ df_eval['preds'] = df_eval['text'].apply(lambda text: predict_label(text, model, language_mapping_dict, use_mapping=use_mapping))
309
+
310
+ # now drop the columns that are not needed, i.e. 'text'
311
+ df_eval = df_eval.drop(columns=['text', 'metadata', 'dataset_source'])
312
+
313
+ # Compute evaluation metrics
314
+ print(f"[INFO] Computing metrics...")
315
+ result_df, _ = compute_classification_metrics(df_eval)
316
+
317
+ # update_darija_multilingual_leaderboard(result_df, model_path, MULTILINGUAL_LEADERBOARD_FILE)
318
+
319
+ return result_df, df_eval
320
+
321
+ def process_results_file(file, uploaded_model_name, base_path_save="./atlasia/submissions/"):
322
+ try:
323
+ if file is None:
324
+ return "Please upload a file."
325
+
326
+ # Clean the model name to be safe for file paths
327
+ uploaded_model_name = uploaded_model_name.strip().replace(" ", "_")
328
+ print(f"[INFO] uploaded_model_name: {uploaded_model_name}")
329
+
330
+ # Create the directory for saving submissions
331
+ path_saving = os.path.join(base_path_save, uploaded_model_name)
332
+ os.makedirs(path_saving, exist_ok=True)
333
+
334
+ # Define the full path to save the file
335
+ saved_file_path = os.path.join(path_saving, 'submission.csv')
336
+
337
+ # Read the uploaded file as DataFrame
338
+ print(f"[INFO] Loading results...")
339
+ df_eval = pd.read_csv(file.name)
340
+
341
+ # Save the DataFrame
342
+ print(f"[INFO] Saving the file locally in: {saved_file_path}")
343
+ df_eval.to_csv(saved_file_path, index=False)
344
+
345
+ except Exception as e:
346
+ return f"Error processing file: {str(e)}"
347
+
348
+ # Compute evaluation metrics
349
+ print(f"[INFO] Computing metrics...")
350
+ result_df, _ = compute_classification_metrics(df_eval)
351
+
352
+ # Update the leaderboards
353
+ update_darija_multilingual_leaderboard(result_df, uploaded_model_name, MULTILINGUAL_LEADERBOARD_FILE)
354
+
355
+ # result_df_binary = run_eval_one_vs_all(model, data_test, TARGET_LANG='Morocco', language_mapping_dict=None, use_mapping=False)
356
+ # update_darija_binary_leaderboard(result_df, uploaded_model_name, BINARY_LEADERBOARD_FILE)
357
+
358
+ # update the leaderboard table
359
+ df = load_leaderboard_multilingual()
360
+
361
+ return create_leaderboard_display_multilingual(df, 'Morocco', default_metrics)
362
+
363
+ def update_darija_multilingual_leaderboard(result_df, model_name, MULTILINGUAL_LEADERBOARD_FILE="darija_leaderboard_multilingual.json"):
364
+
365
+ # Load leaderboard data
366
+ current_dir = os.path.dirname(os.path.abspath(__file__))
367
+ MULTILINGUAL_LEADERBOARD_FILE = os.path.join(current_dir, MULTILINGUAL_LEADERBOARD_FILE)
368
+
369
+ try:
370
+ with open(MULTILINGUAL_LEADERBOARD_FILE, "r") as f:
371
+ data = json.load(f)
372
+ except FileNotFoundError:
373
+ data = []
374
+
375
+ # Process the results for each dialect/country
376
+ for _, row in result_df.iterrows():
377
+ country = row['country']
378
+ # skip 'Other' class, it is considered as the null space
379
+ if country == 'Other':
380
+ continue
381
+
382
+ # Create metrics dictionary directly
383
+ metrics = {
384
+ 'f1_score': float(row['f1_score']),
385
+ 'precision': float(row['precision']),
386
+ 'recall': float(row['recall']),
387
+ 'specificity': float(row['specificity']),
388
+ 'false_positive_rate': float(row['false_positive_rate']),
389
+ 'false_negative_rate': float(row['false_negative_rate']),
390
+ 'negative_predictive_value': float(row['negative_predictive_value']),
391
+ 'n_test_samples': int(row['samples'])
392
+ }
393
+
394
+ # Find existing country entry or create new one
395
+ country_entry = next((item for item in data if country in item), None)
396
+ if country_entry is None:
397
+ country_entry = {country: {}}
398
+ data.append(country_entry)
399
+
400
+ # Update the model metrics directly under the model name
401
+ if country not in country_entry:
402
+ country_entry[country] = {}
403
+ country_entry[country][model_name] = metrics
404
+
405
+ # Save updated leaderboard data
406
+ with open(MULTILINGUAL_LEADERBOARD_FILE, "w") as f:
407
+ json.dump(data, f, indent=4)
408
+
409
+
410
+ def load_leaderboard_multilingual(MULTILINGUAL_LEADERBOARD_FILE="darija_leaderboard_multilingual.json"):
411
+ current_dir = os.path.dirname(os.path.abspath(__file__))
412
+ MULTILINGUAL_LEADERBOARD_FILE = os.path.join(current_dir, MULTILINGUAL_LEADERBOARD_FILE)
413
+
414
+ with open(MULTILINGUAL_LEADERBOARD_FILE, "r") as f:
415
+ data = json.load(f)
416
+
417
+ # Initialize lists to store the flattened data
418
+ rows = []
419
+
420
+ # Process each country's data
421
+ for country_data in data:
422
+ for country, models in country_data.items():
423
+ for model_name, metrics in models.items():
424
+ row = {
425
+ 'country': country,
426
+ 'model': model_name,
427
+ }
428
+ # Add all metrics to the row
429
+ row.update(metrics)
430
+ rows.append(row)
431
+
432
+ # Convert to DataFrame
433
+ df = pd.DataFrame(rows)
434
+ return df
435
+
436
+ def create_leaderboard_display_multilingual(df, selected_country, selected_metrics):
437
+ # Filter by country if specified
438
+ if selected_country and selected_country.upper() != 'ALL':
439
+ print(f"Filtering leaderboard by country: {selected_country}")
440
+ df = df[df['country'] == selected_country]
441
+ df = df.drop(columns=['country'])
442
+
443
+ # Select only the chosen metrics (plus 'model' column)
444
+ columns_to_show = ['model'] + [metric for metric in selected_metrics if metric in df.columns]
445
+
446
+ else:
447
+ # Select all metrics (plus 'country' and 'model' columns), if no country is selected or 'All' is selected for ease of comparison
448
+ columns_to_show = ['model', 'country'] + selected_metrics
449
+
450
+ # Sort by first selected metric by default
451
+ if selected_metrics:
452
+ df = df.sort_values(by=selected_metrics[0], ascending=False)
453
+
454
+ df = df[columns_to_show]
455
+
456
+ # Format numeric columns to 4 decimal places
457
+ numeric_cols = df.select_dtypes(include=['float64']).columns
458
+ df[numeric_cols] = df[numeric_cols].round(4)
459
+
460
+ return df
461
+
462
+ def update_leaderboard_multilingual(country, selected_metrics):
463
+ if not selected_metrics: # If no metrics selected, show all
464
+ selected_metrics = metrics
465
+ df = load_leaderboard_multilingual()
466
+ display_df = create_leaderboard_display_multilingual(df, country, selected_metrics)
467
+ return display_df
468
+
469
+ def encode_image_to_base64(image_path):
470
+ with open(image_path, "rb") as image_file:
471
+ encoded_string = base64.b64encode(image_file.read()).decode()
472
+ return encoded_string
473
+
474
+ def create_html_image(image_path):
475
+ # Get base64 string of image
476
+ img_base64 = encode_image_to_base64(image_path)
477
+
478
+ # Create HTML string with embedded image and centering styles
479
+ html_string = f"""
480
+ <div style="display: flex; justify-content: center; align-items: center; width: 100%; text-align: center;">
481
+ <div style="max-width: 800px; margin: auto;">
482
+ <img src="data:image/jpeg;base64,{img_base64}"
483
+ style="max-width: 75%; height: auto; display: block; margin: 0 auto; margin-top: 50px;"
484
+ alt="Displayed Image">
485
+ </div>
486
+ </div>
487
+ """
488
+ return html_string