Spaces:

atlasia
/

darija-tokenizers-leaderboard

Running

nouamanetazi HF staff commited on Dec 26, 2024

Commit

4df8d75

1 Parent(s): 59752f1

.

Files changed (1) hide show

tabs/comparison.py CHANGED Viewed

@@ -53,7 +53,7 @@ def comparison_tab(df):
                     st.write(f"**Tokens/Character Ratio:** {metrics['Tokens/Character Ratio']:.4f}")
                     tokenizer = get_tokenizer(model_1)
                     tokens = tokenizer.tokenize(input_text)
-                    tokens_html = ' '.join([f'<span style="background-color: #e6f3ff; padding: 2px 5px; margin-right: 5px; border-radius: 3px;">{token}</span>' for token in tokens])
                     st.markdown(f'<div style="line-height: 2.5;">{tokens_html}</div>', unsafe_allow_html=True)
     with col2:
@@ -72,5 +72,5 @@ def comparison_tab(df):
                     st.write(f"**Tokens/Character Ratio:** {metrics['Tokens/Character Ratio']:.4f}")
                     tokenizer = get_tokenizer(model_2)
                     tokens = tokenizer.tokenize(input_text)
-                    tokens_html = ' '.join([f'<span style="background-color: #fa87b5; padding: 2px 5px; margin-right: 5px; border-radius: 3px;">{token}</span>' for token in tokens])
-                    st.markdown(f'<div style="line-height: 2.5;">{tokens_html}</div>', unsafe_allow_html=True)

                     st.write(f"**Tokens/Character Ratio:** {metrics['Tokens/Character Ratio']:.4f}")
                     tokenizer = get_tokenizer(model_1)
                     tokens = tokenizer.tokenize(input_text)
+                    tokens_html = ' '.join([f'<span style="background-color: #00BFFF; padding: 2px 5px; margin-right: 5px; border-radius: 3px;">{token}</span>' for token in tokens])
                     st.markdown(f'<div style="line-height: 2.5;">{tokens_html}</div>', unsafe_allow_html=True)
     with col2:
                     st.write(f"**Tokens/Character Ratio:** {metrics['Tokens/Character Ratio']:.4f}")
                     tokenizer = get_tokenizer(model_2)
                     tokens = tokenizer.tokenize(input_text)
+                    tokens_html = ' '.join([f'<span style="background-color: #FF1493; padding: 2px 5px; margin-right: 5px; border-radius: 3px;">{token}</span>' for token in tokens])
+                    st.markdown(f'<div style="line-height: 2.5;">{tokens_html}</div>', unsafe_allow_html=True)