[
  {
    "name": "FLORES+",
    "author": "Meta",
    "url": "https://huggingface.co/datasets/openlanguagedata/flores_plus",
    "n_languages": 185,
    "tasks": [
      "translation",
      "classification",
      "language_modeling"
    ],
    "parallel": true,
    "base": "FLORES",
    "implemented": true
  },
  {
    "name": "FLEURS",
    "author": "Meta",
    "url": "https://huggingface.co/datasets/google/fleurs",
    "n_languages": 102,
    "tasks": [
      "speech_recognition"
    ],
    "parallel": true,
    "base": "FLORES",
    "implemented": true
  },
  {
    "name": "CommonVoice",
    "author": "Mozilla",
    "url": "https://huggingface.co/datasets/mozilla-foundation/common_voice_1_0",
    "n_languages": 231,
    "tasks": [
      "speech_recognition"
    ],
    "parallel": null
  },
  {
    "name": "MMMLU",
    "author": "OpenAI",
    "url": "https://huggingface.co/datasets/openai/MMMLU",
    "n_languages": "14",
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "MMLU"
  },
  {
    "name": "AfriMMLU",
    "author": "Masakhane",
    "url": "https://huggingface.co/datasets/masakhane/afrimmlu",
    "n_languages": "17",
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "MMLU"
  },
  {
    "name": "Okapi MMLU",
    "author": "Okapi",
    "url": "https://huggingface.co/datasets/jon-tow/okapi_mmlu",
    "n_languages": 16,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "MMLU"
  },
  {
    "name": "Global MMLU",
    "author": "Cohere",
    "url": "https://huggingface.co/datasets/CohereForAI/Global-MMLU",
    "n_languages": 42,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "MMLU"
  },
  {
    "name": "MGSM",
    "author": "Google",
    "url": "https://huggingface.co/datasets/juletxara/mgsm",
    "n_languages": 10,
    "tasks": [
      "math"
    ],
    "parallel": true,
    "base": "MGSM"
  },
  {
    "name": "AfriMGSM",
    "author": "Masakhane",
    "url": "https://huggingface.co/datasets/masakhane/afrimgsm",
    "n_languages": 18,
    "tasks": [
      "math"
    ],
    "parallel": true,
    "base": "MGSM"
  },
  {
    "name": "Okapi ARC Challenge",
    "author": "Okapi",
    "url": "https://huggingface.co/datasets/jon-tow/okapi_arc_challenge",
    "n_languages": 31,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "AI2 ARC"
  },
  {
    "name": "Uhuru ARC Easy",
    "author": "Masakhane",
    "url": "https://huggingface.co/datasets/masakhane/uhura-arc-easy",
    "n_languages": 6,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "AI2 ARC"
  },
  {
    "name": "Okapi TruthfulQA",
    "author": "Okapi",
    "url": "https://huggingface.co/datasets/jon-tow/okapi_truthfulqa/tree/main/data",
    "n_languages": 31,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "TruthfulQA"
  },
  {
    "name": "Uhura TruthfulQA",
    "author": "Masakhane",
    "url": "https://huggingface.co/datasets/masakhane/uhura-truthfulqa",
    "n_languages": 6,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "TruthfulQA"
  },
  {
    "name": "XNLI",
    "author": "Meta",
    "url": "https://huggingface.co/datasets/facebook/xnli",
    "n_languages": 14,
    "tasks": [
      "classification"
    ],
    "parallel": true,
    "base": "XNLI"
  },
  {
    "name": "AfriXNLI",
    "author": "Masakhane",
    "url": "https://huggingface.co/datasets/masakhane/afrixnli",
    "n_languages": 18,
    "tasks": [
      "classification"
    ],
    "parallel": true,
    "base": "XNLI"
  },
  {
    "name": "IndicXNLI",
    "author": "AI4Bharat",
    "url": "https://huggingface.co/datasets/Divyanshu/indicxnli",
    "n_languages": 11,
    "tasks": [
      "classification"
    ],
    "parallel": true,
    "base": "XNLI"
  },
  {
    "name": "Okapi HellaSwag",
    "author": "Okapi",
    "url": "https://huggingface.co/datasets/jon-tow/okapi_hellaswag",
    "n_languages": 31,
    "tasks": [
      "question_answering"
    ],
    "parallel": true,
    "base": "HellaSwag"
  },
  {
    "name": "XTREME",
    "author": "Google",
    "url": "https://huggingface.co/datasets/google/xtreme",
    "n_languages": 40,
    "tasks": [
      "translation",
      "classification",
      "question_answering",
      "ner"
    ],
    "parallel": null
  },
  {
    "name": "IndicXTREME",
    "author": "AI4Bharat",
    "url": "https://huggingface.co/collections/ai4bharat/indicxtreme-66c59f576386ba2955650030",
    "n_languages": 20,
    "tasks": [
      "translation",
      "classification",
      "question_answering",
      "ner"
    ],
    "parallel": null
  },
  {
    "name": "XGLUE",
    "author": "Microsoft",
    "url": "https://huggingface.co/datasets/microsoft/xglue",
    "n_languages": 18,
    "tasks": [
      "pos"
    ],
    "parallel": null,
    "base": "GLUE"
  },
  {
    "name": "IndicGLUE",
    "author": "AI4Bharat",
    "url": "https://huggingface.co/datasets/ai4bharat/indic_glue",
    "n_languages": 11,
    "tasks": [
      "question_answering"
    ],
    "parallel": null,
    "base": "GLUE"
  },
  {
    "name": "CCAligned",
    "author": "Meta",
    "url": "https://huggingface.co/datasets/ahelk/ccaligned_multilingual",
    "n_languages": 137,
    "tasks": [
      "translation"
    ],
    "parallel": false
  },
  {
    "name": "Opus Gnome",
    "author": "Helsinki NLP",
    "url": "https://huggingface.co/datasets/Helsinki-NLP/opus_gnome",
    "n_languages": 187,
    "tasks": [
      "translation"
    ],
    "parallel": true
  },
  {
    "name": "Opus Paracrawl",
    "author": "Helsinki NLP",
    "url": "https://huggingface.co/datasets/Helsinki-NLP/opus_paracrawl",
    "n_languages": 43,
    "tasks": [
      "translation"
    ],
    "parallel": false
  },
  {
    "name": "OPUS Collection",
    "author": "Helsinki NLP",
    "url": "https://opus.nlpl.eu/",
    "n_languages": 747,
    "tasks": [
      "translation"
    ],
    "parallel": false
  },
  {
    "name": "WikiANN / PAN-X",
    "author": "Academic",
    "url": "https://huggingface.co/datasets/unimelb-nlp/wikiann",
    "n_languages": 176,
    "tasks": [
      "ner"
    ],
    "parallel": false
  },
  {
    "name": "MSVAMP",
    "author": "Microsoft",
    "url": "https://huggingface.co/datasets/Mathoctopus/MSVAMP",
    "n_languages": 10,
    "tasks": [
      "math"
    ],
    "parallel": true
  },
  {
    "name": "XLSUM",
    "author": "Academic",
    "url": "https://huggingface.co/datasets/csebuetnlp/xlsum",
    "n_languages": 45,
    "tasks": [
      "summarization"
    ],
    "parallel": true
  },
  {
    "name": "SEA-IFEVAL",
    "author": "AI Singapore",
    "url": "https://huggingface.co/datasets/aisingapore/instruction_following-ifeval",
    "n_languages": 7,
    "tasks": [
      "instruction_following"
    ],
    "parallel": true,
    "base": "IFEVAL"
  },
  {
    "name": "MasakhaNER",
    "author": "Masakhane",
    "url": "https://huggingface.co/datasets/masakhane/masakhaner",
    "n_languages": 10,
    "tasks": [
      "ner"
    ],
    "parallel": null
  },
  {
    "name": "Multilingual Sentiments",
    "url": "https://huggingface.co/datasets/tyqiangz/multilingual-sentiments",
    "n_languages": 12,
    "tasks": [
      "sentiment_analysis"
    ],
    "parallel": null
  },
  {
    "name": "CulturaX",
    "author": "Academic",
    "url": "https://huggingface.co/datasets/uonlp/CulturaX",
    "n_languages": 167,
    "tasks": [
      "language_modeling"
    ],
    "parallel": false
  },
  {
    "name": "T\u00fclu 3 SFT Mixture",
    "author": "AllenAI",
    "url": "https://huggingface.co/datasets/allenai/tulu-3-sft-mixture",
    "n_languages": 70,
    "tasks": [
      "instruction_following"
    ],
    "parallel": false
  },
  {
    "name": "xP3",
    "author": "BigScience",
    "url": "https://huggingface.co/datasets/bigscience/xP3",
    "n_languages": 46,
    "tasks": [
      "instruction_following"
    ],
    "parallel": false
  },
  {
    "name": "Aya",
    "author": "Cohere",
    "url": "https://huggingface.co/datasets/CohereForAI/aya_dataset",
    "n_languages": 65,
    "tasks": [
      "instruction_following"
    ],
    "parallel": null
  },
  {
    "name": "Lanfrica",
    "author": "Lanfrica",
    "url": "https://lanfrica.com/records?language=yor&task=machine%20translation",
    "n_languages": 2200,
    "tasks": [
      "datasets"
    ],
    "parallel": null
  },
  {
    "name": "HuggingFace Languages",
    "author": "HuggingFace",
    "url": "https://huggingface.co/languages",
    "n_languages": 4680,
    "tasks": [
      "datasets",
      "models"
    ],
    "parallel": null
  },
  {
    "name": "HuggingFace Multilingual Datasets",
    "author": "HuggingFace",
    "url": "https://huggingface.co/datasets?other=multilinguality:multilingual",
    "n_languages": null,
    "tasks": [
      "datasets"
    ],
    "parallel": false
  }
]