oist/multimodal_nli_model

Multilingual & Multimodal NLI (MMNLI)

The full details of the MMNLI model, including architecture, training, and evaluation, are described in the paper Beyond Similarity Scoring: Detecting Entailment and Contradiction in Multilingual and Multimodal Contexts by Istaiteh, O., Mdhaffar, S., & Estève, Y. (Interspeech 2025). Please cite this paper if you use the MMNLI model in your research.

This repository provides the MMNLI model, a multilingual and multimodal Natural Language Inference classifier.
It extends the BLASER architecture into multiclass NLI, supporting entailment, contradiction, and neutrality across text-text, text-speech, speech-text, and speech-speech input pairs.

The model is trained on the oist/multimodal_nli_dataset.
Please refer to that dataset card for details.

Results

On the test set of the dataset, the MMNLI model achieves an F1-micro score of 0.749.

---

Usage

The model depends on SONAR embeddings. You can use the official SONAR encoders (for text and speech) from GitHub or the ported SONAR text encoder cointegrated/SONAR_200_text_encoder.

---

Example 1: Speech–Text Inference

import torch
from sonar.inference_pipelines.speech import SpeechToEmbeddingModelPipeline
from sonar.inference_pipelines.text import TextToEmbeddingModelPipeline
from transformers import AutoModel

# 1. Load SONAR encoders
speech_encoder = SpeechToEmbeddingModelPipeline(encoder="sonar_speech_encoder_eng")
text_encoder = TextToEmbeddingModelPipeline(encoder="text_sonar_basic_encoder", tokenizer="text_sonar_basic_encoder")

# 2. Encode premise (speech) and hypothesis (text)
premise_embs = speech_encoder.predict(["audio.wav"])
hypothesis_embs = text_encoder.predict(["The cat sat on the mat."], source_lang="eng_Latn")

# 3. Load MMNLI model 
mmnli_model_name = "oist/multimodal_nli_model"
mmnli_model = AutoModel.from_pretrained(mmnli_model_name, trust_remote_code=True)
mmnli_model.eval()

# 4. Run inference
with torch.inference_mode():
    logits = mmnli_model(premise_embs, hypothesis_embs)  # returns [batch_size, 3]
    pred_class = torch.argmax(logits, dim=-1).item()

print("Prediction:", pred_class)
# 0 = Entailment, 1 = Neutral, 2 = Contradiction

Example 2: Text–Text Inference (Official SONAR)

import torch
from sonar.inference_pipelines.text import TextToEmbeddingModelPipeline
from transformers import AutoModel

# 1. Load official SONAR text encoder
text_encoder = TextToEmbeddingModelPipeline(
    encoder="text_sonar_basic_encoder", 
    tokenizer="text_sonar_basic_encoder"
)

# 2. Encode premise and hypothesis
premise_texts = ["Le chat s'assit sur le tapis."]
hypothesis_texts = ["The cat sat on the mat."]

premise_embs = text_encoder.predict(premise_texts, source_lang="fra_Latn")
hypothesis_embs = text_encoder.predict(hypothesis_texts, source_lang="eng_Latn")

# 3. Load MMNLI model 
mmnli_model = AutoModel.from_pretrained("oist/multimodal_nli_model", trust_remote_code=True)
mmnli_model.eval()

# 4. Run inference
with torch.inference_mode():
    logits = mmnli_model(premise_embs, hypothesis_embs)
    pred_class = torch.argmax(logits, dim=-1).item()

print("Prediction:", pred_class)
# 0 = Entailment, 1 = Neutral, 2 = Contradiction

Example 3: Text–Text Inference (Ported SONAR)

# !pip install transformers sentencepiece  torch -q
import torch
from transformers import AutoTokenizer, AutoModel
from transformers.models.m2m_100.modeling_m2m_100 import M2M100Encoder

# 1. Load ported SONAR text encoder
sonar_model_name = "cointegrated/SONAR_200_text_encoder"
encoder = M2M100Encoder.from_pretrained(sonar_model_name)
tokenizer = AutoTokenizer.from_pretrained(sonar_model_name)

def encode_mean_pool(texts, tokenizer, encoder, lang='eng_Latn', norm=False):
    tokenizer.src_lang = lang
    with torch.inference_mode():
        batch = tokenizer(texts, return_tensors='pt', padding=True)
        seq_embs = encoder(**batch).last_hidden_state
        mask = batch.attention_mask
        mean_emb = (seq_embs * mask.unsqueeze(-1)).sum(1) / mask.unsqueeze(-1).sum(1)
        if norm:
            mean_emb = torch.nn.functional.normalize(mean_emb)
    return mean_emb

# Example sentences
premise_sentences = ["Le chat s'assit sur le tapis."]
hypothesis_sentences = ["The cat sat on the mat."]

# 2. Encode premise and hypothesis
premise_embs = encode_mean_pool(premise_sentences, tokenizer, encoder, lang="fra_Latn")
hypothesis_embs = encode_mean_pool(hypothesis_sentences, tokenizer, encoder, lang="eng_Latn")


mmnli_model_name = "oist/multimodal_nli_model"
mmnli_model = AutoModel.from_pretrained(mmnli_model_name, trust_remote_code=True)
mmnli_model.eval()

# 4. Run inference
with torch.inference_mode():
    logits = mmnli_model(premise_embs, hypothesis_embs)  # returns [batch_size, 3]
    pred_class = torch.argmax(logits, dim=-1).item()

print("Prediction:", pred_class)
# 0 = Entailment, 1 = Neutral, 2 = Contradiction

Example 4: Using BLASER Semantic Score with MMNLI

You can use the BLASER semantic score in combination with the MMNLI NLI class to get a better understanding of the relationship between source and candidate translations. The NLI class gives the entailment/contradiction/neutral label, while the BLASER score provides a fine-grained semantic similarity.

# !pip install transformers sentencepiece  torch -q
import torch
from transformers import AutoTokenizer, AutoModel
from transformers.models.m2m_100.modeling_m2m_100 import M2M100Encoder

# -------------------------
# 1️⃣ Load ported SONAR text encoder
# -------------------------
sonar_model_name = "cointegrated/SONAR_200_text_encoder"
encoder = M2M100Encoder.from_pretrained(sonar_model_name)
tokenizer = AutoTokenizer.from_pretrained(sonar_model_name)

def encode_mean_pool(texts, tokenizer, encoder, lang='eng_Latn', norm=False):
    tokenizer.src_lang = lang
    with torch.inference_mode():
        batch = tokenizer(texts, return_tensors='pt', padding=True)
        seq_embs = encoder(**batch).last_hidden_state
        mask = batch.attention_mask
        mean_emb = (seq_embs * mask.unsqueeze(-1)).sum(1) / mask.unsqueeze(-1).sum(1)
        if norm:
            mean_emb = torch.nn.functional.normalize(mean_emb)
    return mean_emb

# -------------------------
# 2️⃣ Example sentences
# -------------------------
src_sentence = ["He is happy."]
mt_sentences = [
    "Il est content.",                   # entailment blaser:4.515
    "Il est malheureux."                  # contradiction blaser: 4.41
]

# Encode source and MT sentences
src_embs = encode_mean_pool(src_sentence, tokenizer, encoder, lang="eng_Latn")
mt_embs = encode_mean_pool(mt_sentences, tokenizer, encoder, lang="fra_Latn")

# -------------------------
# 3️⃣ Load MMNLI model
# -------------------------
mmnli_model_name = "oist/multimodal_nli_model"
mmnli_model = AutoModel.from_pretrained(mmnli_model_name, trust_remote_code=True)
mmnli_model.eval()

# -------------------------
# 4️⃣ Load BLASER QE model
# -------------------------
qe_model_name = "oist/blaser_2_0_qe_ported"
qe_model = AutoModel.from_pretrained(qe_model_name, trust_remote_code=True)
qe_model.eval()

# -------------------------
# 5️⃣ Run inference
# -------------------------
for i, mt_sentence in enumerate(mt_sentences):
    mt_emb = mt_embs[i].unsqueeze(0)  # keep batch dimension
    
    # NLI prediction
    with torch.inference_mode():
        logits = mmnli_model(src_embs, mt_emb)
        pred_class = torch.argmax(logits, dim=-1).item()
    
    # BLASER semantic score
    with torch.inference_mode():
        qe_score = qe_model(src_embs, mt_emb)  # shape [1, 1]
    
    print(f"\nMT sentence: '{mt_sentence}'")
    print("NLI prediction:", ["Entailment", "Neutral", "Contradiction"][pred_class])
    print("BLASER semantic score:", qe_score.item())

---

Labels

• 0 = Entailment
• 1 = Neutral
• 2 = Contradiction

---

Citation

If you use this model, please cite:

@inproceedings{istaiteh2025beyond,
  title={Beyond Similarity Scoring: Detecting Entailment and Contradiction in Multilingual and Multimodal Contexts},
  author={Istaiteh, Othman and Mdhaffar, Salima and Est{\`e}ve, Yannick},
  booktitle={Proc. Interspeech 2025},
  pages={286--290},
  year={2025}
}