init push

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.10
+
+WORKDIR /app
+
+COPY ./vectara_theme.py /app/vectara_theme.py
+COPY ./requirements.txt /app/requirements.txt
+COPY ./app.py /app/app.py
+
+# RUN apt-get update && apt-get install -y git-lfs
+
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+
+# RUN useradd -m -u 1000 user
+# ENV HOME=/home/user \
+# 	PATH=/home/user/.local/bin:$PATH
+
+# RUN mkdir -p /app/results
+# RUN chown -R user /app
+
+CMD ["funix", "app.py", "--host", "0.0.0.0", "--port", "7860", "--no-browser"]

README.md

@@ -8,4 +8,6 @@ pinned: false
 short_description: The GUI demo for HHEM-2.1-Open
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Vectara's HHEM 2.1 Open demo
+
+Powered by [Funix.io](http://funix.io)

app.py

@@ -0,0 +1,46 @@
+# %%
+from typing import List, Literal
+from pydantic import BaseModel
+from IPython.display import display, Markdown
+
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import pipeline
+
+# %%
+class HHEMOutput(BaseModel):
+    score: float # we need score for ROC curve
+    label: Literal[0,1]
+
+# %% 
+PROMPT_TEMPLATE = "<pad> Determine if the hypothesis is true given the premise?\n\nPremise: {text1}\n\nHypothesis: {text2}"
+
+CHECKPOINT = "vectara/hallucination_evaluation_model"
+FOUNDATION = "google/flan-t5-small"
+
+tokenizer = AutoTokenizer.from_pretrained(FOUNDATION)
+classifier = pipeline("text-classification", model=CHECKPOINT, tokenizer=tokenizer, trust_remote_code=True)
+
+def predict(premise: str, hypothesis: str) -> Markdown:
+    
+    texts_prompted: List[str] = [PROMPT_TEMPLATE.format(text1=premise, text2=hypothesis)]
+
+    full_scores = classifier(texts_prompted, top_k=None) # List[List[Dict[str, float]]]
+
+    # Optional: Extract the scores for the 'consistent' label
+    simple_scores = [score_dict['score'] for score_for_both_labels in full_scores for score_dict in score_for_both_labels if score_dict['label'] == 'consistent']
+
+    threshold = 0.5
+    preds = [0 if s < threshold else 1 for s in simple_scores]
+
+    output = HHEMOutput(score=simple_scores[0], label=preds[0])
+    verdict = "consistent" if output.label == 1 else "hallucinated"
+
+    output_string = f"""
+    **Premise**: {premise}
+
+    **Hypothesis**: {hypothesis}
+    
+    **HHEM's judgement is**: {verdict} **with the score**: {output.score}
+    """
+    
+    return Markdown(output_string)

requirements.txt

@@ -0,0 +1,32 @@
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.7
+datasets==2.14.5
+gradio==4.44.0
+gradio_client==1.3.0
+huggingface-hub>=0.18.0
+litellm==1.15.1
+matplotlib==3.7.1
+numpy==1.26.4
+pandas==2.0.0
+python-dateutil==2.8.2
+requests==2.31.0
+tqdm==4.66.5
+tokenizers>=0.15.0
+sentence-transformers==2.2.2
+google-generativeai
+replicate
+anthropic
+openai
+cohere
+mistralai
+peft
+markdown-it-py
+mdit_plain
+google-cloud-aiplatform>=1.38
+qwen-vl-utils
+vertexai
+# git+https://github.com/huggingface/transformers
+transformers==4.45.2
+together==1.3.0
+spacy

vectara_theme.py

@@ -0,0 +1,29 @@
+vectara_theme = {
+    "name": "vectara",
+    "funix": {
+        "run_button": "Refresh",
+        "grid_height": 960,
+        "grid_checkbox": False
+    },
+    "overrides": {
+        "MuiAppBar": {
+            "styleOverrides": {
+                "root": {
+                    "backgroundColor": "#ffffff",
+                    "color": "rgba(0, 0, 0, 0.87)",
+                    "& .MuiToolbar-root:before": {
+                        "content": '""',
+                        "background": "url('https://huggingface.co/spaces/vectara/README/resolve/main/Vectara-logo.png')",
+                        "display": "block",
+                        "background-size": "contain",
+                        "background-repeat": "no-repeat",
+                        "background-position": "left",
+                        "width": "125px",
+                        "height": "40px",
+                        "margin-right": "10px",
+                    },
+                },
+            }
+        },
+    },
+}