import streamlit as st
import pandas as pd
import torch
from transformers import pipeline
import datetime
from rapidfuzz import process, fuzz
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
# Load the CSV file
df = pd.read_csv("anomalies.csv", quotechar='"')
# Filter 'real' higher than 10 Million
df= df[df['real'] >= 1000000.]
# Convert 'real' column to standard float format and then to strings
df['real'] = df['real'].apply(lambda x: f"{x:.2f}")
# Fill NaN values and convert all columns to strings
df = df.fillna('').astype(str)
print(df)
# Function to remove stopwords
def remove_stopwords(text, stopwords=ENGLISH_STOP_WORDS):
return ' '.join([word for word in text.split() if word.lower() not in stopwords])
# Function to filter DataFrame by checking if any of the user question words are in the columns
def filter_dataframe(df, user_question, threshold=80):
user_question = remove_stopwords(user_question) # Remove stopwords
question_words = user_question.split()
mask = pd.Series([False] * len(df), index=df.index)
for column in df.columns:
for word in question_words:
# Apply RapidFuzz fuzzy matching on the column
matches = process.extract(word, df[column], scorer=fuzz.token_sort_ratio, limit=None)
match_indices = [match[2] for match in matches if match[1] >= threshold]
mask.loc[match_indices] = True # Ensure the mask is aligned with the DataFrame index
filtered_df = df[mask]
return filtered_df
# Function to generate a response using the TAPAS model
def response(user_question, df):
a = datetime.datetime.now()
# Filter the DataFrame dynamically by user question
subset_df = filter_dataframe(df, user_question)
# Check if the DataFrame is empty
if subset_df.empty:
return {"Resposta": "Desculpe, não há dados disponíveis para responder à sua pergunta."}
# Initialize the TAPAS model
tqa = pipeline(task="table-question-answering", model="google/tapas-large-finetuned-wtq",
tokenizer_kwargs={"clean_up_tokenization_spaces": False})
# Debugging information
print("Filtered DataFrame shape:", subset_df.shape)
print("Filtered DataFrame head:\n", subset_df.head())
print("User question:", user_question)
# Query the TAPAS model
try:
answer = tqa(table=subset_df, query=user_question)['answer']
except ValueError as e:
print(f"Error: {e}")
answer = "Desculpe, ocorreu um erro ao processar sua pergunta."
query_result = {
"Resposta": answer
}
b = datetime.datetime.now()
print("Time taken:", b - a)
return query_result
# Streamlit interface
st.markdown("""
""", unsafe_allow_html=True)
# Chat history
if 'history' not in st.session_state:
st.session_state['history'] = []
# Input box for user question
user_question = st.text_input("Escreva sua questão aqui:", "")
if user_question:
# Add human emoji when user asks a question
st.session_state['history'].append(('👤', user_question))
st.markdown(f"**👤 {user_question}**")
# Generate the response
bot_response = response(user_question, df)["Resposta"]
# Add robot emoji when generating response and align to the right
st.session_state['history'].append(('🤖', bot_response))
st.markdown(f"**🤖 {bot_response}**
", unsafe_allow_html=True)
# Clear history button
if st.button("Limpar"):
st.session_state['history'] = []
# Display chat history
for sender, message in st.session_state['history']:
if sender == '👤':
st.markdown(f"**👤 {message}**")
elif sender == '🤖':
st.markdown(f"**🤖 {message}**
", unsafe_allow_html=True)