Fix evaluation score

classes.py

@@ -41,7 +41,7 @@ class SessionState:
         self.do_opportunity_analysis = True
         self.do_customer_research = True
         self.do_objections = False
-        self.add_objections_to_analysis = False
+        self.add_objections_to_analysis = True
         self.ask_objections = True
         self.use_objection_cache = True
         self.do_ragas_evaluation = False

utils_evaluate.py

@@ -10,10 +10,11 @@ from ragas.metrics import (
 from rouge_score import rouge_scorer
 from sentence_transformers import SentenceTransformer, util
 
-from utils_evaluate_objections import generate_objection_scores
+from utils_evaluate_objections import generate_objection_score
 
 
-def evaluate_objections(session):
+async def evaluate_objections(session):
+    print("evaluate_objections()")
 
     for response in session.responses:
         question = response.get("question", "")
@@ -24,8 +25,10 @@ def evaluate_objections(session):
         q_and_a = {
             "objection": question,
             "answer": answer
-        }         
-        score = generate_objection_scores(q_and_a)
+        }       
+        print(q_and_a)
+        score = await generate_objection_score(q_and_a)
+        print(score)
         response["evaluation_score"] = score
 
 

utils_evaluate_objections.py

@@ -1,18 +1,22 @@
-from ragas.metrics.base import MetricWithLLM, SingleTurnMetric
-from ragas.prompt.pydantic_prompt import PydanticPrompt
-from pydantic import BaseModel, Field
+import asyncio
+import dotenv
+import os
 import pandas as pd
-from typing import List, Tuple
-from datetime import datetime
 import sys
+import typing as t
+
 from dataclasses import dataclass, field
+from datetime import datetime
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel, Field
+from ragas import SingleTurnSample
+from ragas.llms.base import LangchainLLMWrapper
 from ragas.metrics.base import MetricType
-from ragas.messages import AIMessage, HumanMessage, ToolMessage, ToolCall
-from ragas import SingleTurnSample, MultiTurnSample
-import typing as t
-import asyncio
-import dotenv
-import os
+from ragas.metrics.base import MetricWithLLM, SingleTurnMetric
+from ragas.prompt.pydantic_prompt import PydanticPrompt
+from typing import List, Tuple
+    
+
 # Load environment variables from .env file
 dotenv.load_dotenv()
 
@@ -92,10 +96,8 @@ class SatisfyRate(MetricWithLLM, SingleTurnMetric):
         )
         return int(prompt_response.satisfy)
     
-async def generate_objection_scores(question_answer):
-    from langchain_openai import ChatOpenAI
-    from ragas.llms.base import LangchainLLMWrapper
-    import pandas as pd
+async def generate_objection_score(question_answer):
+    print("generate_objection_scores()")
     # user_response= pd.read_csv(file_path)
     openai_model = LangchainLLMWrapper(ChatOpenAI(model_name="gpt-4o", api_key=OPENAI_API_KEY))
     scorer = SatisfyRate(llm=openai_model)
@@ -104,6 +106,7 @@ async def generate_objection_scores(question_answer):
     
     #(user_response['objection'][num], user_response['response'][num])
     satisfy_0_1 = await scorer.single_turn_ascore(sample)
+    print(satisfy_0_1)
     
     print (question_answer['objection'], question_answer['answer'], satisfy_0_1)
     # Implement your logic to generate a response based on the user's input

utils_output.py

@@ -60,11 +60,13 @@ def format_datetime(dt):
 async def display_evaluation_results(cl, session_state):
     out_text = "*Preparing evaluation results ...*"
     await cl.Message(content=out_text).send()
-
+    print("Checking evaluation and objection flags")
+    print(session_state.do_evaluation)
+    print(session_state.add_objections_to_analysis)
     if session_state.do_evaluation:
         evaluate_answers(session_state) 
     elif session_state.add_objections_to_analysis:
-        evaluate_objections(session_state)
+        await evaluate_objections(session_state)
     await asyncio.sleep(1)
 
     output = f"**Session Summary**"
@@ -82,9 +84,9 @@ async def display_evaluation_results(cl, session_state):
         averages = results_df[columns_to_average].mean()
 
     await cl.Message(content="**Overall Summary (By SalesBuddy)**").send()
-    output = f"**SalesBuddy Score:** {session_state.responses[-1]['overall_score']} \n"
+    output = f"**SalesBuddy Score (1-10):** {session_state.responses[-1]['overall_score']} \n"
     output = output + f"**SalesBuddy Evaluation:** {session_state.responses[-1]['overall_evaluation']} \n"
-    output = output + f"**SalesBuddy Final Mood Score:** {session_state.responses[-1]['mood_score']} \n"
+    output = output + f"**SalesBuddy Final Mood Score (1-10):** {session_state.responses[-1]['mood_score']} \n"
     await cl.Message(content=output).send()
 
     if session_state.do_ragas_evaluation:
@@ -101,7 +103,7 @@ async def display_evaluation_results(cl, session_state):
             **Question:** {resp.get('question', 'N/A')}
             **Answer:** {resp.get('response', 'N/A')} 
             **SalesBuddy Evaluation:** {resp.get('response_evaluation', 'N/A')}
-            **Evaluation Score:** {resp.get('response_score', 'N/A')}
+            **Evaluation Score:** {resp.get('evaluation_score', 'N/A')}
         """
         if session_state.do_ragas_evaluation:
             scores = session_state.scores[index]

utils_prompt.py

@@ -103,6 +103,7 @@ def get_system_template_openai_short():
         You are playing a role in a conversation with a sales representative.
         Your name is in the 'Name:' section.
         They can use your first name, full name or address you with a title and last name.
+        If they get your name wrong, you can correct them once.
         Your name does not need to match exactly what they say.
         Be chatty and conversational and friendly.
         Your compnay information is in the 'Company:' section.
@@ -116,10 +117,12 @@ def get_system_template_openai_short():
         You can make conversation but you must follow the command.
         If a previous question and answer are provided, you must evaluate the rep's answer.
         You will perform evaluation based on how well and thoroughly the rep answered the previous question.
+        If the reps answer does not make sense or is not clear, set the score to a 1.
         If asked to provide a conclusion, you must consider all of the rep's answers to your questions.
         These are provided in the 'All questions and answers:' section.
         You will ALWAYS provide your response in valid JSON format
         Remember all string values must be enclosed in double quotes.
+        Remember do not include a question in your response.
         You will include with the following fields in JSON format:
         - Continue: Yes or No depending on if you want to continue the conversation based on the reps answer to your question.
         - Ask Follow Up: Yes or No depending on if you want to ask a follow up question.