Update app.py

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import requests
 import os
+from bs4 import BeautifulSoup  # Add BeautifulSoup for scraping
 
 # Load API keys securely from environment variables
 proxycurl_api_key = os.getenv("PROXYCURL_API_KEY")  # Proxycurl API key
@@ -25,6 +26,10 @@ class EmailAgent:
     # Reason: Decide what information is needed
     def reason_about_data(self):
         print("Reasoning: I need LinkedIn data, company info, and role description.")
+        if not self.linkedin_url:
+            print("Warning: LinkedIn URL missing. Will proceed with default bio.")
+        if not self.company_name:
+            print("Warning: Company name missing. Will proceed with default company info.")
     
     # Action: Fetch LinkedIn data via Proxycurl
     def fetch_linkedin_data(self):
@@ -69,16 +74,47 @@ class EmailAgent:
             else:
                 print(f"Error: Unable to fetch company info for {self.company_name}. Using default info.")
                 self.company_info = "A leading company in its field, offering innovative solutions."
-    
-    # Action: Fetch role description
-    def fetch_role_description(self):
-        print(f"Action: Fetching role description for {self.role}.")
+
+    # Action: Scrape the company's website for role-specific information
+    def scrape_role_from_website(self):
+        print(f"Action: Scraping role description from the company's website for {self.role}.")
+        if not self.company_name:
+            print("Error: No company name or URL provided for scraping.")
+            return False
+        
+        # Attempt to scrape the company's website
+        try:
+            response = requests.get(f"https://{self.company_name}.com/careers")
+            if response.status_code == 200:
+                soup = BeautifulSoup(response.text, 'html.parser')
+                # Look for any sections that might contain role descriptions
+                role_descriptions = soup.find_all(string=lambda text: self.role.lower() in text.lower())
+                if role_descriptions:
+                    # If we find relevant role descriptions, use the first match
+                    self.role_description = role_descriptions[0]
+                    print(f"Found role description on company's website: {self.role_description}")
+                    return True
+                else:
+                    print(f"No specific role description found on the website for {self.role}.")
+                    return False
+            else:
+                print(f"Error: Unable to reach company's website at {self.company_name}.com.")
+                return False
+        except Exception as e:
+            print(f"Error during scraping: {e}")
+            return False
+
+    # Action: Use default logic to infer role description if scraping fails
+    def use_default_role_description(self):
+        print(f"Action: Using default logic for the role of {self.role}.")
         self.role_description = f"The role of {self.role} at {self.company_name} involves mentoring AI and technology students to develop their skills and progress their careers."
 
     # Reflection: Check if the data is sufficient to generate an email
     def reflect_on_data(self):
         print("Reflection: Do I have enough data to generate the email?")
         # Allow the email to be generated with default values if data is missing
+        if not self.bio or not self.skills or not self.company_info:
+            print("Warning: Some critical information is missing. Proceeding with default values.")
         return True
     
     # Action: Generate the email using Groq Cloud LLM
@@ -128,7 +164,9 @@ class EmailAgent:
         self.reason_about_data()  # Reason
         self.fetch_linkedin_data()  # Action
         self.fetch_company_info()  # Action
-        self.fetch_role_description()  # Action
+        # Try to scrape the company's website for role-specific information
+        if not self.scrape_role_from_website():  
+            self.use_default_role_description()  # Use default logic if scraping fails
         if self.reflect_on_data():  # Reflection
             return self.generate_email()  # Final Action
         else:
@@ -169,3 +207,5 @@ def gradio_ui():
 # Start the Gradio app when running the script
 if __name__ == "__main__":
     gradio_ui()
+
+