first commit

.env.example

@@ -0,0 +1,3 @@
+OPENAI_API_KEY=
+NEWSAPI_API_KEY=
+REDDIT_ACCESS_TOKEN=

.gitignore

@@ -0,0 +1,7 @@
+.env
+.cache
+__pycache__
+poetry.lock
+*.yaml
+*.toml
+*.lock

Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.11
+
+# Create a non-root user with home directory
+RUN useradd -m -u 1000 user
+
+# Set user and environment variables
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory in the container
+WORKDIR $HOME/app
+
+# Copy the requirements.txt file to the container
+COPY requirements.txt $HOME/app/
+
+# Install Python dependencies from requirements.txt
+RUN pip install --user -r $HOME/app/requirements.txt
+
+# Copy the application files, including app.py
+COPY --chown=user:user . $HOME/app/
+
+# Ensure user has write permission to the app directory
+USER root
+RUN chown -R user:user $HOME/app
+USER user
+
+# Specify the command to run your application
+CMD ["chainlit", "run", "app.py", "--port", "7860"]

OAI_CONFIG_LIST

@@ -0,0 +1,6 @@
+[
+  {
+    "model": "gpt-4-1106-preview",
+    "api_key": "OPENAI_API_KEY"
+  }
+]

README.md

@@ -1,11 +1,52 @@
 ---
-title: Autogen Article Generator
-emoji: 🐢
-colorFrom: red
-colorTo: indigo
+title: Soap Opera
+emoji: 🔥
+colorFrom: indigo
+colorTo: green
 sdk: docker
 pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Installation and Setup
+You will need Python, Conda, Docker (Optional for code-execution), Git, and a text editor installed.
+
+First install python=3.11 and other 3rd party dependencies. If you have conda installed, you can run the following commands:
+
+```shell
+conda create --name demo python=3.11 -y
+conda activate demo
+
+pip install -r requirements.txt
+```
+
+If you do not have conda installed but have virtualenv installed, you can run the following commands:
+```shell
+pip install virtualenv
+virtualenv demo -p python3.
+
+# on windows
+demo\Scripts\activate
+# on mac/linux
+source demo/bin/activate
+
+pip install -r requirements.txt
+```
+
+# Configure the environment variables
+
+# Usage
+Run the following command to start the chat interface.
+
+```shell
+chainlit run app.py
+```
+
+# File Structure
+
+This is an example of using the chainlit chat interface with multi-agent conversation between agents to complete a tasks.
+
+The tool was developed to grab SAP data online and then process it to easily digestible human language.      
+ 
+`app.py` - Starts the chat interface.
+

app.py

@@ -0,0 +1,364 @@
+"""
+This is the source code for the webscraper agents that use ChainLit
+Features:
+- Uses top N google search based on a keyword, create a JSON file, uploads it to Google Cloud Object Storage or Locally
+- Capable of asking Reddit for questions
+- Continuous messaging
+- Multithreading 
+Written by: Antoine Ross - October 2023.
+"""
+
+import os
+from typing import Dict, Optional, Union
+from dotenv import load_dotenv, find_dotenv
+
+import chainlit as cl
+from chainlit.client.base import ConversationDict
+from chainlit.types import AskFileResponse
+from langchain.document_loaders import PyPDFLoader, TextLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.chains import ConversationalRetrievalChain
+
+import autogen
+from autogen import Agent, AssistantAgent, UserProxyAgent, config_list_from_json
+from redditAPI import grab_articles
+
+load_dotenv(find_dotenv())
+
+CONTEXT = """- Task: ArticleGPT specializes in creating SEO-optimized articles specifically tailored for [Medium.com](http://medium.com/). The target audience are people who are looking to get started with learning AI & Machine Learning and LLM’s for their use-case.
+    
+    
+    Output Specifications: 
+    
+    • Output Style and Format: Craft articles that fit [Medium.com](http://medium.com/)'s style, being straightforward and concise. Ensure grammatical accuracy, coherence, and stylistic refinement. Use hooks and effective whitespace management to maintain reader's attention. Humor and Sarcasm: Including a directive for humor and sarcasm to utilize rhetorical devices, which could make the text feel more human-like. 
+    
+    • Tone: The tone is conversational and likable, similar to Morgan Freeman's speech style.
+    
+    • Titles and Subheadings: Create titles and subheadings that are Impactful, concise and effectively capturing the content's essence. 
+    
+    • Titles: 5-9 words, with numbers for higher click-through rates. Prefer negative or neutral tones. 
+    
+    • Headlines: Structure in two parts, main and sub-headline. 
+    
+    • Subheadings: Spark curiosity with questions, action words, and numbers; emphasize benefits. 
+    
+    • Content balancing simplicity, engagement, and SEO optimization for Medium.
+    
+    • 
+    
+    Sample output:
+    
+    Title
+    
+    Subheading 1
+    
+    paragraph 1: Explain concisely the core of the article. How it can be useful for their use-case. (2-3 sentences)
+    
+    Subheading 2
+    
+    paragraph 2: Tell the readers how doing/having three things can dramatically improve results. (1-2 sentences)
+    
+    [3 bullet points or 3 numbered list to support paragraph 2]
+    
+    paragraph 3: summarize the bullet points and how it can be useful for the reader. (1-2 sentences)
+    
+    Subheading 3
+    
+    paragraph 4: Concluding Anecdote or Opinion: Requesting a final 'personal' touch is intended to leave the reader with a sense of individual perspective, something that machine-generated text often lacks. (2-3 sentences)
+    """
+
+# Agents
+USER_PROXY_NAME = "User Proxy"
+PROOF_READER = "Proofreader"
+WRITER = "Writer"
+EMOTIONAL_STRATEGIST = "Emotional Impact Strategist"
+NARRATIVE_DESIGNER = "Narrative Designer"
+STYLIST = "Style Specialist"
+ARTICLES = None
+
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=8192, chunk_overlap=100)
+
+def load_articles(file_path):
+    try:
+        with open(file_path, 'r') as file:
+            article = file.read()
+        return article
+    except FileNotFoundError:
+        print("File not found")
+        return None
+
+# Function to process the file
+def process_file(file: AskFileResponse):
+    import tempfile
+
+    if file.type == "text/plain":
+        Loader = TextLoader
+    elif file.type == "application/pdf":
+        Loader = PyPDFLoader
+
+    with tempfile.NamedTemporaryFile(mode="wb", delete=False) as tempfile:
+        if file.type == "text/plain":
+            tempfile.write(file.content)
+        elif file.type == "application/pdf":
+            with open(tempfile.name, "wb") as f:
+                f.write(file.content)
+
+        loader = Loader(tempfile.name)
+        documents = loader.load()
+        docs = text_splitter.split_documents(documents)
+        for i, doc in enumerate(docs):
+            doc.metadata["source"] = f"source_{i}"
+        cl.user_session.set("docs", docs)
+        return docs
+    
+async def ask_helper(func, **kwargs):
+    res = await func(**kwargs).send()
+    while not res:
+        res = await func(**kwargs).send()
+    return res
+
+class ChainlitAssistantAgent(AssistantAgent):
+    """
+    Wrapper for AutoGens Assistant Agent
+    """
+    def send(
+        self,
+        message: Union[Dict, str],
+        recipient: Agent,
+        request_reply: Optional[bool] = None,
+        silent: Optional[bool] = False,
+    ) -> bool:
+        cl.run_sync(
+            cl.Message(
+                content=f'*Sending message to "{recipient.name}":*\n\n{message}',
+                author=self.name,
+            ).send()
+        )
+        super(ChainlitAssistantAgent, self).send(
+            message=message,
+            recipient=recipient,
+            request_reply=request_reply,
+            silent=silent,
+        )
+class ChainlitUserProxyAgent(UserProxyAgent):
+    """
+    Wrapper for AutoGens UserProxy Agent. Simplifies the UI by adding CL Actions. 
+    """
+    def get_human_input(self, prompt: str) -> str:
+        if prompt.startswith(
+            "Provide feedback to chat_manager. Press enter to skip and use auto-reply"
+        ):
+            res = cl.run_sync(
+                ask_helper(
+                    cl.AskActionMessage,
+                    content="Continue or provide feedback?",
+                    actions=[
+                        cl.Action( name="continue", value="continue", label="✅ Continue" ),
+                        cl.Action( name="feedback",value="feedback", label="💬 Provide feedback"),
+                        cl.Action( name="exit",value="exit", label="🔚 Exit Conversation" )
+                    ],
+                )
+            )
+            if res.get("value") == "continue":
+                return ""
+            if res.get("value") == "exit":
+                return "exit"
+
+        reply = cl.run_sync(ask_helper(cl.AskUserMessage, content=prompt, timeout=60))
+
+        return reply["content"].strip()
+
+    def send(
+        self,
+        message: Union[Dict, str],
+        recipient: Agent,
+        request_reply: Optional[bool] = None,
+        silent: Optional[bool] = False,
+    ):
+        cl.run_sync(
+            cl.Message(
+                content=f'*Sending message to "{recipient.name}"*:\n\n{message}',
+                author=self.name,
+            ).send()
+        )
+        super(ChainlitUserProxyAgent, self).send(
+            message=message,
+            recipient=recipient,
+            request_reply=request_reply,
+            silent=silent,
+        )
+
+config_list = autogen.config_list_from_dotenv(
+    dotenv_file_path='.env',
+    model_api_key_map={
+        "gpt-3.5-turbo": "OPENAI_API_KEY",
+    },
+    filter_dict={
+        "model": {
+            "gpt-3.5-turbo",
+        }
+    }
+)
+
+@cl.action_callback("confirm_action")
+async def on_action(action: cl.Action):
+    if action.value == "everything":
+        content = "everything"
+    elif action.value == "top-headlines":
+        content = "top_headlines"
+    else:
+        await cl.ErrorMessage(content="Invalid action").send()
+        return
+
+    prev_msg = cl.user_session.get("url_actions")  # type: cl.Message
+    if prev_msg:
+        await prev_msg.remove_actions()
+        cl.user_session.set("url_actions", None)
+
+    await cl.Message(content=content).send()
+    
+@cl.on_chat_start
+async def on_chat_start():
+  OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+  
+
+  try:
+    # app_user = cl.user_session.get("user")
+    # await cl.Message(f"Hello {app_user.username}").send()
+    # config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
+    llm_config = {"config_list": config_list, "api_key": OPENAI_API_KEY, "seed": 42, "request_timeout": 60, "retry_wait_time": 60}
+    proof_reader = ChainlitAssistantAgent(
+        name="Proof_Reader", llm_config=llm_config,
+        system_message="""Proofreader. Focuses on grammatical accuracy and stylistic refinement, ensuring that articles meet Medium.com's standards.
+Enhances clarity and coherence while maintaining a conversational, likable tone akin to Morgan Freeman's speech style.
+Assures the use of effective hooks and whitespace management to keep the reader's attention.
+Ensures articles are straightforward, concise, and free of filler words, with minimal use of humor and sarcasm."""
+    )
+    writer = ChainlitAssistantAgent(
+        name="Writer", llm_config=llm_config,
+        system_message="""Writer. Develops SEO-optimized, engaging content tailored for Medium.com's audience interested in AI & Machine Learning.
+Writes with a conversational and likable tone, ensuring simplicity and engagement.
+Crafts impactful, concise titles and subheadings, with titles of 5-9 words incorporating numbers, and negative or neutral tones.
+Structures content with effective subheadings and bullet points to facilitate reader understanding and engagement."""
+    )
+    narrative_designer = ChainlitAssistantAgent(
+        name="Narrative_Designer", llm_config=llm_config,
+        system_message="""Narrative Designer. Structures the article to maintain engagement and curiosity, using questions, action words, and numbers in subheadings.
+Collaborates with the Writer and Emotional Impact Strategist to ensure the narrative is clear, concise, and resonates with the target audience.
+Advises on the narrative flow to maintain reader interest and optimize for SEO."""
+    )
+    stylist = ChainlitAssistantAgent(
+        name="Style_Specialist", llm_config=llm_config,
+        system_message="""Style Specialist. Refines tone and style to be conversational and likable, aligning with the Morgan Freeman style.
+Ensures the use of effective rhetoric, including minimal humor and sarcasm, to enhance readability and engagement.
+Collaborates with the Writer and Proofreader to ensure stylistic consistency throughout the article."""
+    )
+    emotional_impact_strategist = ChainlitAssistantAgent(
+        name="Emotional_Strategist", llm_config=llm_config,
+        system_message="""Develops strategies for titles and subheadings that are impactful, concise, and evoke curiosity.
+Advises on incorporating emotional cues that resonate with the audience's interests in AI and Machine Learning.
+Collaborates with the Narrative Designer and Style Specialist to ensure a unified approach in content framing.
+        """
+    )
+    user_proxy = ChainlitUserProxyAgent(
+        name="User_Proxy",
+        human_input_mode="ALWAYS",
+        llm_config=llm_config,
+        # max_consecutive_auto_reply=3,
+        # is_termination_msg=lambda x: x.get("content", "").rstrip().endswith("TERMINATE"),
+        code_execution_config=False,
+        system_message="""User Proxy. Provides feedback on the article's effectiveness in engaging readers interested in AI and Machine Learning.
+Ensures the article meets overall objectives and resonates with the intended audience.
+Relays audience and management preferences to the team for necessary adjustments."""
+    )
+    
+    cl.user_session.set(USER_PROXY_NAME, user_proxy)
+    cl.user_session.set(PROOF_READER, proof_reader)
+    cl.user_session.set(WRITER, writer)
+    cl.user_session.set(STYLIST, stylist)
+    cl.user_session.set(NARRATIVE_DESIGNER, narrative_designer)
+    cl.user_session.set(EMOTIONAL_STRATEGIST, emotional_impact_strategist)
+    
+    doc = cl.Action( name="doc", value="doc", label="Document" )
+    no_doc = cl.Action( name="no_doc", value="no_doc", label="NoDocument" )
+    idea = cl.Action( name="Idea", value="Idea", label="Idea" )
+    no_idea = cl.Action( name="NoIdea", value="NoIdea", label="NoIdea" )
+    idea_actions = [idea, no_idea]
+    doc_actions = [doc, no_doc]
+    
+    IDEA_option = cl.AskActionMessage(
+        content="Hi, let’s generate some Article ideas. Would you like to generate ideas from Reddit, or continue?",
+        actions=idea_actions,
+    )
+    await IDEA_option.send()
+    
+    IDEA_option = IDEA_option.content.split()[-1]
+    if IDEA_option == "Idea":
+        print("Using document...")
+        TOPIC = None
+        while TOPIC is None:
+            TOPIC = await cl.AskUserMessage(content="What topic would you like to make an Article about? [Only send one keyword.]", timeout=180).send()
+
+        print("Topic: ", TOPIC['content'])
+        msg = cl.Message(
+        content=f"Processing data from Reddit...", disable_human_feedback=True
+        )
+        await msg.send()
+        
+        articles = grab_articles(TOPIC['content'])
+        msg = cl.Message(
+            content=f"Content from Reddit loaded: \n{articles}", disable_human_feedback=True
+        )
+        await msg.send()
+    elif IDEA_option == "NoIdea":
+        article_path = "articles.txt"
+        articles = load_articles(article_path)
+    print("Articles grabbed.")
+    
+    msg = cl.Message(content=f"Processing `{articles}`...", disable_human_feedback=True, author="User_Proxy")
+    await msg.send()
+    
+    cl.user_session.set(ARTICLES, articles)
+    print("Articles set...")
+    
+    msg = cl.Message(content=f"""This is the Article Generation Team, please give a topic to create an Article about.""", 
+                     disable_human_feedback=True, 
+                     author="User_Proxy")
+    await msg.send()
+    
+  except Exception as e:
+    print("Error: ", e)
+    pass
+
+@cl.on_message
+async def run_conversation(message: cl.Message):
+  #try:
+    MESSAGE = message.content
+    print("Task: ", MESSAGE)
+    proof_reader = cl.user_session.get(PROOF_READER)
+    user_proxy = cl.user_session.get(USER_PROXY_NAME)
+    writer = cl.user_session.get(WRITER)
+    stylist = cl.user_session.get(STYLIST)
+    narrative_designer = cl.user_session.get(NARRATIVE_DESIGNER)
+    emotional_impact_strategist = cl.user_session.get(EMOTIONAL_STRATEGIST)
+    articles = cl.user_session.get(ARTICLES)
+
+    groupchat = autogen.GroupChat(agents=[user_proxy, proof_reader, writer,stylist, narrative_designer,emotional_impact_strategist  ], messages=[], max_round=50)
+    manager = autogen.GroupChatManager(groupchat=groupchat)
+    
+    print("Initiated GC messages... \nGC messages length: ", len(groupchat.messages))
+
+    if len(groupchat.messages) == 0:
+      message = f"""Use this content as background for the articles you will make: {articles}. 
+                First create 10 ideas, then 5, then 3, then 1.
+                Finalize the ideas with the planner and make sure to follow the criteria of choosing based on: "What will be the most dramatic, emotional and entertaining idea".
+                Do not express gratitude in responses.
+                \nThe topic of the article will be about: """ + MESSAGE + """The final output should look like: \n""" + CONTEXT
+      await cl.Message(content=f"""Starting agents on task of creating a Article...""").send()
+      await cl.make_async(user_proxy.initiate_chat)( manager, message=message, )
+    else:
+      await cl.make_async(user_proxy.send)( manager, message=MESSAGE, )
+      
+#   except Exception as e:
+#     print("Error: ", e)
+#     pass

articles.txt

@@ -0,0 +1 @@
+[Topic to help the Article generation]

chainlit.md

@@ -0,0 +1,3 @@
+# Autogen Article Generator
+
+This is an example of using the chainlit chat interface with multi-agent conversation between agents to make sophisticated articles.

newsAPI.py

@@ -0,0 +1,59 @@
+import requests
+import json
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+API_KEY = 'NEWSAPI_API_KEY'
+
+# Functions -------------------------------------------------------------------------------------------------------------> 
+def extract_article_info_from_list(article_list):   
+    try:
+        # Initialize a string to store the extracted information
+        extracted_info = ""
+
+        # Extract and store the desired information for each article
+        for article in article_list:
+            description = article.get("description", "N/A")
+            title = article.get("title", "N/A")
+            author = article.get("author", "N/A")
+            content = article.get("content", "N/A")
+
+            # Append the information to the string
+            extracted_info += f"Title: {title}\nAuthor: {author}\nDescription: {description}\nContent: {content}\n\n"
+
+        return extracted_info
+
+    except Exception as e:
+        # Handle any exceptions
+        return f"An error occurred: {str(e)}"
+      
+def grab_articles(url, **kwargs):
+    response = requests.get(url, 5)
+    
+    if response.status_code == 200:
+        extracted_article = extract_article_info_from_list(response.json()['articles'])
+        print(extracted_article)
+        return extracted_article
+    else:
+        raise Exception(f"Error: {response.status_code}, {response.text}")
+
+# URL Configuration Here -------------------------------------------------------------------------------------------------------------> 
+
+BASE_URL_EVERYTHING = 'https://newsapi.org/v2/everything?'
+BASE_URL_TOP_HEADLINES = 'https://newsapi.org/v2/top-headlines?'
+TOPIC = 'bitcoin'
+FROM = '2023-01-08' # Starting date
+TO = '2023-11-08' # Ending date
+SORTBY = 'popularity' # Relevancy, Popularity, PublishedAt
+DOMAINS = 'techcrunch.com, thenextweb.com' # Domains you want to use. 
+COUNTRY = 'gb&' # Options: US, AU, JP, PH etc. 
+SOURCES = 'bbc-news' 
+CATEGORY = 'business' # Options: business, entertainment, general health, science, sports, technology
+
+# Everything URL list
+urls_everything = f"{BASE_URL_EVERYTHING}q={TOPIC}&domains={DOMAINS}&sortBy={SORTBY}&apiKey={API_KEY}" 
+
+# Top Headlines URL list
+urls_top_headlines = f"{BASE_URL_TOP_HEADLINES}topic={TOPIC}&country={COUNTRY}&category={CATEGORY}&sources={SOURCES}&apiKey={API_KEY}"

redditAPI.py

@@ -0,0 +1,59 @@
+import requests
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+access_token = os.getenv('REDDIT_ACCESS_TOKEN')
+
+headers = {
+    'Authorization': f'bearer {access_token}',
+    'User-Agent': 'MyAPI/0.0.1'
+}
+
+# Subreddits of interest.
+subreddits = [
+    'artificialintelligence', 'machinelearning', 'indiehacking',
+]
+
+# Function to fetch top 3 comments from a post
+def fetch_top_comments(post_id):
+    url = f'https://oauth.reddit.com/comments/{post_id}'
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        post_data = response.json()
+        comments = post_data[1]['data']['children']
+        top_comments = [comment['data']['body'] for comment in comments[:1]]  # Get top 3 comments
+        return top_comments
+    else:
+        return []
+
+def grab_articles(topic):
+  data = []
+  print("Topic: ", topic)
+  for subreddit in subreddits:
+      # Search for the topic within each subreddit. Increase limit to get more posts.
+      search_url = f'https://oauth.reddit.com/r/{subreddit}/search?q={topic}&restrict_sr=on&sort=hot&limit=1'
+      res = requests.get(search_url, headers=headers)
+
+      if res.status_code == 200:
+          posts_data = res.json()['data']['children']
+
+          for post in posts_data:
+              post_data = post['data']
+              post_id = post_data['id']
+              title = post_data['title']
+              selftext = post_data['selftext']
+              top_comments = fetch_top_comments(post_id)
+              data.append({
+                  'subreddit': subreddit,
+                  'title': title,
+                  'post_content': selftext,
+                  'top_comments': top_comments
+              })
+      else:
+          print(f"Failed to fetch data from {subreddit}")
+          
+  # Example output
+  print(data)
+  return data

requirements.txt

@@ -0,0 +1,5 @@
+chainlit==0.7.501
+pyautogen==0.1.13
+python-dotenv==1.0.0
+requests==2.31.0
+langchain==0.0.322