Duplicate from asiffarhankhan/custom-gpt-voice-assistant

Co-authored-by: Asif Farhan Khan <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,6 @@
+.venv
+__pycache__
+.chroma
+initialize.sh
+conversations.log
+custom_gpt_voice assistant_demo.mp4

README.md

@@ -0,0 +1,14 @@
+---
+title: Custom Gpt Voice Assistant
+emoji: 🌖
+colorFrom: indigo
+colorTo: gray
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: false
+license: other
+duplicated_from: asiffarhankhan/custom-gpt-voice-assistant
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,132 @@
+import os
+import nltk 
+import openai
+import time
+import gradio as gr
+from threading import Thread
+
+from assets.char_poses_base64 import (
+    CHAR_IDLE_HTML, CHAR_THINKING_HTML, CHAR_TALKING_HTML)
+
+from app_utils import (
+    get_chat_history, initialize_knowledge_base, 
+    text_to_speech_gen, logging, buzz_user)
+
+global FUNC_CALL
+FUNC_CALL = 0
+
+global BUZZ_TIMEOUT
+BUZZ_TIMEOUT = 60
+
+GENERAL_RSPONSE_TRIGGERS = ["I don't understand the question.", "I don't know", "Hello, my name is", "mentioned in the context provided"]
+MESSAGES = [{"role": "system", "content": "You are a helpful assistant.."}]
+
+LOGGER = logging.getLogger('voice_agent')
+AUDIO_HTML = ''
+
+# Uncomment If this is your first Run: 
+nltk.download('averaged_perceptron_tagger')
+conv_model, voice_model = initialize_knowledge_base()
+
+
+def idle_timer():
+    global BUZZ_TIMEOUT
+
+    while True:
+        time.sleep(BUZZ_TIMEOUT)
+        buzz_user()
+
+        if BUZZ_TIMEOUT == 80:
+            time.sleep(BUZZ_TIMEOUT)
+            BUZZ_TIMEOUT = 60
+
+
+def update_img():
+    global FUNC_CALL
+    FUNC_CALL += 1
+
+    if FUNC_CALL % 2== 0:
+        return CHAR_TALKING_HTML
+    else:
+        return CHAR_THINKING_HTML
+
+
+def get_response(history, audio_input):
+
+    query_type = 'text'
+    question =history[-1][0]
+
+    global BUZZ_TIMEOUT
+    BUZZ_TIMEOUT = 80
+
+    if not question:
+        if audio_input:
+            query_type = 'audio'
+            os.rename(audio_input, audio_input + '.wav')
+            audio_file = open(audio_input + '.wav', "rb")
+            transcript = openai.Audio.transcribe("whisper-1", audio_file)
+            question = transcript['text']
+        else:
+            return None, None
+
+    LOGGER.info("\nquery_type: %s", query_type)
+    LOGGER.info("query_text: %s", question)
+    print('\nquery_type:', query_type)
+    print('\nquery_text:', question)
+
+    if question.lower().strip() == 'hi':
+        question = 'hello'
+    
+    answer = conv_model.run(question)
+    LOGGER.info("\ndocument_response: %s", answer)
+    print('\ndocument_response:', answer)
+
+    for trigger in GENERAL_RSPONSE_TRIGGERS:
+        if trigger in answer:    
+            MESSAGES.append({"role": "user", "content": question})
+            chat = openai.ChatCompletion.create(
+                    model="gpt-3.5-turbo", 
+                    messages=MESSAGES,
+                    temperature=0.7,
+                    n=128,
+                    stop="\n"
+                    )
+            answer = chat.choices[0].message.content
+            MESSAGES.append({"role": "assistant", "content": answer})
+            LOGGER.info("general_response: %s", answer)
+            print('\ngeneral_response:', answer)
+
+    AUDIO_HTML = text_to_speech_gen(answer)
+    history[-1][1] = answer
+
+    return history, AUDIO_HTML
+
+# buzz_usr_proc = Thread(target=idle_timer)
+
+with gr.Blocks(title="Your Assistance Pal!") as demo: 
+
+    with gr.Row():
+        output_html = gr.HTML(label="Felix's Voice", value=AUDIO_HTML)
+        output_html.visible = False
+        assistant_character = gr.HTML(label=None, value=CHAR_IDLE_HTML, show_label=False)
+        with gr.Column(scale=0.1):
+            chatbot = gr.Chatbot(label='Send a text or a voice input').style(height=285)
+            with gr.Row():
+                msg = gr.Textbox(placeholder='Write a chat & press Enter.', show_label=False).style(container=False)
+                with gr.Column(scale=0.5):
+                    audio_input = gr.Audio(source="microphone", type='filepath', show_label=False).style(container=False)
+                    button = gr.Button(value="Send")
+
+    msg.submit(get_chat_history, [msg, chatbot], [msg, chatbot]
+                ).then(update_img, outputs=[assistant_character]
+                ).then(get_response, [chatbot, audio_input], [chatbot, output_html]
+                ).then(update_img, outputs=[assistant_character])
+
+    button.click(get_chat_history, [msg, chatbot], [msg, chatbot]
+                ).then(update_img, outputs=[assistant_character]
+                ).then(get_response, [chatbot, audio_input], [chatbot, output_html]
+                ).then(update_img, outputs=[assistant_character])
+    
+    # buzz_usr_proc.start()
+    
+demo.launch(debug=False, favicon_path='assets/favicon.png', show_api=False, share=False)

app_utils.py

@@ -0,0 +1,90 @@
+import os
+import whisper
+from io import BytesIO
+import base64
+import boto3
+from pydub import AudioSegment
+from pydub.playback import play
+import logging
+
+from langchain import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.vectorstores import Chroma
+from langchain.document_loaders import DirectoryLoader
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+AWS_REGION_NAME = 'ap-south-1'
+
+
+logging.basicConfig(level="INFO",
+                    filename='conversations.log',
+                    filemode='a',
+                    format='%(asctime)s %(message)s',
+                    datefmt='%H:%M:%S')
+
+
+def buzz_user():
+    input_prompt = AudioSegment.from_mp3('assets/timeout_audio.mp3')
+    play(input_prompt)
+    
+
+def initialize_knowledge_base():
+    
+    loader = DirectoryLoader('profiles', glob='**/*.txt')
+    docs = loader.load()
+
+    char_text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    doc_texts = char_text_splitter.split_documents(docs)
+
+    openAI_embeddings = OpenAIEmbeddings()
+    vStore = Chroma.from_documents(doc_texts, openAI_embeddings)
+
+    conv_model = RetrievalQA.from_chain_type(
+        llm=OpenAI(), 
+        chain_type="stuff", 
+        retriever=vStore.as_retriever(
+            search_kwargs={"k": 1}
+            )
+        )
+    voice_model = whisper.load_model("tiny")
+
+    return conv_model, voice_model
+
+
+def text_to_speech_gen(answer):
+
+    polly = boto3.client('polly',
+                aws_access_key_id=AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+                region_name=AWS_REGION_NAME)
+
+    response = polly.synthesize_speech(
+        Text=answer, 
+        VoiceId='Matthew', 
+        OutputFormat='mp3', 
+        Engine = "neural")
+    
+    audio_stream = response['AudioStream'].read()
+    audio_html = audio_to_html(audio_stream)
+
+    return audio_html
+    
+
+def audio_to_html(audio_bytes):
+    audio_io = BytesIO(audio_bytes)
+    audio_io.seek(0)
+    audio_base64 = base64.b64encode(audio_io.read()).decode("utf-8")
+    audio_html = f'<audio src="data:audio/mpeg;base64,{audio_base64}" controls autoplay></audio>'
+
+    return audio_html
+
+
+def get_chat_history(user_message, history):
+    return "", history + [[user_message, None]]
+
+

profiles/john.txt

@@ -0,0 +1,11 @@
+Full name is Alfred Brandon
+you were born in 12th March 1958
+John is at the shop
+Brian is on holiday
+Sarah is coming home soon
+Sarah is net door and has left
+You have granddaughters, jennny and michelle, they are 7 and 10
+Your cousin Arthur is in Australia, he’s back next July
+Your neighbuor Steven passed away a few days ago
+Your wife hospital, broke arm
+John will be back by 8

requirements.txt

@@ -0,0 +1,147 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+argilla==1.6.0
+async-timeout==4.0.2
+attrs==23.1.0
+backoff==2.2.1
+boto3==1.26.115
+botocore==1.29.115
+certifi==2022.12.7
+charset-normalizer==3.1.0
+chromadb==0.3.21
+click==8.1.3
+clickhouse-connect==0.5.20
+cmake==3.26.3
+commonmark==0.9.1
+contourpy==1.0.7
+cycler==0.11.0
+dataclasses-json==0.5.7
+Deprecated==1.2.13
+duckdb==0.7.1
+entrypoints==0.4
+et-xmlfile==1.1.0
+fastapi==0.95.1
+ffmpeg-python==0.2.0
+ffmpy==0.3.0
+filelock==3.12.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+future==0.18.3
+gradio==3.27.0
+gradio_client==0.1.3
+greenlet==2.0.2
+h11==0.14.0
+hnswlib==0.7.0
+httpcore==0.16.3
+httptools==0.5.0
+httpx==0.23.3
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.2.0
+jsonschema==4.17.3
+kiwisolver==1.4.4
+langchain==0.0.144
+linkify-it-py==2.0.0
+lit==16.0.1
+llvmlite==0.39.1
+lxml==4.9.2
+lz4==4.3.2
+Markdown==3.4.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+monotonic==1.6
+more-itertools==9.1.0
+mpmath==1.3.0
+msg-parser==1.2.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.1
+nltk==3.8.1
+numba==0.56.4
+numexpr==2.8.4
+numpy==1.23.5
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+olefile==0.46
+openai==0.27.4
+openai-whisper @ git+https://github.com/openai/whisper.git@c09a7ae299c4c34c5839a76380ae407e7d785914
+openapi-schema-pydantic==1.2.4
+openpyxl==3.1.2
+orjson==3.8.10
+packaging==23.1
+pandas==1.5.3
+Pillow==9.5.0
+posthog==3.0.0
+pydantic==1.10.7
+pydub==0.25.1
+Pygments==2.15.1
+pypandoc==1.11
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==1.0.0
+python-magic==0.4.27
+python-multipart==0.0.6
+python-pptx==0.6.21
+pytz==2023.3
+PyYAML==6.0
+regex==2023.3.23
+requests==2.28.2
+rfc3986==1.5.0
+rich==13.0.1
+s3transfer==0.6.0
+scikit-learn==1.2.2
+scipy==1.10.1
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.98
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==1.4.47
+starlette==0.26.1
+sympy==1.11.1
+tenacity==8.2.2
+threadpoolctl==3.1.0
+tiktoken==0.3.1
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.0
+torchvision==0.15.1
+tqdm==4.65.0
+transformers==4.28.1
+triton==2.0.0
+typing-inspect==0.8.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.1
+unstructured==0.5.13
+urllib3==1.26.15
+uvicorn==0.21.1
+uvloop==0.17.0
+watchfiles==0.19.0
+websockets==11.0.2
+wrapt==1.14.1
+XlsxWriter==3.1.0
+yarl==1.8.2
+zstandard==0.21.0