feat(example): Quivr whisper (#3495)

# Description

Talk with quivr handsfree via tts and stt.

## Checklist before requesting a review

Please delete options that are not relevant.

- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my code
- [ ] I have commented hard-to-understand areas
- [ ] I have ideally added tests that prove my fix is effective or that
my feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged

## Screenshots (if appropriate):

![image](https://github.com/user-attachments/assets/1c169e80-45ce-4541-b244-5f3b85b866f2)

.vscode/settings.json

@@ -44,5 +44,6 @@
     "reportUnusedImport": "warning",
     "reportGeneralTypeIssues": "warning"
   },
-  "makefile.configureOnOpen": false
+  "makefile.configureOnOpen": false,
+  "djlint.showInstallError": false
 }

examples/quivr-whisper/.gitignore

@@ -1 +1,2 @@
 .env
+uploads

examples/quivr-whisper/app.py

@@ -1,40 +1,116 @@
-from flask import Flask, render_template, request, jsonify
+from flask import Flask, render_template, request, jsonify, session
 import openai
 import base64
 import os
 import requests
 from dotenv import load_dotenv
+from quivr_core import Brain
+from quivr_core.rag.entities.config import RetrievalConfig
 from tempfile import NamedTemporaryFile
+from werkzeug.utils import secure_filename
+from asyncio import to_thread
+import asyncio
+
+
+UPLOAD_FOLDER = "uploads"
+ALLOWED_EXTENSIONS = {"txt"}
+
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
 
 app = Flask(__name__)
+app.secret_key = "secret"
+app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
+app.config["CACHE_TYPE"] = "SimpleCache"  # In-memory cache for development
+app.config["CACHE_DEFAULT_TIMEOUT"] = 60 * 60  # 1 hour cache timeout
 load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
 
-quivr_token = os.getenv("QUIVR_API_KEY", "")
-quivr_chat_id = os.getenv("QUIVR_CHAT_ID", "")
-quivr_brain_id = os.getenv("QUIVR_BRAIN_ID", "")
-quivr_url = (
-    os.getenv("QUIVR_URL", "https://api.quivr.app")
-    + f"/chat/{quivr_chat_id}/question?brain_id={quivr_brain_id}"
-)
+openai.api_key = os.getenv("OPENAI_API_KEY")
 
-headers = {
-    "Content-Type": "application/json",
-    "Authorization": f"Bearer {quivr_token}",
-}
+brains = {}
 
 
 @app.route("/")
 def index():
     return render_template("index.html")
 
 
-@app.route("/transcribe", methods=["POST"])
-def transcribe_audio():
+def run_in_event_loop(func, *args, **kwargs):
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    if asyncio.iscoroutinefunction(func):
+        result = loop.run_until_complete(func(*args, **kwargs))
+    else:
+        result = func(*args, **kwargs)
+    loop.close()
+    return result
+
+
+def allowed_file(filename):
+    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
+
+
+@app.route("/upload", methods=["POST"])
+async def upload_file():
+    if "file" not in request.files:
+        return "No file part", 400
+
+    file = request.files["file"]
+
+    if file.filename == "":
+        return "No selected file", 400
+    if not (file and file.filename and allowed_file(file.filename)):
+        return "Invalid file type", 400
+
+    filename = secure_filename(file.filename)
+    filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
+    file.save(filepath)
+
+    print(f"File uploaded and saved at: {filepath}")
+
+    print("Creating brain instance...")
+
+    brain: Brain = await to_thread(
+        run_in_event_loop, Brain.from_files, name="user_brain", file_paths=[filepath]
+    )
+
+    # Store brain instance in cache
+    session_id = session.sid if hasattr(session, "sid") else os.urandom(16).hex()
+    session["session_id"] = session_id
+    # cache.set(session_id, brain)  # Store the brain instance in the cache
+    brains[session_id] = brain
+    print(f"Brain instance created and stored in cache for session ID: {session_id}")
+
+    return jsonify({"message": "Brain created successfully"})
+
+
+@app.route("/ask", methods=["POST"])
+async def ask():
+    if "audio_data" not in request.files:
+        return "Missing audio data", 400
+
+    # Retrieve the brain instance from the cache using the session ID
+    session_id = session.get("session_id")
+    if not session_id:
+        return "Session ID not found. Upload a file first.", 400
+
+    brain = brains.get(session_id)
+    if not brain:
+        return "Brain instance not found in dict. Upload a file first.", 400
+
+    print("Brain instance loaded from cache.")
+
+    print("Speech to text...")
     audio_file = request.files["audio_data"]
     transcript = transcribe_audio_file(audio_file)
-    quivr_response = ask_quivr_question(transcript)
-    audio_base64 = synthesize_speech(quivr_response)
+    print("Transcript result: ", transcript)
+
+    print("Getting response...")
+    quivr_response = await to_thread(run_in_event_loop, brain.ask, transcript)
+
+    print("Text to speech...")
+    audio_base64 = synthesize_speech(quivr_response.answer)
+
+    print("Done")
     return jsonify({"audio_base64": audio_base64})
 
 
@@ -55,16 +131,6 @@ def transcribe_audio_file(audio_file):
     return transcript
 
 
-def ask_quivr_question(transcript):
-    response = requests.post(quivr_url, headers=headers, json={"question": transcript})
-    if response.status_code == 200:
-        quivr_response = response.json().get("assistant")
-        return quivr_response
-    else:
-        print(f"Error from Quivr API: {response.status_code}, {response.text}")
-        return "Sorry, I couldn't understand that."
-
-
 def synthesize_speech(text):
     speech_response = openai.audio.speech.create(
         model="tts-1", voice="nova", input=text

examples/quivr-whisper/pyproject.toml

@@ -6,9 +6,10 @@ authors = [
     { name = "Stan Girard", email = "[email protected]" }
 ]
 dependencies = [
-    "flask>=3.1.0",
+    "flask[async]>=3.1.0",
     "openai>=1.54.5",
     "quivr-core>=0.0.24",
+    "flask-caching>=2.3.0",
 ]
 readme = "README.md"
 requires-python = ">= 3.11"

examples/quivr-whisper/requirements-dev.lock

@@ -32,6 +32,8 @@ anyio==4.6.2.post1
     # via httpx
     # via openai
     # via starlette
+asgiref==3.8.1
+    # via flask
 attrs==24.2.0
     # via aiohttp
 backoff==2.2.1
@@ -42,6 +44,8 @@ beautifulsoup4==4.12.3
     # via unstructured
 blinker==1.9.0
     # via flask
+cachelib==0.9.0
+    # via flask-caching
 cachetools==5.5.0
     # via google-auth
 certifi==2024.8.30
@@ -112,6 +116,9 @@ filetype==1.2.0
     # via llama-index-core
     # via unstructured
 flask==3.1.0
+    # via flask-caching
+    # via quivr-whisper
+flask-caching==2.3.0
     # via quivr-whisper
 flatbuffers==24.3.25
     # via onnxruntime

examples/quivr-whisper/requirements.lock

@@ -32,6 +32,8 @@ anyio==4.6.2.post1
     # via httpx
     # via openai
     # via starlette
+asgiref==3.8.1
+    # via flask
 attrs==24.2.0
     # via aiohttp
 backoff==2.2.1
@@ -42,6 +44,8 @@ beautifulsoup4==4.12.3
     # via unstructured
 blinker==1.9.0
     # via flask
+cachelib==0.9.0
+    # via flask-caching
 cachetools==5.5.0
     # via google-auth
 certifi==2024.8.30
@@ -112,6 +116,9 @@ filetype==1.2.0
     # via llama-index-core
     # via unstructured
 flask==3.1.0
+    # via flask-caching
+    # via quivr-whisper
+flask-caching==2.3.0
     # via quivr-whisper
 flatbuffers==24.3.25
     # via onnxruntime