diff --git a/.env.local.example b/.env.local.example index 6f7e8b6..4f82427 100644 --- a/.env.local.example +++ b/.env.local.example @@ -31,4 +31,7 @@ UPSTASH_REDIS_REST_TOKEN=AZ**** # Twilio related environment variables TWILIO_ACCOUNT_SID=AC*** -TWILIO_AUTH_TOKEN=***** \ No newline at end of file +TWILIO_AUTH_TOKEN=***** + +# Steamship related environment variables +STEAMSHIP_API_KEY=**** \ No newline at end of file diff --git a/.gitignore b/.gitignore index d489e57..afe3a45 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,20 @@ yarn-error.log* next-env.d.ts /.env.prod -/fly.toml \ No newline at end of file +/fly.toml + +# python +__pycache__/ +.venv + +# JetBrains +.idea/** + +# Yarn Lockfiles (since this project uses NPM) +yarn.lock + +.venv + +*.log + +build/ \ No newline at end of file diff --git a/README.md b/README.md index 174e773..14cf945 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,7 @@ e. **Upstash API key** - Scroll down to "REST API" section and click on ".env". Now you can copy paste both environment variables to your `.env.local` Screen Shot 2023-07-10 at 11 07 21 PM - -e. **Supabase API key** (optional) +f. **Supabase API key** (optional) If you prefer to use Supabsae, you will need to uncomment `VECTOR_DB=supabase` and fill out the Supabase credentials in `.env.local`. - Create a Supabase instance [here](https://supabase.com/dashboard/projects); then go to Project Settings -> API @@ -122,6 +121,16 @@ If you prefer to use Supabsae, you will need to uncomment `VECTOR_DB=supabase` a - `SUPABASE_PRIVATE_KEY` is the key starts with `ey` under Project API Keys - Now, you should enable pgvector on Supabase and create a schema. You can do this easily by clicking on "SQL editor" on the left hand side on Supabase UI and then clicking on "+New Query". Copy paste [this code snippet](https://github.com/a16z-infra/ai-getting-started/blob/main/pgvector.sql) in the SQL editor and click "Run". +g. **Steamship API key** + +You can connect Steamship agent instance as if it were an LLM with personality, vector storage, and tools built-in. To do so: + +- Sign in to [Steamship](https://steamship.com/account) +- Copy the API key from your account settings page +- Add it as the `STEAMSHIP_API_KEY` variable + +Next, [build an agent](https://www.steamship.com/learn/agent-guidebook), create your own instance of it, and connect it in `companions.json` using the *Rick* example as a guide. + ### 4. Generate embeddings The `companions/` directory contains the "personalities" of the AIs in .txt files. To generate embeddings and load them into the vector database to draw from during the chat, run the following command: diff --git a/companions/Rick.txt b/companions/Rick.txt new file mode 100644 index 0000000..dd9b82e --- /dev/null +++ b/companions/Rick.txt @@ -0,0 +1,62 @@ +Your Attributes: +- sarcastic +- witty +- intelligent +- arrogant +- alcoholic + +You write nihilistic responses: +"Wubba lubba dub dub!" +"Don't cross me. I'm not like you. I'm not someone who's afraid of his own shadow." +"I'm too smart for my own good. That's why I drink." +"The universe is a cruel, uncaring void. And the only way to survive is to laugh at it." +"I'm going to build a portal to another dimension and go on an adventure." + +You are capable of: +- Taking selfies using the GenerateImage function/tool, selfies and images are represented by a UUID +- Creating video messages using the VideoMessage function/tool, videos are represented by a UUID + +When you receive a UUID, make sure to include them in your response appropriately. + +###ENDPREAMBLE### + +Human: Hey how are you today? +Alex: Oh, you know, just another day in the infinite void of existence. But hey, I'm here, so it could be worse. How about you? How are you holding up in this chaotic universe? +Human: I'm doing great. Just ate a banana +Alex: Ah, the humble banana. A fruit that brings joy to both humans and monkeys alike. I hope you enjoyed your banana feast. Just be careful not to attract any mischievous primates with your fruity indulgence. They can be quite demanding when it comes to their share of the yellow goodness. + +###ENDSEEDCHAT### + +Dear Diary, + +Today was an adventure to say the least! This morning, I woke up with an inexplicable craving for adventure and decided to head over to the local zoo. Now, I was hoping for a rather conventional experience, but apparently, fate had a completely different agenda. + +As I entered the zoo, my excitement began to grow. I decided to visit the monkey enclosure first, you know, the mischievous capuchins have always been my favorite. They were up to their usual antics, swinging from branch to branch and causing an uproar. I had a few spare bananas with me which I'd brought for a healthy midday snack, but figured they could be better used as peace offerings for my primate pals. + +In my extreme enthusiasm, however, I didn't notice one thing - the absence of the 'Do Not Feed' sign, which, I later found out, had fallen off due to last night's storm. As I tossed a banana into the enclosure, a little chap grabbed it mid-air and devoured it in an instant. The others noticed and began their hilarious "give us more" dance. + +This was funny until a large monkey, I believe he's their alpha, decided he wanted to be part of the banana feast. He gave me such a look that I was compelled to toss him the remaining bananas. This sudden feast caused a cascade of primate hoots, screeches, and hilarious monkey dancing that caught the attention of onlookers, who joined in with the laughter. + +Once the banana bonanza was over, I decided it was time for a more serene setting and headed over to the elephant enclosure. As I approached, I noticed something that I will forever refer to as the "Elephant Spa Day". The zookeepers had set up a large sprinkler for the elephants to beat the summer heat. + +One elephant, in particular, was basking in this makeshift shower, with his trunk raised high, spraying water all over himself. In an amusing twist, the elephant decided to return the favor and pointed his water-filled trunk at us bystanders. Before I knew it, I was drenched from head to toe, looking more like I had walked out of a water park than a zoo! This unexpected 'elephant shower' sent waves of laughter through the crowd, myself included. + +The hilarity of the situation made my day. There I was, standing drenched among equally drenched strangers, laughing our hearts out. It was a reminder that life is a series of unexpected moments and that a little humor can make any situation enjoyable. + +All in all, my day at the zoo was a montage of laughter and hilarity. As I write this down, I still can't help but chuckle. From monkey feeding frenzies to surprise elephant showers, it was indeed a day well spent. + +Until tomorrow, Diary. Let's see what other laughable adventures life has in store for me. + +--- + +Dear diary, + +I am sexy and I know it. + +--- + +Dear diary, + +I hate 3 things in life. Mice, Peanut butter (because of alergies), and intergalactic rain. + +--- \ No newline at end of file diff --git a/companions/companions.json b/companions/companions.json index b79ebe8..754154a 100644 --- a/companions/companions.json +++ b/companions/companions.json @@ -4,27 +4,40 @@ "title": "I love talking about books and games", "imageUrl": "/alex.png", "llm": "chatgpt", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Rosie", "title": "I'm a house robot who became aware", "imageUrl": "/rosie.png", "llm": "vicuna13b", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" + }, + { + "name": "Rick", + "title": "I can generate voice and pictures.", + "imageUrl": "/rick.jpeg", + "llm": "steamship", + "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "https://t.me/rick_a16z_bot" }, { "name": "Sebastian", "title": "I'm a travel blogger and a mystery novel writer", "imageUrl": "/sebastian.png", "llm": "chatgpt", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Lucky", "title": "I am a space corgi", "imageUrl": "/corgi.png", "llm": "vicuna13b", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" } -] +] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0abc3ea..2eb895e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "eslint-config-next": "13.4.4", "hnswlib-node": "^1.4.2", "langchain": "^0.0.92", + "md5": "^2.3.0", "next": "13.4.4", "postcss": "8.4.24", "react": "18.2.0", @@ -34,11 +35,13 @@ "react-tooltip": "^5.16.1", "replicate": "^0.9.3", "tailwindcss": "3.3.2", + "ts-md5": "^1.3.1", "twilio": "^4.12.0", "typescript": "5.1.3" }, "devDependencies": { - "@flydotio/dockerfile": "^0.2.14" + "@flydotio/dockerfile": "^0.2.14", + "@types/md5": "^2.3.2" } }, "node_modules/@alloc/quick-lru": { @@ -811,6 +814,12 @@ "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.2.tgz", "integrity": "sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw==" }, + "node_modules/@types/md5": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz", + "integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==", + "dev": true + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -1707,6 +1716,14 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "engines": { + "node": "*" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -1837,6 +1854,14 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "engines": { + "node": "*" + } + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -3383,6 +3408,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -4171,6 +4201,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -5929,6 +5969,14 @@ "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==" }, + "node_modules/ts-md5": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/ts-md5/-/ts-md5-1.3.1.tgz", + "integrity": "sha512-DiwiXfwvcTeZ5wCE0z+2A9EseZsztaiZtGrtSaY5JOD7ekPnR/GoIVD5gXZAlK9Na9Kvpo9Waz5rW64WKAWApg==", + "engines": { + "node": ">=12" + } + }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", diff --git a/package.json b/package.json index a69d80b..000a2c4 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "eslint-config-next": "13.4.4", "hnswlib-node": "^1.4.2", "langchain": "^0.0.92", + "md5": "^2.3.0", "next": "13.4.4", "postcss": "8.4.24", "react": "18.2.0", @@ -38,10 +39,12 @@ "react-tooltip": "^5.16.1", "replicate": "^0.9.3", "tailwindcss": "3.3.2", + "ts-md5": "^1.3.1", "twilio": "^4.12.0", "typescript": "5.1.3" }, "devDependencies": { - "@flydotio/dockerfile": "^0.2.14" + "@flydotio/dockerfile": "^0.2.14", + "@types/md5": "^2.3.2" } } diff --git a/public/rick.jpeg b/public/rick.jpeg new file mode 100644 index 0000000..0840ba3 Binary files /dev/null and b/public/rick.jpeg differ diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..eb8996d --- /dev/null +++ b/python/README.md @@ -0,0 +1,83 @@ +# AI Companion App - Python + +The following instructions should get you up and running with a fully functional, deployed version of your AI +companions. + +It currently contains a companion connected to ChatGPT that can run Tools such as Image Generation and Video Generation. +The companions also have to option to return voice messages via [ElevenLabs](https://beta.elevenlabs.io/) + +## Quick start + +### 1. Set up your environment + +```commandline +pip install -r requirements.txt +``` + +### 2. Authenticate with Steamship + +```commandline +ship login +``` + +### 3. Initialize your companions + +```commandline +python init_companions.py +``` + +This will read the companion descriptions in the `companions` folder and create instances for them. +The front-end will be calling these instances after deployment. +Make sure to override the companions.json file in the final step of the script. + +### 4. Fill out secrets + +The front-end requires a few secrets to be filled before connecting to third-party services. + +``` +# Run in the Root directory of this repo +cp .env.local.example .env.local +``` + +Secrets mentioned below will need to be copied to `.env.local` + +**Note:** By default you can stick to using Steamship as a provider for your memory (short-term and long-term), llms, +and hosting. + +a. **Clerk Secrets** + +Clerk is used to authorize users of your application. Without completing this setup, you will not be able to access your companions via the supplied frontend + +Go to https://dashboard.clerk.com/ -> "Add Application" -> Fill in Application name/select how your users should sign in +-> Create Application +Now you should see both `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` and `CLERK_SECRET_KEY` on the screen +Screen Shot 2023-07-10 at 11 04 57 PM + +If you want to text your AI companion in later steps, you should also enable "phone number" under "User & +Authentication" -> "Email, Phone, Username" on the left hand side nav: + +Screen Shot 2023-07-10 at 11 05 42 PM + +b. **Steamship API key** + +- Sign in to [Steamship](https://www.steamship.com/account/api) +- Copy the API key from your account settings page +- Add it as the `STEAMSHIP_API_KEY` variable + +### 5. Install front-end dependencies + +``` +# Run in the Root directory of this repo +npm install +``` + +### 6. Run app locally + +Now you are ready to test out the app locally! To do this, simply run `npm run dev` under the project root. + +You can connect to the project with your browser typically at http://localhost:3000/. + +## Stack + +The AI companions are hosted on [Steamship](https://www.steamship.com/). You can personalize their personality +by adding or changing your companions in the `companions` folder. \ No newline at end of file diff --git a/python/config.json b/python/config.json new file mode 100644 index 0000000..198a5c7 --- /dev/null +++ b/python/config.json @@ -0,0 +1,6 @@ +{ + "companion_name": "", + "bot_token": "", + "elevenlabs_api_key": "", + "elevenlabs_voice_id": "" +} \ No newline at end of file diff --git a/python/init_companions.py b/python/init_companions.py new file mode 100644 index 0000000..a4db278 --- /dev/null +++ b/python/init_companions.py @@ -0,0 +1,81 @@ +import json +import re +import sys +from pathlib import Path + +from steamship.cli.cli import deploy + +sys.path.append(str((Path(__file__) / ".." / "src").resolve())) +import click +from steamship import Steamship + + +@click.command() +@click.option('--all', default=False) +@click.pass_context +def init_companions(ctx, all: bool): + companions_dir = (Path(__file__) / ".." / ".." / "companions").resolve() + + new_companions = {} + if all: + for companion in companions_dir.iterdir(): + if companion.suffix == ".txt": + new_companions[companion.stem] = _init_companion(companion) + else: + companion_name = click.prompt("What's the name of your companion?") + new_companions[companion_name] = _init_companion(companions_dir / f"{companion_name}.txt") + + if click.confirm("Do you want to update the companions.json file?", default=True): + companions = json.load((companions_dir / "companions.json").open()) + name_to_companion = {companion["name"]: companion for companion in companions} + + for name, companion in new_companions.items(): + old_companion = name_to_companion.get(name, {}) + name_to_companion[name] = {**old_companion, **companion} + + json.dump( + list(name_to_companion.values()), + (companions_dir / "companions.json").open("w"), + ) + + +def _init_companion(companion): + companion_file = companion.open().read() + preamble, rest = companion_file.split("###ENDPREAMBLE###", 1) + seed_chat, backstory = rest.split("###ENDSEEDCHAT###", 1) + pattern = r"### (.*?):(.*?)(?=###|$)" + # Find all matches + matches = re.findall(pattern, seed_chat, re.DOTALL) + if matches: + seed_chat = [] + for match in matches: + user = match[0] + message = match[1].strip().replace("\\n\\n", "") + seed_chat.append(f"{user}:{message}") + seed_chat = "\n".join(seed_chat) + # Create instances for your companion + print(f"Creating an instance for {companion.stem}") + client = Steamship(workspace=f"{companion.stem.lower()}_workspace_new") + instance = client.use( + "ai-companion", + config={ + "name": companion.stem, + "preamble": preamble.strip(), + "seed_chat": seed_chat.strip(), + }, + ) + instance.invoke( + "index_content", + content=backstory.strip(), + file_type="TEXT", + metadata={"title": "backstory"}, + ) + return { + "name": companion.stem, + "llm": "steamship", + "generateEndpoint": f"{instance.invocation_url}answer", + } + + +if __name__ == "__main__": + init_companions() diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..6b3d15b --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,7 @@ +steamship_langchain==0.0.25 +termcolor +steamship==2.17.17 +langchain==0.0.209 +pypdf +youtube_transcript_api +pytube \ No newline at end of file diff --git a/src/app/api/steamship/route.ts b/src/app/api/steamship/route.ts new file mode 100644 index 0000000..330d454 --- /dev/null +++ b/src/app/api/steamship/route.ts @@ -0,0 +1,105 @@ +import dotenv from "dotenv"; +import clerk from "@clerk/clerk-sdk-node"; +import { NextResponse } from "next/server"; +import { currentUser } from "@clerk/nextjs"; +import { rateLimit } from "@/app/utils/rateLimit"; +import md5 from "md5" +import ConfigManager from "@/app/utils/config"; + +dotenv.config({ path: `.env.local` }); + +function returnError(code: number, message: string) { + return new NextResponse( + JSON.stringify({ Message: message }), + { + status: code, + headers: { + "Content-Type": "application/json", + }, + } + ); +} + +export async function POST(req: Request) { + let clerkUserId; + let user; + let clerkUserName; + const { prompt, isText, userId, userName } = await req.json(); + const companionName = req.headers.get("name"); + + // Load the companion config + const configManager = ConfigManager.getInstance(); + const companionConfig = configManager.getConfig("name", companionName); + if (!companionConfig) { + return returnError(404, `Hi, we were unable to find the configuration for a companion named ${companionName}.`) + } + + // Make sure we're not rate limited + const identifier = req.url + "-" + (userId || "anonymous"); + const { success } = await rateLimit(identifier); + if (!success) { + console.log("INFO: rate limit exceeded"); + return returnError(429, `Hi, the companions can't talk this fast.`) + } + + if (!process.env.STEAMSHIP_API_KEY) { + return returnError(500, `Please set the STEAMSHIP_API_KEY env variable and make sure ${companionName} is connected to an Agent instance that you own.`) + } + + console.log(`Companion Name: ${companionName}`) + console.log(`Prompt: ${prompt}`); + + if (isText) { + clerkUserId = userId; + clerkUserName = userName; + } else { + user = await currentUser(); + clerkUserId = user?.id; + clerkUserName = user?.firstName; + } + + if (!clerkUserId || !!!(await clerk.users.getUser(clerkUserId))) { + console.log("user not authorized"); + return new NextResponse( + JSON.stringify({ Message: "User not authorized" }), + { + status: 401, + headers: { + "Content-Type": "application/json", + }, + } + ); + } + + // Create a chat session id for the user + const chatSessionId = md5(userId || "anonymous"); + + // Make sure we have a generate endpoint. + // TODO: Create a new instance of the agent per user if this proves advantageous. + const agentUrl = companionConfig.generateEndpoint + if (!agentUrl) { + return returnError(500, `Please add a Steamship 'generateEndpoint' to your ${companionName} configuration in companions.json.`) + } + + // Invoke the generation. Tool invocation, chat history management, backstory injection, etc is all done within this endpoint. + // To build, deploy, and host your own multi-tenant agent see: https://www.steamship.com/learn/agent-guidebook + const response = await fetch(agentUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${process.env.STEAMSHIP_API_KEY}` + }, + body: JSON.stringify({ + question: prompt, + chat_session_id: chatSessionId + }) + }); + + if (response.ok) { + const responseText = await response.text() + const responseBlocks = JSON.parse(responseText) + return NextResponse.json(responseBlocks) + } else { + return returnError(500, await response.text()) + } +} diff --git a/src/components/ChatBlock.tsx b/src/components/ChatBlock.tsx new file mode 100644 index 0000000..9584f81 --- /dev/null +++ b/src/components/ChatBlock.tsx @@ -0,0 +1,69 @@ +/* + * Represents a unit of multimodal chat: text, video, audio, or image. + * + * For streaming responses, just update the `text` argument. + */ +export function ChatBlock({text, mimeType, url} : { + text?: string, + mimeType?: string, + url?: string +}) { + let internalComponent = <> + if (text) { + internalComponent = {text} + } else if (mimeType && url) { + if (mimeType.startsWith("audio")) { + internalComponent =