-
Notifications
You must be signed in to change notification settings - Fork 959
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Multimodal block display + Steamship agent (#49)
* Contribute initial multimodal agent support & Steamship agent
- Loading branch information
Showing
12 changed files
with
239 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| import dotenv from "dotenv"; | ||
| import clerk from "@clerk/clerk-sdk-node"; | ||
| import { NextResponse } from "next/server"; | ||
| import { currentUser } from "@clerk/nextjs"; | ||
| import { rateLimit } from "@/app/utils/rateLimit"; | ||
| import {Md5} from 'ts-md5' | ||
| import ConfigManager from "@/app/utils/config"; | ||
|
|
||
| dotenv.config({ path: `.env.local` }); | ||
|
|
||
| function returnError(code: number, message: string) { | ||
| return new NextResponse( | ||
| JSON.stringify({ Message: message }), | ||
| { | ||
| status: code, | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| }, | ||
| } | ||
| ); | ||
| } | ||
|
|
||
| export async function POST(req: Request) { | ||
| let clerkUserId; | ||
| let user; | ||
| let clerkUserName; | ||
| const { prompt, isText, userId, userName } = await req.json(); | ||
| const companionName = req.headers.get("name"); | ||
|
|
||
| // Load the companion config | ||
| const configManager = ConfigManager.getInstance(); | ||
| const companionConfig = configManager.getConfig("name", companionName); | ||
| if (!companionConfig) { | ||
| return returnError(404, `Hi, we were unable to find the configuration for a companion named ${companionName}.`) | ||
| } | ||
|
|
||
| // Make sure we're not rate limited | ||
| const identifier = req.url + "-" + (userId || "anonymous"); | ||
| const { success } = await rateLimit(identifier); | ||
| if (!success) { | ||
| console.log("INFO: rate limit exceeded"); | ||
| return returnError(429, `Hi, the companions can't talk this fast.`) | ||
| } | ||
|
|
||
| if (!process.env.STEAMSHIP_API_KEY) { | ||
| return returnError(500, `Please set the STEAMSHIP_API_KEY env variable and make sure ${companionName} is connected to an Agent instance that you own.`) | ||
| } | ||
|
|
||
| console.log(`Companion Name: ${companionName}`) | ||
| console.log(`Prompt: ${prompt}`); | ||
|
|
||
| if (isText) { | ||
| clerkUserId = userId; | ||
| clerkUserName = userName; | ||
| } else { | ||
| user = await currentUser(); | ||
| clerkUserId = user?.id; | ||
| clerkUserName = user?.firstName; | ||
| } | ||
|
|
||
| if (!clerkUserId || !!!(await clerk.users.getUser(clerkUserId))) { | ||
| console.log("user not authorized"); | ||
| return new NextResponse( | ||
| JSON.stringify({ Message: "User not authorized" }), | ||
| { | ||
| status: 401, | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| }, | ||
| } | ||
| ); | ||
| } | ||
|
|
||
| // Create a chat session id for the user | ||
| const chatSessionId = Md5.hashStr(userId || "anonymous"); | ||
|
|
||
| // Make sure we have a generate endpoint. | ||
| // TODO: Create a new instance of the agent per user if this proves advantageous. | ||
| const agentUrl = companionConfig.generateEndpoint | ||
| if (!agentUrl) { | ||
| return returnError(500, `Please add a Steamship 'generateEndpoint' to your ${companionName} configuration in companions.json.`) | ||
| } | ||
|
|
||
| // Invoke the generation. Tool invocation, chat history management, backstory injection, etc is all done within this endpoint. | ||
| // To build, deploy, and host your own multi-tenant agent see: https://www.steamship.com/learn/agent-guidebook | ||
| const response = await fetch(agentUrl, { | ||
| method: "POST", | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| "Authorization": `Bearer ${process.env.STEAMSHIP_API_KEY}` | ||
| }, | ||
| body: JSON.stringify({ | ||
| question: prompt, | ||
| chat_session_id: chatSessionId | ||
| }) | ||
| }); | ||
|
|
||
| if (response.ok) { | ||
| const responseText = await response.text() | ||
| const responseBlocks = JSON.parse(responseText) | ||
| return NextResponse.json(responseBlocks) | ||
| } else { | ||
| return returnError(500, await response.text()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * Represents a unit of multimodal chat: text, video, audio, or image. | ||
| * | ||
| * For streaming responses, just update the `text` argument. | ||
| */ | ||
| export function ChatBlock({text, mimeType, url} : { | ||
| text?: string, | ||
| mimeType?: string, | ||
| url?: string | ||
| }) { | ||
| let internalComponent = <></> | ||
| if (text) { | ||
| internalComponent = <span>{text}</span> | ||
| } else if (mimeType && url) { | ||
| if (mimeType.startsWith("audio")) { | ||
| internalComponent = <audio controls={true} src={url} /> | ||
| } else if (mimeType.startsWith("video")) { | ||
| internalComponent = <video controls width="250"> | ||
| <source src={url} type={mimeType} /> | ||
| Download the <a href={url}>video</a> | ||
| </video> | ||
| } else if (mimeType.startsWith("image")) { | ||
| internalComponent = <img src={url} /> | ||
| } | ||
| } else if (url) { | ||
| internalComponent = <a href={url}>Link</a> | ||
| } | ||
|
|
||
| return ( | ||
| <p className="text-sm text-gray-200 pb-2"> | ||
| {internalComponent} | ||
| </p> | ||
| ); | ||
| } | ||
|
|
||
| /* | ||
| * Take a completion, which may be a string, JSON encoded as a string, or JSON object, | ||
| * and produce a list of ChatBlock objects. This is intended to be a one-size-fits-all | ||
| * method for funneling different LLM output into structure that supports different media | ||
| * types and can easily grow to support more metadata (such as speaker). | ||
| */ | ||
| export function responseToChatBlocks(completion: any) { | ||
| // First we try to parse completion as JSON in case we're dealing with an object. | ||
| console.log("got completoin", completion, typeof completion) | ||
| if (typeof completion == "string") { | ||
| try { | ||
| completion = JSON.parse(completion) | ||
| } catch { | ||
| // Do nothing; we'll just treat it as a string. | ||
| console.log("Couldn't parse") | ||
| } | ||
| } | ||
| let blocks = [] | ||
| if (typeof completion == "string") { | ||
| console.log("still string") | ||
| blocks.push(<ChatBlock text={completion} />) | ||
| } else if (Array.isArray(completion)) { | ||
| console.log("Is array") | ||
| for (let block of completion) { | ||
| console.log(block) | ||
| blocks.push(<ChatBlock {...block} />) | ||
| } | ||
| } else { | ||
| blocks.push(<ChatBlock {...completion} />) | ||
| } | ||
| console.log(blocks) | ||
| return blocks | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters