progress

app/components/github-contribution-summary.tsx

@@ -1,5 +1,7 @@
 import { useNavigation, useNavigate } from '@remix-run/react'
 import { useEffect, useState } from 'react'
+import Markdown from 'react-markdown'
+
 import { useBufferedEventSource } from '~/utils/use-buffered-event-source.ts'
 import { type StreamData } from '~/utils/types.tsx'
 
@@ -111,10 +113,10 @@ function GithubContributionSummary({ userName, timePeriod }: Props) {
 							target="_blank"
 							rel="noopener noreferrer"
 						>
-							{pr.title}
+							<Markdown>{pr.title}</Markdown>
 						</a>
 					</div>
-					<div>{pr.summary}</div>
+					<Markdown>{pr.summary}</Markdown>
 					<br />
 				</div>
 			))

app/utils/ai.ts

@@ -1,8 +1,39 @@
 import { createSimpleCompletion } from './chatGPT.ts'
 import type { PullRequest } from './github.ts'
+import { getCache, setCache } from './redis.ts'
 
 const MAX_DIFF_LENGTH = 1000
 
+function getMetadataAction(pr: ReturnType<typeof getPrContentData>) {
+	return {
+		action: 'metadata',
+		data: {
+			title: pr.title,
+			link: pr.link,
+			id: pr.id,
+			closedAt: pr.closedAt,
+		},
+	} as const
+}
+
+function generateSummaryAction(summary: string, id: number) {
+	return {
+		action: 'summary',
+		data: { text: summary, id },
+	} as const
+}
+
+function getPrContentData(pr: PullRequest) {
+	return {
+		title: pr.title,
+		body: pr.body,
+		link: pr.html_url,
+		id: pr.id,
+		diffUrl: pr?.pull_request?.diff_url,
+		closedAt: pr.closed_at as string, // we've already filtered out PRs that are open
+	}
+}
+
 export async function generateSummaryForPrs({
 	name,
 	prs,
@@ -14,45 +45,56 @@ export async function generateSummaryForPrs({
 	customPrompt?: string
 	userId: number
 }) {
-	const prDataArray = await Promise.all(
-		prs.map(async pr => {
-			// Add metadata related to the PR
-			const prContent = {
-				title: pr.title,
-				body: pr.body,
-				link: pr.html_url,
-				id: pr.id,
-				diffUrl: pr?.pull_request?.diff_url,
-				closedAt: pr.closed_at as string, // we've already filtered out PRs that are open
+	let prDataArray = await Promise.all(prs.map(getPrContentData))
+
+	// load all summaries in the cache in parallel
+	const cachedPRSummaries = await Promise.all(
+		prDataArray.map(async pr => {
+			const cached = await getCache(pr.id.toString())
+			if (cached) {
+				return { summary: cached, ...pr }
 			}
-			return prContent
+			return { summary: '', ...pr }
 		}),
 	)
 
+	const prsWithCachedSummaries = []
+	// iterate through the cached responses that exist and yield them
+	for (const cached of cachedPRSummaries) {
+		// we only want to immediately pre-populate consecutive summaries from the beginning
+		// if we don't, then flickers will happen
+		if (!cached.summary) {
+			break
+		}
+		if (cached.summary) {
+			// remove the item so we don't yield it again
+			prsWithCachedSummaries.push(cached)
+			prDataArray = prDataArray.filter(prItem => prItem.id !== cached.id)
+		}
+	}
+
+	// first do all the cached summaries, then the uncached ones
 	return Promise.all(
-		prDataArray.map(async function* (pr) {
-			// load the diff if it's avaialable on-demand
-			let diff = ''
-			if (pr.diffUrl) {
-				const response = await fetch(pr.diffUrl)
-				const diffText = await response.text()
-				diff = diffText.substring(0, MAX_DIFF_LENGTH)
-			}
+		prsWithCachedSummaries
+			.map(async function* (pr) {
+				// yield the metadata then the summary
+				yield getMetadataAction(pr)
+				yield generateSummaryAction(pr.summary, pr.id)
+			})
+			.concat(
+				prDataArray.map(async function* (pr) {
+					// load the diff if it's avaialable on-demand
+					let diff = ''
+					if (pr.diffUrl) {
+						const response = await fetch(pr.diffUrl)
+						const diffText = await response.text()
+						diff = diffText.substring(0, MAX_DIFF_LENGTH)
+					}
 
-			// TODO: add comment data
+					// TODO: add comment data
 
-			const prMetadata = {
-				action: 'metadata',
-				data: {
-					title: pr.title,
-					link: pr.link,
-					id: pr.id,
-					closedAt: pr.closedAt,
-				},
-			} as const
-			yield prMetadata
-			// Construct the prompt for OpenAI
-			const prompt = `
+					// Construct the prompt for OpenAI
+					const prompt = `
 				Create a summary of this PR based on the JSON representation of the PR below.
 				The summary should be 2-3 sentences.
 				${customPrompt || ''}: 
@@ -66,21 +108,31 @@ export async function generateSummaryForPrs({
 					body: pr.body,
 					diff: diff,
 				})}`
-			const generator = createSimpleCompletion(prompt, userId)
-
-			// Generate the summary using OpenAI
-			while (true) {
-				const newItem = await generator.next()
-				console.log('newItem', newItem)
-				if (newItem.done) {
-					return
-				}
-				const message = {
-					action: 'summary',
-					data: { text: newItem.value, id: pr.id },
-				} as const
-				yield message
-			}
-		}),
+					const generator = createSimpleCompletion(prompt, userId)
+
+					// Generate the summary using OpenAI
+					let summary = ''
+
+
+					let first = true
+					while (true) {
+						const newItem = await generator.next()
+						if (newItem.done) {
+							// cache the summary
+							await setCache(pr.id.toString(), summary)
+							return
+						}
+						summary += newItem.value
+
+						if (first) {
+							// yield the metadata right before the first stream of data
+							yield getMetadataAction(pr)
+							first = false
+						}
+
+						yield generateSummaryAction(newItem.value, pr.id)
+					}
+				}),
+			),
 	)
 }

app/utils/chatGPT.ts

@@ -26,31 +26,16 @@ export async function* createSimpleCompletionNoCache(prompt: string) {
 	}
 }
 
+/**
+ *
+ */
 export async function* createSimpleCompletion(prompt: string, userId: number) {
-	// If the result is cached, we don't have to rate limit
-	let useCache: boolean = false
-	try {
-		const cached = await getCache(prompt)
-		useCache = true
-		if (cached) {
-			yield cached
-			return
-		}
-	} catch (err) {
-		console.error('Could not get cache', err)
-		useCache = false
-	}
-
 	// Increment and check rate limit
 	const currentTimestamp = Date.now()
 	const startOfDay = new Date().setHours(0, 0, 0, 0)
 	const rateLimitKey = `llm_user_rate_limit:${userId}`
 
-	// if cache doesn't work, just let it through
-	const rateLimitCountJson = useCache
-		? await getCache(rateLimitKey)
-		: JSON.stringify({ timestamp: 0, count: 0 })
-	console.log('rateLimitCountJson', rateLimitCountJson, rateLimitKey)
+	const rateLimitCountJson = await getCache(rateLimitKey)
 
 	// parse the rate limit count from cache
 	let rateLimitCount: RateLimitCount = { timestamp: 0, count: 0 }
@@ -72,15 +57,11 @@ export async function* createSimpleCompletion(prompt: string, userId: number) {
 
 	// now that we have the rate limit count, we can generate the prompt
 	const result = createSimpleCompletionNoCache(prompt)
-	const output = []
 	for await (const message of result) {
-		output.push(message)
 		yield message
 	}
 
 	try {
-		await setCache(prompt, output.join(''))
-
 		// Update rate limit count
 		if (rateLimitCount.timestamp < startOfDay) {
 			// Reset count for a new day