refactor: change the filter condition of the tool search_knowledge (#434

) * 文档入库和知识查询改为以repo_name为索引，防止不同机器人但是同个repo的重复入库造成的资源浪费
petercat-ai · Oct 15, 2024 · ab76db6 · ab76db6
2 parents e418497 + 8f3bbee
commit ab76db6
Show file tree

Hide file tree

Showing 15 changed files with 379 additions and 122 deletions.
diff --git a/client/app/factory/edit/components/Knowledge.tsx b/client/app/factory/edit/components/Knowledge.tsx
@@ -21,6 +21,7 @@ import { useBotTask } from './TaskContext';
 
 type IProps = {
   botId: string;
+  repoName: string;
   goBack: () => void;
 };
 const ChunkCard = ({ update_timestamp, content, file_path }: RAGDoc) => {
@@ -78,7 +79,7 @@ const ChunkList = ({ data }: { data: RAGDoc[] }) => {
   );
 };
 
-export default function Knowledge({ botId, goBack }: IProps) {
+export default function Knowledge({ botId, repoName, goBack }: IProps) {
   const { botProfile } = useBot();
   const [pageSize, setPageSize] = React.useState(12);
   const [pageNumber, setPageNumber] = React.useState(1);
@@ -88,7 +89,7 @@ export default function Knowledge({ botId, goBack }: IProps) {
     isPending,
     isLoading: isListLoading,
   } = useBotRAGChunkList(
-    botId,
+    repoName,
     pageSize,
     pageNumber,
     true,

diff --git a/client/app/factory/edit/page.tsx b/client/app/factory/edit/page.tsx
@@ -447,7 +447,8 @@ export default function Edit() {
                       }
                     }}
                   >
-                    {I18N.edit.page.baoCunYuBuShu}</Button>
+                    {I18N.edit.page.baoCunYuBuShu}
+                  </Button>
                 </div>
               </div>
               <div className="position absolute top-[73px] left-0 w-full">
@@ -489,6 +490,7 @@ export default function Edit() {
         {visibleType === VisibleTypeEnum.KNOWLEDGE_DETAIL ? (
           <Knowledge
             botId={botProfile.id}
+            repoName={botProfile.repoName!}
             goBack={() => {
               setVisibleType(VisibleTypeEnum.BOT_CONFIG);
             }}

diff --git a/client/app/hooks/useBot.ts b/client/app/hooks/useBot.ts
@@ -116,15 +116,15 @@ export function useBotConfigGenerator() {
 }
 
 export const useBotRAGChunkList = (
-  botId: string,
+  repoName: string,
   page_size: number,
   page_number: number,
   enabled: boolean = true,
   refetchInterval: boolean = false,
 ) => {
   return useQuery({
-    queryKey: [`rag.chunk.list`, page_number, botId],
-    queryFn: async () => getChunkList(botId, page_size, page_number),
+    queryKey: [`rag.chunk.list`, page_number, repoName],
+    queryFn: async () => getChunkList(repoName, page_size, page_number),
     select: (data) => data,
     enabled,
     retry: true,

diff --git a/client/app/services/BotsController.ts b/client/app/services/BotsController.ts
@@ -74,12 +74,12 @@ export async function getBotInfoByRepoName(
 }
 
 export async function getChunkList(
-  bot_id: string,
+  repo_name: string,
   page_size: number,
   page_number: number,
 ): Promise<{ rows: RAGDoc[]; total: number }> {
   const response = await axios.get(
-    `${apiDomain}/api/rag/chunk/list?bot_id=${bot_id}&page_size=${page_size}&page_number=${page_number}`,
+    `${apiDomain}/api/rag/chunk/list?repo_name=${repo_name}&page_size=${page_size}&page_number=${page_number}`,
   );
   return response.data;
 }

diff --git a/client/types/database.types.ts b/client/types/database.types.ts
@@ -4,7 +4,7 @@ export type Json =
   | boolean
   | null
   | { [key: string]: Json | undefined }
-  | Json[]
+  | Json[];
 
 export type Database = {
   graphql_public: {
@@ -536,6 +536,19 @@ export type Database = {
           similarity: number;
         }[];
       };
+      match_embedding_docs: {
+        Args: {
+          query_embedding: string;
+          filter?: Json;
+        };
+        Returns: {
+          id: string;
+          content: string;
+          metadata: Json;
+          embedding: string;
+          similarity: number;
+        }[];
+      };
       match_text: {
         Args: {
           query_embedding: string;
@@ -923,84 +936,84 @@ export type Database = {
   };
 };
 
-type PublicSchema = Database[Extract<keyof Database, "public">]
+type PublicSchema = Database[Extract<keyof Database, 'public'>];
 
 export type Tables<
   PublicTableNameOrOptions extends
-    | keyof (PublicSchema["Tables"] & PublicSchema["Views"])
+    | keyof (PublicSchema['Tables'] & PublicSchema['Views'])
     | { schema: keyof Database },
   TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
-    ? keyof (Database[PublicTableNameOrOptions["schema"]]["Tables"] &
-        Database[PublicTableNameOrOptions["schema"]]["Views"])
+    ? keyof (Database[PublicTableNameOrOptions['schema']]['Tables'] &
+        Database[PublicTableNameOrOptions['schema']]['Views'])
     : never = never,
 > = PublicTableNameOrOptions extends { schema: keyof Database }
-  ? (Database[PublicTableNameOrOptions["schema"]]["Tables"] &
-      Database[PublicTableNameOrOptions["schema"]]["Views"])[TableName] extends {
-      Row: infer R
+  ? (Database[PublicTableNameOrOptions['schema']]['Tables'] &
+      Database[PublicTableNameOrOptions['schema']]['Views'])[TableName] extends {
+      Row: infer R;
     }
     ? R
     : never
-  : PublicTableNameOrOptions extends keyof (PublicSchema["Tables"] &
-        PublicSchema["Views"])
-    ? (PublicSchema["Tables"] &
-        PublicSchema["Views"])[PublicTableNameOrOptions] extends {
-        Row: infer R
-      }
-      ? R
-      : never
+  : PublicTableNameOrOptions extends keyof (PublicSchema['Tables'] &
+      PublicSchema['Views'])
+  ? (PublicSchema['Tables'] &
+      PublicSchema['Views'])[PublicTableNameOrOptions] extends {
+      Row: infer R;
+    }
+    ? R
     : never
+  : never;
 
 export type TablesInsert<
   PublicTableNameOrOptions extends
-    | keyof PublicSchema["Tables"]
+    | keyof PublicSchema['Tables']
     | { schema: keyof Database },
   TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
-    ? keyof Database[PublicTableNameOrOptions["schema"]]["Tables"]
+    ? keyof Database[PublicTableNameOrOptions['schema']]['Tables']
     : never = never,
 > = PublicTableNameOrOptions extends { schema: keyof Database }
-  ? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends {
-      Insert: infer I
+  ? Database[PublicTableNameOrOptions['schema']]['Tables'][TableName] extends {
+      Insert: infer I;
     }
     ? I
     : never
-  : PublicTableNameOrOptions extends keyof PublicSchema["Tables"]
-    ? PublicSchema["Tables"][PublicTableNameOrOptions] extends {
-        Insert: infer I
-      }
-      ? I
-      : never
+  : PublicTableNameOrOptions extends keyof PublicSchema['Tables']
+  ? PublicSchema['Tables'][PublicTableNameOrOptions] extends {
+      Insert: infer I;
+    }
+    ? I
     : never
+  : never;
 
 export type TablesUpdate<
   PublicTableNameOrOptions extends
-    | keyof PublicSchema["Tables"]
+    | keyof PublicSchema['Tables']
     | { schema: keyof Database },
   TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
-    ? keyof Database[PublicTableNameOrOptions["schema"]]["Tables"]
+    ? keyof Database[PublicTableNameOrOptions['schema']]['Tables']
     : never = never,
 > = PublicTableNameOrOptions extends { schema: keyof Database }
-  ? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends {
-      Update: infer U
+  ? Database[PublicTableNameOrOptions['schema']]['Tables'][TableName] extends {
+      Update: infer U;
     }
     ? U
     : never
-  : PublicTableNameOrOptions extends keyof PublicSchema["Tables"]
-    ? PublicSchema["Tables"][PublicTableNameOrOptions] extends {
-        Update: infer U
-      }
-      ? U
-      : never
+  : PublicTableNameOrOptions extends keyof PublicSchema['Tables']
+  ? PublicSchema['Tables'][PublicTableNameOrOptions] extends {
+      Update: infer U;
+    }
+    ? U
     : never
+  : never;
 
 export type Enums<
   PublicEnumNameOrOptions extends
-    | keyof PublicSchema["Enums"]
+    | keyof PublicSchema['Enums']
     | { schema: keyof Database },
   EnumName extends PublicEnumNameOrOptions extends { schema: keyof Database }
-    ? keyof Database[PublicEnumNameOrOptions["schema"]]["Enums"]
+    ? keyof Database[PublicEnumNameOrOptions['schema']]['Enums']
     : never = never,
 > = PublicEnumNameOrOptions extends { schema: keyof Database }
-  ? Database[PublicEnumNameOrOptions["schema"]]["Enums"][EnumName]
-  : PublicEnumNameOrOptions extends keyof PublicSchema["Enums"]
-    ? PublicSchema["Enums"][PublicEnumNameOrOptions]
-    : never
+  ? Database[PublicEnumNameOrOptions['schema']]['Enums'][EnumName]
+  : PublicEnumNameOrOptions extends keyof PublicSchema['Enums']
+  ? PublicSchema['Enums'][PublicEnumNameOrOptions]
+  : never;
diff --git a/migrations/supabase/migrations/20240911082857_remote_schema.sql b/migrations/supabase/migrations/20240911082857_remote_schema.sql
@@ -149,7 +149,7 @@ $$;
 
 ALTER FUNCTION "public"."match_documents"("query_embedding" "public"."vector", "filter" "jsonb") OWNER TO "postgres";
 
-CREATE OR REPLACE FUNCTION "public"."match_rag_docs"("query_embedding" "public"."vector", "filter" "jsonb" DEFAULT '{}'::"jsonb") RETURNS TABLE("id" "uuid", "content" "text", "metadata" "jsonb", "embedding" "public"."vector", "similarity" double precision)
+CREATE OR REPLACE FUNCTION "public"."match_embedding_docs"("query_embedding" "public"."vector", "filter" "jsonb" DEFAULT '{}'::"jsonb") RETURNS TABLE("id" "uuid", "content" "text", "metadata" "jsonb", "embedding" "public"."vector", "similarity" double precision)
     LANGUAGE "plpgsql"
     AS $$
 #variable_conflict use_column
@@ -164,7 +164,7 @@ begin
   ) as similarity
   from rag_docs
   where metadata @> jsonb_extract_path(filter, 'metadata')
-  and bot_id = jsonb_extract_path_text(filter, 'bot_id')
+  and repo_name = jsonb_extract_path_text(filter, 'repo_name')
   order by rag_docs.embedding <=> query_embedding;
 end;
 $$;

diff --git a/package.json b/package.json
@@ -7,8 +7,7 @@
     "client": "cd client && yarn run dev",
     "assistant": "cd assistant && yarn run dev",
     "server": "cd server && ./venv/bin/python3 -m uvicorn main:app --reload",
-    "env:pull": "cd server && ./venv/bin/python3 scripts/envs.py pull",
-    "env:push": "cd server && ./venv/bin/python3 scripts/envs.py push",
+    "env:pull": "server/venv/bin/python3 scripts/envs.py pull",
     "client:server": "concurrently \"yarn run server\" \"yarn run client\"",
     "assistant:server": "concurrently \"yarn run server\" \"yarn run assistant\"",
     "build:docker": "docker build -t petercat .",