vertexai-chat | lint

tmc · Aug 22, 2023 · 3612dc5 · 3612dc5
1 parent db5b418
commit 3612dc5
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/textsplitter/split_documents.go b/textsplitter/split_documents.go
@@ -68,7 +68,7 @@ func joinDocs(docs []string, separator string) string {
 }
 
 // mergeSplits merges smaller splits into splits that are closer to the chunkSize.
-func mergeSplits(splits []string, separator string, chunkSize int, chunkOverlap int) []string {
+func mergeSplits(splits []string, separator string, chunkSize int, chunkOverlap int) []string { //nolint:cyclop
 	docs := make([]string, 0)
 	currentDoc := make([]string, 0)
 	total := 0
@@ -87,11 +87,14 @@ func mergeSplits(splits []string, separator string, chunkSize int, chunkOverlap
 			}
 
 			for shouldPop(chunkOverlap, chunkSize, total, len(split), len(separator), len(currentDoc)) {
+				if len(currentDoc) == 0 {
+					continue
+				}
+
 				total -= len(currentDoc[0])
 				if len(currentDoc) > 1 {
 					total -= len(separator)
 				}
-
 				currentDoc = currentDoc[1:]
 			}
 		}