From 0869ec16772e8219c5007e2ce4beb5621b5069c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Amaury=20S=C3=A9chet?= <deadalnix@gmail.com>
Date: Mon, 21 Oct 2024 14:05:15 +0000
Subject: [PATCH] [GC] Inline worklist management when marking dense objects.

---
 sdlib/d/gc/scanner.d | 39 +++++++++++++++++++++------------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/sdlib/d/gc/scanner.d b/sdlib/d/gc/scanner.d
index 36065304b..749e3d04a 100644
--- a/sdlib/d/gc/scanner.d
+++ b/sdlib/d/gc/scanner.d
@@ -302,15 +302,29 @@ public:
 					auto pd = lastDenseSlabPageDescriptor;
 					auto slotSize = lastDenseBin.slotSize;
 
-					if (!markDense(base, index, pd)) {
+					if (!markDense(pd, index)) {
 						continue;
 					}
 
-					if (pd.containsPointers) {
-						auto item = WorkItem(base + index * slotSize, slotSize);
-						addToWorkList(item);
+					if (!pd.containsPointers) {
+						continue;
+					}
+
+					auto i = WorkItem(base + index * slotSize, slotSize);
+					if (likely(cursor < WorkListCapacity)) {
+						worklist[cursor++] = i;
+						continue;
 					}
 
+					/**
+					 * We want to make sure that no matter what, this worker processes
+					 * the WorkItem it is provided directly. Everything else in the
+					 * worklist can be sent to other threads.
+					 */
+					scanner.addToWorkList(worklist[0 .. WorkListCapacity]);
+
+					cursor = 1;
+					worklist[0] = i;
 					continue;
 				}
 
@@ -353,7 +367,7 @@ public:
 	}
 
 private:
-	bool markDense(const void* base, uint index, PageDescriptor pd) {
+	static bool markDense(PageDescriptor pd, uint index) {
 		auto e = pd.extent;
 
 		/**
@@ -382,19 +396,6 @@ private:
 		return true;
 	}
 
-	void addToWorkList(WorkItem item) {
-		if (likely(cursor < WorkListCapacity)) {
-			worklist[cursor++] = item;
-			return;
-		}
-
-		// Flush the current worklist and add the item.
-		scanner.addToWorkList(worklist[0 .. WorkListCapacity]);
-
-		cursor = 1;
-		worklist[0] = item;
-	}
-
 	void markSparse(PageDescriptor pd, const void* ptr, ubyte cycle) {
 		import d.gc.slab;
 		auto se = SlabEntry(pd, ptr);
@@ -499,6 +500,7 @@ public:
 
 	this(const void* ptr, size_t length) {
 		assert(isAligned(ptr, PointerSize), "Invalid ptr!");
+		assert(length >= PointerSize, "WorkItem cannot be empty!");
 
 		auto storedLength = length / PointerSize - 1;
 		assert(storedLength < (1 << FreeBits), "Invalid length!");
@@ -509,6 +511,7 @@ public:
 
 	this(const(void*)[] range) {
 		assert(range.length > 0, "WorkItem cannot be empty!");
+		assert(range.length <= (1 << FreeBits), "Invalid length!");
 
 		payload = cast(size_t) range.ptr;
 		payload |= (range.length - 1) << LengthShift;