Resize-in-place for large allocs. (Shrink-only)

sdlib/d/gc/arena.d

@@ -148,6 +148,34 @@ public:
 		return null;
 	}
 
+	bool resizeLarge(shared(ExtentMap)* emap, Extent* e, size_t size) shared {
+		assert(e !is null, "Extent is null!");
+		assert(e.isLarge(), "Expected a large extent!");
+		assert(e.arenaIndex == index, "Invalid arena index!");
+		assert(isLargeSize(size));
+
+		// Huge is not supported:
+		if (e.isHuge()) {
+			return false;
+		}
+
+		auto computedPageCount = alignUp(size, PageSize) / PageSize;
+		uint newPages = computedPageCount & uint.max;
+		assert(newPages == computedPageCount, "Unexpected page count!");
+
+		auto currentSize = e.size;
+		uint oldPages = cast(uint) (currentSize / PageSize);
+
+		// Growing is not yet supported:
+		if (newPages >= oldPages) {
+			return newPages == oldPages;
+		}
+
+		shrinkAlloc(emap, e, newPages);
+
+		return true;
+	}
+
 	/**
 	 * Deallocation facility.
 	 */
@@ -244,6 +272,32 @@ private:
 		(cast(Arena*) &this).freePagesImpl(e, n, pages);
 	}
 
+	void shrinkAlloc(shared(ExtentMap)* emap, Extent* e, uint pages) shared {
+		assert(!e.isHuge(), "Does not support huge!");
+		assert(isAligned(e.address, PageSize), "Invalid extent address!");
+		assert(isAligned(e.size, PageSize), "Invalid extent size!");
+		assert(e.hpd.address is alignDown(e.address, HugePageSize),
+		       "Invalid hpd!");
+		assert(pages > 0 && pages <= PageCount, "Invalid number of pages!");
+
+		uint currentPages = cast(uint) (e.size / PageSize);
+		assert(currentPages > pages, "Invalid shrink pages!");
+
+		uint delta = currentPages - pages;
+
+		uint n = ((cast(size_t) e.address) / PageSize) % PageCount;
+		uint index = n + pages;
+
+		auto newSize = pages * PageSize;
+		emap.clear(e.address + newSize, e.size - newSize);
+
+		mutex.lock();
+		scope(exit) mutex.unlock();
+
+		// Shrink:
+		(cast(Arena*) &this).shrinkAllocImpl(e, index, pages, delta);
+	}
+
 private:
 	Extent* allocPagesImpl(uint pages, ulong mask, ExtentClass ec) {
 		assert(mutex.isHeld(), "Mutex not held!");
@@ -338,6 +392,22 @@ private:
 		unusedExtents.insert(e);
 	}
 
+	void shrinkAllocImpl(Extent* e, uint index, uint pages, uint delta) {
+		assert(mutex.isHeld(), "Mutex not held!");
+		assert(index > 0 && index <= PageCount - pages, "Invalid index!");
+		assert(pages > 0 && pages <= PageCount, "Invalid number of pages!");
+		assert(delta > 0, "Invalid delta!");
+
+		auto hpd = e.hpd;
+		unregisterHPD(hpd);
+
+		e.at(e.address, pages * PageSize, hpd);
+		hpd.clear(index, delta);
+
+		assert(!hpd.empty);
+		registerHPD(hpd);
+	}
+
 	/**
 	 * HugePageDescriptor heaps management.
 	 */
@@ -575,3 +645,137 @@ unittest allocHuge {
 	arena.freePages(e3);
 	arena.freePages(e4);
 }
+
+unittest resizeLarge {
+	import d.gc.arena;
+	shared Arena arena;
+
+	auto base = &arena.base;
+	scope(exit) arena.base.clear();
+
+	import d.gc.emap;
+	static shared ExtentMap emap;
+	emap.tree.base = base;
+
+	import d.gc.region;
+	shared RegionAllocator regionAllocator;
+	regionAllocator.base = base;
+
+	arena.regionAllocator = &regionAllocator;
+
+	// Allocation 0: 35 pages:
+	auto ptr0 = arena.allocLarge(&emap, 35 * PageSize);
+	assert(ptr0 !is null);
+	auto pd0 = emap.lookup(ptr0);
+	assert(pd0.extent.address is ptr0);
+	assert(pd0.extent.size == 35 * PageSize);
+	auto pd0x = emap.lookup(ptr0);
+	assert(pd0x.extent.address is ptr0);
+
+	// Allocation 1: 20 pages:
+	auto ptr1 = arena.allocLarge(&emap, 20 * PageSize);
+	assert(ptr1 !is null);
+	assert(ptr1 is ptr0 + 35 * PageSize);
+	auto pd1 = emap.lookup(ptr1);
+	assert(pd1.extent.address is ptr1);
+	assert(pd1.extent.size == 20 * PageSize);
+
+	// Growing by zero is allowed:
+	assert(arena.resizeLarge(&emap, pd0.extent, 35 * PageSize));
+
+	// Shrink no. 0 down to 10 pages:
+	assert(arena.resizeLarge(&emap, pd0.extent, 10 * PageSize));
+	assert(pd0.extent.address is ptr0);
+	assert(pd0.extent.size == 10 * PageSize);
+	auto pd0xx = emap.lookup(ptr0);
+	assert(pd0xx.extent.address is ptr0);
+
+	// See that newly-last page is mapped:
+	auto okpd = emap.lookup(ptr0 + 9 * PageSize);
+	assert(okpd.extent !is null);
+
+	// But the page after the newly-last one, should not be mapped:
+	auto badpd = emap.lookup(ptr0 + 10 * PageSize);
+	assert(badpd.extent is null);
+
+	// Allocate 25 pages + 1 byte, will not fit in the hole after no.0:
+	auto ptr2 = arena.allocLarge(&emap, 1 + 25 * PageSize);
+	assert(ptr2 !is null);
+	auto pd2 = emap.lookup(ptr2);
+	assert(pd2.extent.address is ptr2);
+	assert(ptr2 is ptr1 + 20 * PageSize);
+
+	// Now allocate precisely 25 pages.
+	// This new alloc WILL fit in and fill the free space after no. 0:
+	auto ptr3 = arena.allocLarge(&emap, 25 * PageSize);
+	assert(ptr3 !is null);
+	auto pd3 = emap.lookup(ptr3);
+	assert(pd3.extent.address is ptr3);
+	assert(ptr3 is ptr0 + 10 * PageSize);
+
+	arena.free(&emap, pd0, ptr0);
+	arena.free(&emap, pd1, ptr1);
+	arena.free(&emap, pd2, ptr2);
+	arena.free(&emap, pd3, ptr3);
+
+	// Allocate 128 pages:
+	auto ptr4 = arena.allocLarge(&emap, 128 * PageSize);
+	assert(ptr4 !is null);
+	auto pd4 = emap.lookup(ptr4);
+	assert(pd4.extent.address is ptr4);
+
+	// Allocate 256 pages:
+	auto ptr5 = arena.allocLarge(&emap, 256 * PageSize);
+	assert(ptr5 !is null);
+	auto pd5 = emap.lookup(ptr5);
+	assert(pd5.extent.address is ptr5);
+	assert(pd5.extent.hpd == pd4.extent.hpd);
+
+	// Allocate 128 pages, hpd full:
+	auto ptr6 = arena.allocLarge(&emap, 128 * PageSize);
+	assert(ptr6 !is null);
+	auto pd6 = emap.lookup(ptr6);
+	assert(pd6.extent.address is ptr6);
+	assert(pd6.extent.hpd == pd5.extent.hpd);
+	assert(pd6.extent.hpd.full);
+
+	// Shrink first alloc:
+	assert(arena.resizeLarge(&emap, pd4.extent, 96 * PageSize));
+	assert(pd4.extent.size == 96 * PageSize);
+	assert(!pd6.extent.hpd.full);
+
+	// Shrink second alloc:
+	assert(arena.resizeLarge(&emap, pd5.extent, 128 * PageSize));
+	assert(pd5.extent.size == 128 * PageSize);
+
+	// Shrink third alloc:
+	assert(arena.resizeLarge(&emap, pd6.extent, 64 * PageSize));
+	assert(pd6.extent.size == 64 * PageSize);
+
+	// Allocate 128 pages, should go after second alloc:
+	auto ptr7 = arena.allocLarge(&emap, 128 * PageSize);
+	assert(ptr7 !is null);
+	auto pd7 = emap.lookup(ptr7);
+	assert(pd7.extent.address is ptr7);
+	assert(pd7.extent.hpd == pd6.extent.hpd);
+	assert(ptr7 is ptr5 + 128 * PageSize);
+
+	// Allocate 32 pages, should go after first alloc:
+	auto ptr8 = arena.allocLarge(&emap, 32 * PageSize);
+	assert(ptr8 !is null);
+	auto pd8 = emap.lookup(ptr8);
+	assert(pd8.extent.address is ptr8);
+	assert(pd8.extent.hpd == pd7.extent.hpd);
+	assert(ptr8 is ptr4 + 96 * PageSize);
+
+	// Allocate 64 pages, should go after third alloc:
+	auto ptr9 = arena.allocLarge(&emap, 64 * PageSize);
+	assert(ptr9 !is null);
+	auto pd9 = emap.lookup(ptr9);
+	assert(pd9.extent.address is ptr9);
+	assert(pd9.extent.hpd == pd8.extent.hpd);
+	assert(ptr9 is ptr6 + 64 * PageSize);
+
+	// Now full again:
+	assert(pd9.extent.hpd.full);
+}

sdlib/d/gc/tcache.d

@@ -98,12 +98,13 @@ public:
 			}
 		} else {
 			auto esize = pd.extent.size;
-			if (samePointerness && alignUp(size, PageSize) == esize) {
+			if (samePointerness && (alignUp(size, PageSize) == esize
+				    || (isLargeSize(size)
+					    && pd.arena.resizeLarge(emap, pd.extent, size)))) {
 				pd.extent.setUsedCapacity(size);
 				return ptr;
 			}
 
-			// TODO: Try to extend/shrink in place.
 			import d.gc.util;
 			copySize = min(size, pd.extent.usedCapacity);
 		}
@@ -440,10 +441,12 @@ unittest getCapacity {
 	assert(threadCache.getCapacity(p3[0 .. 20000]) == 0);
 	assert(threadCache.getCapacity(p3[0 .. 20001]) == 20480);
 
+	// This realloc happens in-place:
 	auto p4 = threadCache.realloc(p3, 16000, false);
-	assert(p4 !is p3);
+	assert(p4 is p3);
 	assert(threadCache.getCapacity(p4[0 .. 16000]) == 16384);
 
+	// This one, not:
 	auto p5 = threadCache.realloc(p4, 20000, false);
 	assert(p5 !is p4);
 	assert(threadCache.getCapacity(p5[0 .. 20000]) == 20480);