From f7e737cfd9db25d478c7b0b2471a6a2db9b72af4 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 19 Apr 2015 12:08:07 +0200 Subject: [PATCH 1/9] precise GC as presented at DConf 2013 --- src/core/memory.d | 21 +++ src/gc/bits.d | 288 ++++++++++++++++++++++++++++++++++ src/gc/config.d | 2 +- src/gc/gc.d | 283 +++++++++++++++++++++++++++++---- src/gc/proxy.d | 9 ++ src/gcstub/gc.d | 9 ++ src/object.d | 28 +++- src/rt/lifetime.d | 58 ++++++- src/rt/typeinfo/ti_byte.d | 2 + src/rt/typeinfo/ti_cdouble.d | 2 + src/rt/typeinfo/ti_cfloat.d | 2 + src/rt/typeinfo/ti_char.d | 2 + src/rt/typeinfo/ti_creal.d | 2 + src/rt/typeinfo/ti_dchar.d | 2 + src/rt/typeinfo/ti_delegate.d | 2 + src/rt/typeinfo/ti_double.d | 2 + src/rt/typeinfo/ti_float.d | 2 + src/rt/typeinfo/ti_int.d | 2 + src/rt/typeinfo/ti_long.d | 2 + src/rt/typeinfo/ti_ptr.d | 2 + src/rt/typeinfo/ti_real.d | 2 + src/rt/typeinfo/ti_short.d | 2 + src/rt/typeinfo/ti_ubyte.d | 2 + src/rt/typeinfo/ti_uint.d | 2 + src/rt/typeinfo/ti_ulong.d | 2 + src/rt/typeinfo/ti_ushort.d | 2 + src/rt/typeinfo/ti_wchar.d | 2 + 27 files changed, 698 insertions(+), 38 deletions(-) diff --git a/src/core/memory.d b/src/core/memory.d index 22e2fe9b43..f32456148b 100644 --- a/src/core/memory.d +++ b/src/core/memory.d @@ -126,6 +126,7 @@ private extern (C) size_t gc_extend( void* p, size_t mx, size_t sz, const TypeInfo = null ) pure nothrow; extern (C) size_t gc_reserve( size_t sz ) nothrow; extern (C) void gc_free( void* p ) pure nothrow; + extern (C) bool gc_emplace( void *p, size_t len, const TypeInfo ti ) pure nothrow; extern (C) void* gc_addrOf( void* p ) pure nothrow; extern (C) size_t gc_sizeOf( void* p ) pure nothrow; @@ -248,6 +249,10 @@ struct GC NO_INTERIOR = 0b0001_0000, STRUCTFINAL = 0b0010_0000, // the block has a finalizer for (an array of) structs + + // additional info for allocating with type info + NO_RTINFO = 0b0100_0000, // do not copy RTInfo in malloc/realloc + REP_RTINFO = 0b1000_0000, // repeat RTInfo if allocation is larger than type info } @@ -659,6 +664,22 @@ struct GC return gc_query( p ); } + /** + * Describe the memory at the given address range for precise collection + * + * Params: + * p = A pointer to the root or the interior of a valid memory block + * len = Length of the memory range + * ti = Type info describing the memory usage + * + * Returns: + * true if p points to GC managed memory + */ + static bool emplace( void *p, size_t len, const TypeInfo ti ) pure nothrow + { + return gc_emplace( p, len, ti ); + } + /** * Adds an internal root pointing to the GC memory block referenced by p. diff --git a/src/gc/bits.d b/src/gc/bits.d index 60ba4f4eef..ca8acf1584 100644 --- a/src/gc/bits.d +++ b/src/gc/bits.d @@ -19,6 +19,11 @@ import core.stdc.string; import core.stdc.stdlib; import core.exception : onOutOfMemoryError; + +// use version bitwise to disable optimizations that use word operands +// on bulk operation copyRange, setRange, clrRange, etc. +// version = bitwise; + struct GCBits { alias size_t wordtype; @@ -26,7 +31,9 @@ struct GCBits enum BITS_PER_WORD = (wordtype.sizeof * 8); enum BITS_SHIFT = (wordtype.sizeof == 8 ? 6 : 5); enum BITS_MASK = (BITS_PER_WORD - 1); + enum BITS_0 = cast(wordtype)0; enum BITS_1 = cast(wordtype)1; + enum BITS_2 = cast(wordtype)2; wordtype* data; size_t nbits; @@ -78,6 +85,287 @@ struct GCBits return core.bitop.btr(data, i); } + mixin template RangeVars() + { + size_t firstWord = (target >> BITS_SHIFT); + size_t firstOff = target & BITS_MASK; + size_t last = target + len - 1; + size_t lastWord = (last >> BITS_SHIFT); + size_t lastOff = last & BITS_MASK; + } + + // target = the biti to start the copy to + // destlen = the number of bits to copy from source + void copyRange(size_t target, size_t len, const(wordtype)* source) nothrow + { + version(bitwise) + { + for (size_t i = 0; i < len; i++) + if(source[(i >> BITS_SHIFT)] & (BITS_1 << (i & BITS_MASK))) + set(target+i); + else + clear(target+i); + } + else + { + if(len == 0) + return; + + mixin RangeVars!(); + + if(firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] = (data[firstWord] & ~mask) | ((source[0] << firstOff) & mask); + } + else if(firstOff == 0) + { + for(size_t w = firstWord; w < lastWord; w++) + data[w] = source[w - firstWord]; + + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] = (data[lastWord] & ~mask) | (source[lastWord - firstWord] & mask); + } + else + { + size_t cntWords = lastWord - firstWord; + wordtype mask = ~BITS_0 << firstOff; + data[firstWord] = (data[firstWord] & ~mask) | (source[0] << firstOff); + for(size_t w = 1; w < cntWords; w++) + data[firstWord + w] = (source[w - 1] >> (BITS_PER_WORD - firstOff)) | (source[w] << firstOff); + + wordtype src = (source[cntWords - 1] >> (BITS_PER_WORD - firstOff)) | (source[cntWords] << firstOff); + mask = (BITS_2 << lastOff) - 1; + data[lastWord] = (data[lastWord] & ~mask) | (src & mask); + } + } + } + + void copyRangeRepeating(size_t target, size_t destlen, const(wordtype)* source, size_t sourcelen) nothrow + { + version(bitwise) + { + for (size_t i=0; i < destlen; i++) + { + bool b; + size_t j = i % sourcelen; + b = (source[j >> BITS_SHIFT] & (BITS_1 << (j & BITS_MASK))) != 0; + if (b) set(target+i); + else clear(target+i); + } + } + else + { + if (destlen > 4 * sourcelen && destlen > 4 * BITS_PER_WORD) + { + // precalculate the number of words where a bit pattern of the + // source length repeats on word alignment + static ubyte lcm(ubyte i) + { + // calc lcm(i,BITS_PER_WORD)/BITS_PER_WORD + // by just stripping all factors 2 from i + if ((i & (i - 1)) == 0) + return 1; + while((i & 1) == 0) + i >>= 1; + return i; + } + static calcRepLength() + { + ubyte[BITS_PER_WORD] rep; + for (ubyte i = 0; i < BITS_PER_WORD; i++) + rep[i] = lcm(i); + return rep; + } + static immutable repLength = calcRepLength(); + + // make some initial copies until we have a pattern that + // repeats on word boundary + size_t rep = repLength[sourcelen & BITS_MASK]; + size_t repwords = ((sourcelen >> BITS_SHIFT) + 1) * rep; + size_t alignbits = (target & BITS_MASK ? BITS_PER_WORD - (target & BITS_MASK) : 0); + size_t initbits = BITS_PER_WORD * repwords + alignbits; + + if (initbits < destlen) + { + while (initbits > sourcelen) + { + copyRange(target, sourcelen, source); + target += sourcelen; + destlen -= sourcelen; + initbits -= sourcelen; + } + copyRange(target, initbits, source); + target += sourcelen; + destlen -= sourcelen; + assert((target & BITS_MASK) == 0); + + size_t tpos = target >> BITS_SHIFT; + while (destlen >= BITS_PER_WORD) + { + data[tpos] = data[tpos - repwords]; + destlen -= BITS_PER_WORD; + tpos++; + } + + wordtype mask = (BITS_2 << destlen) - 1; + data[tpos] = (data[tpos] & ~mask) | (data[tpos - repwords] & mask); + return; + } + } + + while (destlen > sourcelen) + { + copyRange(target, sourcelen, source); + target += sourcelen; + destlen -= sourcelen; + } + copyRange(target, destlen, source); + } + } + + void setRange(size_t target, size_t len) nothrow + { + version(bitwise) + { + for (size_t i = 0; i < len; i++) + set(target+i); + } + else + { + if(len == 0) + return; + + mixin RangeVars!(); + + if(firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] |= mask; + } + else + { + data[firstWord] |= ~BITS_0 << firstOff; + for(size_t w = firstWord + 1; w < lastWord; w++) + data[w] = ~0; + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] |= mask; + } + } + } + + void clrRange(size_t target, size_t len) nothrow + { + version(bitwise) + { + for (size_t i = 0; i < len; i++) + clear(target+i); + } + else + { + if(len == 0) + return; + + mixin RangeVars!(); + + if(firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] &= ~mask; + } + else + { + data[firstWord] &= ~(~BITS_0 << firstOff); + for(size_t w = firstWord + 1; w < lastWord; w++) + data[w] = 0; + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] &= ~mask; + } + } + } + + unittest + { + GCBits bits; + bits.alloc(1000); + auto data = bits.data; + + bits.setRange(0,1); + assert(data[0] == 1); + + bits.clrRange(0,1); + assert(data[0] == 0); + + bits.setRange(BITS_PER_WORD-1,1); + assert(data[0] == BITS_1 << (BITS_PER_WORD-1)); + + bits.clrRange(BITS_PER_WORD-1,1); + assert(data[0] == 0); + + bits.setRange(12,7); + assert(data[0] == 0b0111_1111_0000_0000_0000); + + bits.clrRange(14,4); + assert(data[0] == 0b0100_0011_0000_0000_0000); + + bits.clrRange(0,BITS_PER_WORD); + assert(data[0] == 0); + + bits.setRange(0,BITS_PER_WORD); + assert(data[0] == ~0); + assert(data[1] == 0); + + bits.setRange(BITS_PER_WORD,BITS_PER_WORD); + assert(data[0] == ~0); + assert(data[1] == ~0); + assert(data[2] == 0); + bits.clrRange(BITS_PER_WORD/2,BITS_PER_WORD); + assert(data[0] == (BITS_1 << (BITS_PER_WORD/2)) - 1); + assert(data[1] == ~data[0]); + assert(data[2] == 0); + + bits.setRange(8*BITS_PER_WORD+1,4*BITS_PER_WORD-2); + assert(data[8] == ~0 << 1); + assert(data[9] == ~0); + assert(data[10] == ~0); + assert(data[11] == cast(wordtype)~0 >> 1); + + bits.clrRange(9*BITS_PER_WORD+1,2*BITS_PER_WORD); + assert(data[8] == ~0 << 1); + assert(data[9] == 1); + assert(data[10] == 0); + assert(data[11] == ((cast(wordtype)~0 >> 1) & ~1)); + + wordtype[4] src = [ 0xa, 0x5, 0xaa, 0x55 ]; + + void testCopyRange(size_t start, size_t len, int repeat = 1) + { + bits.setRange(0, bits.nbits); + if (repeat > 1) + bits.copyRangeRepeating(start, repeat * len, src.ptr, len); + else + bits.copyRange(start, len, src.ptr); + foreach (i; 0 .. start) + assert(bits.test(i)); + foreach (r; 0 .. repeat) + foreach (i; 0 .. len) + assert(!bits.test(start + r*len + i) == !core.bitop.bt(src.ptr, i)); + foreach (i; start + repeat*len .. 10*BITS_PER_WORD) + assert(bits.test(i)); + } + + testCopyRange(20, 10); // short copy range within same word + testCopyRange(50, 20); // short copy range spanning two words + testCopyRange(64, 3 * BITS_PER_WORD + 3); // aligned copy range + testCopyRange(77, 2 * BITS_PER_WORD + 15); // unaligned copy range + testCopyRange(64, 127); // copy range within critical end alignment + + testCopyRange(10, 4, 5); // repeating small range within same word + testCopyRange(20, 5, 10); // repeating small range spanning two words + testCopyRange(40, 21, 7); // repeating medium range + testCopyRange(73, 2 * BITS_PER_WORD + 15, 5); // repeating multi-word range + } + void zero() nothrow { memset(data, 0, nwords * wordtype.sizeof); diff --git a/src/gc/config.d b/src/gc/config.d index 95c6fb724f..89f8ef45c6 100644 --- a/src/gc/config.d +++ b/src/gc/config.d @@ -24,7 +24,7 @@ struct Config { bool disable; // start disabled ubyte profile; // enable profiling with summary when terminating program - bool precise; // enable precise scanning + bool precise = 1; // enable precise scanning bool concurrent; // enable concurrent collection size_t initReserve; // initial reserve (MB) diff --git a/src/gc/gc.d b/src/gc/gc.d index c380966c86..77b2fe5e05 100644 --- a/src/gc/gc.d +++ b/src/gc/gc.d @@ -117,10 +117,12 @@ __gshared long numFrees; __gshared long numReallocs; __gshared long numExtends; __gshared long numOthers; +__gshared long numEmplaces; __gshared long mallocTime; // using ticks instead of MonoTime for better performance __gshared long freeTime; __gshared long reallocTime; __gshared long extendTime; +__gshared long emplaceTime; __gshared long otherTime; __gshared long lockTime; @@ -252,9 +254,40 @@ debug (LOGGING) /* ============================ GC =============================== */ -const uint GCVERSION = 1; // increment every time we change interface +const uint GCVERSION = 2; // increment every time we change interface // to GC. +debug(PRINTF) +void printGCBits(GCBits* bits) +{ + for (size_t i = 0; i %p\n", p); return p; } @@ -590,7 +626,7 @@ struct GC { void *p2; size_t psize; - //debug(PRINTF) printf("GC::realloc(p = %p, size = %zu)\n", p, size); + //debug(PRINTF) printf("GC::realloc(p = %p, size = %llu)\n", p, cast(long)size); debug (SENTINEL) { sentinel_Invariant(p); @@ -639,6 +675,8 @@ struct GC auto newsz = (size + PAGESIZE - 1) / PAGESIZE; if (newsz == psz) { + if(config.precise && !(bits & BlkAttr.NO_RTINFO)) + pool.setPointerBitmap(p, size, newsz * PAGESIZE, ti, (bits & BlkAttr.REP_RTINFO) != 0); alloc_size = psize; return p; } @@ -674,6 +712,8 @@ struct GC pool.setBits(biti, bits); } alloc_size = newsz * PAGESIZE; + if(config.precise && !(bits & BlkAttr.NO_RTINFO)) + pool.setPointerBitmap(p, size, newsz * PAGESIZE, ti, (bits & BlkAttr.REP_RTINFO) != 0); return p; } @@ -704,7 +744,11 @@ struct GC p = p2; } else + { alloc_size = psize; + if(config.precise && !(bits & BlkAttr.NO_RTINFO)) + pool.setPointerBitmap(p, size, psize, ti, (bits & BlkAttr.REP_RTINFO) != 0); + } } } return p; @@ -736,7 +780,7 @@ struct GC } body { - //debug(PRINTF) printf("GC::extend(p = %p, minsize = %zu, maxsize = %zu)\n", p, minsize, maxsize); + //debug(PRINTF) printf("GC::extend(p = %p, minsize = %llu, maxsize = %llu)\n", p, cast(ulong)minsize, cast(ulong)maxsize); debug (SENTINEL) { return 0; @@ -777,6 +821,13 @@ struct GC lpool.updateOffsets(pagenum); lpool.freepages -= sz; gcx.usedLargePages += sz; + if (config.precise) + { + auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + if(!pool.noscan.test(biti)) + pool.setPointerBitmap(p, psize + sz * PAGESIZE, psize + sz * PAGESIZE, ti, true); + } + return (psz + sz) * PAGESIZE; } } @@ -1061,6 +1112,42 @@ struct GC } + /** + * Tell the GC the type of the memory range + */ + bool emplace(void *p, size_t len, const(TypeInfo) ti) nothrow + { + if(!config.precise) + return false; + if (!p) + return false; + + mixin doLock!(); + bool rc = emplaceNoSync(p, len, ti); + mixin doUnlock!(emplaceTime, numEmplaces); + return rc; + } + + + // + // + // + private bool emplaceNoSync(void *p, size_t len, const(TypeInfo) ti) nothrow + { + debug(PRINTF) printf("Emplacing %s at %p + %d\n", debugTypeName(ti).ptr, cast(size_t) p, len); + assert(p); + + sentinel_Invariant(p); + Pool* pool; + BlkInfo info = gcx.getInfo(p, &pool); + if(!info.base) + return false; + size_t allocSize = info.size - (p - info.base); + pool.setPointerBitmap(p, len, allocSize, ti, true); + return true; + } + + /** * add p to list of roots */ @@ -1401,10 +1488,11 @@ struct Gcx printf("\trealloc: %llu calls, %lld ms\n", cast(ulong)numReallocs, toDuration(reallocTime).total!"msecs"); printf("\tfree: %llu calls, %lld ms\n", cast(ulong)numFrees, toDuration(freeTime).total!"msecs"); printf("\textend: %llu calls, %lld ms\n", cast(ulong)numExtends, toDuration(extendTime).total!"msecs"); + printf("\templace: %llu calls, %lld ms\n", cast(ulong)numEmplaces, toDuration(emplaceTime).total!"msecs"); printf("\tother: %llu calls, %lld ms\n", cast(ulong)numOthers, toDuration(otherTime).total!"msecs"); printf("\tlock time: %lld ms\n", toDuration(lockTime).total!"msecs"); - long apiTime = mallocTime + reallocTime + freeTime + extendTime + otherTime + lockTime; + long apiTime = mallocTime + reallocTime + freeTime + extendTime + emplaceTime + otherTime + lockTime; printf("\tGC API: %lld ms\n", toDuration(apiTime).total!"msecs"); sprintf(apitxt.ptr, " API%5ld ms", toDuration(apiTime).total!"msecs"); } @@ -1453,8 +1541,12 @@ struct Gcx for (size_t i = 0; i < B_PAGE; i++) { + size_t j = 0; + List* prev, pprev, ppprev; // keep a short hitory to inspect in the debugger for (auto list = cast(List*)bucket[i]; list; list = list.next) { + ppprev = pprev; pprev = prev; prev = list; + j++; } } } @@ -1629,9 +1721,11 @@ struct Gcx /** * */ - BlkInfo getInfo(void* p) nothrow + BlkInfo getInfo(void* p, Pool** ppool = null) nothrow { Pool* pool = findPool(p); + if(ppool) + *ppool = pool; if (pool) return pool.slGetInfo(p); return BlkInfo(); @@ -1720,14 +1814,15 @@ struct Gcx return isLowOnMem(mappedPages * PAGESIZE); } - void* alloc(size_t size, ref size_t alloc_size, uint bits) nothrow + void* alloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow { - return size <= 2048 ? smallAlloc(binTable[size], alloc_size, bits) - : bigAlloc(size, alloc_size, bits); + return size <= PAGESIZE/2 ? smallAlloc(size, alloc_size, bits, ti) + : bigAlloc(size, alloc_size, bits, ti); } - void* smallAlloc(Bins bin, ref size_t alloc_size, uint bits) nothrow + void* smallAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow { + immutable bin = binTable[size]; alloc_size = binsize[bin]; void* p; @@ -1774,6 +1869,13 @@ struct Gcx pool.setBits((p - pool.baseAddr) >> pool.shiftBy, bits); //debug(PRINTF) printf("\tmalloc => %p\n", p); debug (MEMSTOMP) memset(p, 0xF0, alloc_size); + + if (GC.config.precise && !(bits & (BlkAttr.NO_SCAN | BlkAttr.NO_RTINFO))) + { + bool repeat = (bits & BlkAttr.REP_RTINFO) != 0; + pool.setPointerBitmap(sentinel_add(p), size, alloc_size, ti, repeat); + } + return p; } @@ -1781,7 +1883,7 @@ struct Gcx * Allocate a chunk of memory that is larger than a page. * Return null if out of memory. */ - void* bigAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti = null) nothrow + void* bigAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow { debug(PRINTF) printf("In bigAlloc. Size: %d\n", size); @@ -1855,6 +1957,9 @@ struct Gcx if (bits) pool.setBits(pn, bits); + if (GC.config.precise && !(bits & (BlkAttr.NO_SCAN | BlkAttr.NO_RTINFO))) + pool.setPointerBitmap(sentinel_add(p), size, alloc_size, ti, (bits & BlkAttr.REP_RTINFO) != 0); + return p; } @@ -1937,6 +2042,13 @@ struct Gcx return null; } + static struct ScanRange + { + void* pbot; + void* ptop; + Pool* pool; + } + static struct ToScanStack { nothrow: @@ -1945,25 +2057,25 @@ struct Gcx void reset() { _length = 0; - os_mem_unmap(_p, _cap * Range.sizeof); + os_mem_unmap(_p, _cap * ScanRange.sizeof); _p = null; _cap = 0; } - void push(Range rng) + void push(ScanRange rng) { if (_length == _cap) grow(); _p[_length++] = rng; } - Range pop() + ScanRange pop() in { assert(!empty); } body { return _p[--_length]; } - ref inout(Range) opIndex(size_t idx) inout + ref inout(ScanRange) opIndex(size_t idx) inout in { assert(idx < _length); } body { @@ -1977,20 +2089,20 @@ struct Gcx void grow() { enum initSize = 64 * 1024; // Windows VirtualAlloc granularity - immutable ncap = _cap ? 2 * _cap : initSize / Range.sizeof; - auto p = cast(Range*)os_mem_map(ncap * Range.sizeof); + immutable ncap = _cap ? 2 * _cap : initSize / ScanRange.sizeof; + auto p = cast(ScanRange*)os_mem_map(ncap * ScanRange.sizeof); if (p is null) onOutOfMemoryErrorNoGC(); if (_p !is null) { p[0 .. _length] = _p[0 .. _length]; - os_mem_unmap(_p, _cap * Range.sizeof); + os_mem_unmap(_p, _cap * ScanRange.sizeof); } _p = p; _cap = ncap; } size_t _length; - Range* _p; + ScanRange* _p; size_t _cap; } @@ -1999,7 +2111,7 @@ struct Gcx /** * Search a range of memory values and mark any pointers into the GC pool. */ - void mark(void *pbot, void *ptop) nothrow + void mark(bool precise)(void *pbot, void *ptop) nothrow { void **p1 = cast(void **)pbot; void **p2 = cast(void **)ptop; @@ -2007,8 +2119,15 @@ struct Gcx // limit the amount of ranges added to the toscan stack enum FANOUT_LIMIT = 32; size_t stackPos; - Range[FANOUT_LIMIT] stack = void; + ScanRange[FANOUT_LIMIT] stack = void; + + auto minAddr = pooltable.minAddr; + auto maxAddr = pooltable.maxAddr; + static if (precise) + Pool* p1pool = null; // always starting from a non-heap root + else + enum p1pool = null; Lagain: size_t pcache = 0; @@ -2029,6 +2148,12 @@ struct Gcx if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache) continue; + static if(precise) if (p1pool) + { + if (!p1pool.is_pointer.test(p1 - cast(void**)p1pool.baseAddr)) + continue; + } + Pool* pool = void; size_t low = 0; size_t high = highpool; @@ -2116,12 +2241,12 @@ struct Gcx } } - Range next=void; + ScanRange next=void; if (p1 < p2) { // local stack is full, push it to the global stack assert(stackPos == stack.length); - toscan.push(Range(p1, p2)); + toscan.push(ScanRange(p1, p2, p1pool)); // reverse order for depth-first-order traversal foreach_reverse (ref rng; stack[0 .. $ - 1]) toscan.push(rng); @@ -2145,10 +2270,22 @@ struct Gcx } p1 = cast(void**)next.pbot; p2 = cast(void**)next.ptop; + static if (precise) + p1pool = next.pool; // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); goto Lagain; } + void markConservative(void *pbot, void *ptop) nothrow + { + mark!false(pbot, ptop); + } + + void markPrecise(void *pbot, void *ptop) nothrow + { + mark!true(pbot, ptop); + } + // collection step 1: prepare freebits and mark bits void prepare() nothrow { @@ -2188,20 +2325,20 @@ struct Gcx } // collection step 2: mark roots and heap - void markAll(bool nostack) nothrow + void markAll(alias markFn)(bool nostack) nothrow { if (!nostack) { debug(COLLECT_PRINTF) printf("\tscan stacks.\n"); // Scan stacks and registers for each paused thread - thread_scanAll(&mark); + thread_scanAll(&markFn); } // Scan roots[] debug(COLLECT_PRINTF) printf("\tscan roots[]\n"); foreach (root; roots) { - mark(cast(void*)&root.proot, cast(void*)(&root.proot + 1)); + markFn(cast(void*)&root.proot, cast(void*)(&root.proot + 1)); } // Scan ranges[] @@ -2210,7 +2347,7 @@ struct Gcx foreach (range; ranges) { debug(COLLECT_PRINTF) printf("\t\t%p .. %p\n", range.pbot, range.ptop); - mark(range.pbot, range.ptop); + markFn(range.pbot, range.ptop); } //log--; } @@ -2439,7 +2576,10 @@ struct Gcx start = stop; } - markAll(nostack); + if (GC.config.precise) + markAll!markPrecise(nostack); + else + markAll!markConservative(nostack); thread_processGCMarks(&isMarked); thread_resumeAll(); @@ -2656,12 +2796,16 @@ struct Pool GCBits appendable; // entries that are appendable GCBits nointerior; // interior pointers should be ignored. // Only implemented for large object pools. + GCBits is_pointer; // precise GC only: per-word, not per-block like the rest of them size_t npages; size_t freepages; // The number of pages not in use. ubyte* pagetable; bool isLargeObject; + enum shiftBySmall = 4; + enum shiftByLarge = 12; + uint shiftBy; // shift count for the divisor used for determining bit indices. // This tracks how far back we have to go to find the nearest B_PAGE at @@ -2704,6 +2848,11 @@ struct Pool auto nbits = cast(size_t)poolsize >> shiftBy; mark.alloc(nbits); + if(GC.config.precise) + { + is_pointer.alloc(cast(size_t)poolsize/(void*).sizeof); + is_pointer.setRange(0, is_pointer.nbits); + } // pagetable already keeps track of what's free for the large object // pool. @@ -2761,6 +2910,8 @@ struct Pool cstdlib.free(bPageOffsets); mark.Dtor(); + if(GC.config.precise) + is_pointer.Dtor(); if(isLargeObject) { nointerior.Dtor(); @@ -2965,6 +3116,80 @@ struct Pool } } } + + void setPointerBitmap(void* p, size_t s, size_t allocSize, const TypeInfo ti, bool repeat) nothrow + { + size_t offset = p - baseAddr; + //debug(PRINTF) printGCBits(&pool.is_pointer); + + debug(PRINTF) + printf("Setting a pointer bitmap for %s at %p + %llu\n", debugTypeName(ti).ptr, p, cast(ulong)s); + + if (ti) + { + if (repeat) + s = allocSize; + + auto rtInfo = cast(const(size_t)*)ti.rtInfo(); + + if (rtInfo is rtinfoNoPointers) + { + debug(PRINTF) printf("\tCompiler generated rtInfo: no pointers\n"); + is_pointer.clrRange(offset/(void*).sizeof, s/(void*).sizeof); + } + else if (rtInfo is rtinfoHasPointers) + { + debug(PRINTF) printf("\tCompiler generated rtInfo: has pointers\n"); + is_pointer.setRange(offset/(void*).sizeof, s/(void*).sizeof); + } + else + { + const(size_t)* bitmap = cast (size_t*) rtInfo; + //first element of rtInfo is the size of the object the bitmap encodes + size_t element_size = * bitmap; + bitmap++; + size_t tocopy; + if(repeat) + { + tocopy = s/(void*).sizeof; + is_pointer.copyRangeRepeating(offset/(void*).sizeof, tocopy, bitmap, element_size/(void*).sizeof); + } + else + { + tocopy = (s < element_size ? s : element_size)/(void*).sizeof; + is_pointer.copyRange(offset/(void*).sizeof, tocopy, bitmap); + } + + debug(PRINTF) printf("\tSetting bitmap for new object (%s)\n\t\tat %p\t\tcopying from %p + %llu: ", + debugTypeName(ti).ptr, p, bitmap, cast(ulong)element_size); + debug(PRINTF) + for(size_t i = 0; i < element_size/((void*).sizeof); i++) + printf("%d", (bitmap[i/(8*size_t.sizeof)] >> (i%(8*size_t.sizeof))) & 1); + debug(PRINTF) printf("\n"); + + if(tocopy * (void*).sizeof < s) // better safe than sorry: if allocated more, assume pointers inside + { + debug(PRINTF) printf(" Appending %d pointer bits\n", s/(void*).sizeof - tocopy); + is_pointer.setRange(offset/(void*).sizeof + tocopy, s/(void*).sizeof - tocopy); + } + } + + if(s < allocSize) + { + offset = (offset + s + (void*).sizeof - 1) & ~((void*).sizeof - 1); + is_pointer.clrRange(offset/(void*).sizeof, (allocSize - s)/(void*).sizeof); + } + } + else + { + s = allocSize; + + debug(PRINTF) printf("Allocating a block without TypeInfo\n"); + is_pointer.setRange(offset/(void*).sizeof, s/(void*).sizeof); + } + //debug(PRINTF) printGCBits(&pool.is_pointer); + } + } struct LargeObjectPool diff --git a/src/gc/proxy.d b/src/gc/proxy.d index eb0f7ee275..6a2e2263a8 100644 --- a/src/gc/proxy.d +++ b/src/gc/proxy.d @@ -48,6 +48,7 @@ private size_t function(void*, size_t, size_t, const TypeInfo) gc_extend; size_t function(size_t) gc_reserve; void function(void*) gc_free; + bool function(void*, size_t, const TypeInfo) gc_emplace; void* function(void*) gc_addrOf; size_t function(void*) gc_sizeOf; @@ -86,6 +87,7 @@ private pthis.gc_extend = &gc_extend; pthis.gc_reserve = &gc_reserve; pthis.gc_free = &gc_free; + pthis.gc_emplace = &gc_emplace; pthis.gc_addrOf = &gc_addrOf; pthis.gc_sizeOf = &gc_sizeOf; @@ -239,6 +241,13 @@ extern (C) return proxy.gc_free( p ); } + bool gc_emplace( void* p, size_t len, const TypeInfo ti ) nothrow + { + if( proxy is null ) + return _gc.emplace( p, len, ti ); + return proxy.gc_emplace( p, len, ti ); + } + void* gc_addrOf( void* p ) nothrow { if( proxy is null ) diff --git a/src/gcstub/gc.d b/src/gcstub/gc.d index b357a8074e..355e4409de 100644 --- a/src/gcstub/gc.d +++ b/src/gcstub/gc.d @@ -55,6 +55,7 @@ private extern (C) size_t function(void*, size_t, size_t, const TypeInfo) gc_extend; extern (C) size_t function(size_t) gc_reserve; extern (C) void function(void*) gc_free; + extern (C) bool function(void*, size_t, const TypeInfo) gc_emplace; extern (C) void* function(void*) gc_addrOf; extern (C) size_t function(void*) gc_sizeOf; @@ -92,6 +93,7 @@ private pthis.gc_extend = &gc_extend; pthis.gc_reserve = &gc_reserve; pthis.gc_free = &gc_free; + pthis.gc_emplace = &gc_emplace; pthis.gc_addrOf = &gc_addrOf; pthis.gc_sizeOf = &gc_sizeOf; @@ -258,6 +260,13 @@ extern (C) void gc_free( void* p ) return proxy.gc_free( p ); } +extern (C) bool gc_emplace( void* p, size_t len, const TypeInfo ti ) +{ + if( proxy is null ) + return false; + return proxy.gc_emplace( p, len, ti ); +} + extern (C) void* gc_addrOf( void* p ) { if( proxy is null ) diff --git a/src/object.d b/src/object.d index 75ef67013d..9dcc023f42 100644 --- a/src/object.d +++ b/src/object.d @@ -322,7 +322,8 @@ class TypeInfo /** Return info used by the garbage collector to do precise collection. */ - @property immutable(void)* rtInfo() nothrow pure const @safe @nogc { return null; } + @property immutable(void)* rtInfo() nothrow pure const @safe @nogc { return rtinfoHasPointers; } // better safe than sorry + } class TypeInfo_Typedef : TypeInfo @@ -515,6 +516,8 @@ class TypeInfo_Array : TypeInfo arg2 = typeid(void*); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return RTInfo!(void[]); } } class TypeInfo_StaticArray : TypeInfo @@ -641,6 +644,9 @@ class TypeInfo_StaticArray : TypeInfo arg1 = typeid(void*); return 0; } + + // just return the rtInfo of the element, we have no generic type T to run RTInfo!T on + override @property immutable(void)* rtInfo() nothrow pure const @safe { return value.rtInfo(); } } class TypeInfo_AssociativeArray : TypeInfo @@ -762,6 +768,8 @@ class TypeInfo_Function : TypeInfo return null; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } + TypeInfo next; string deco; } @@ -834,6 +842,8 @@ class TypeInfo_Delegate : TypeInfo arg2 = typeid(void*); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return RTInfo!(int delegate()); } } unittest @@ -1214,7 +1224,7 @@ class TypeInfo_Struct : TypeInfo uint m_align; - override @property immutable(void)* rtInfo() const { return m_RTInfo; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return m_RTInfo; } version (X86_64) { @@ -3189,11 +3199,23 @@ void __ctfeWriteln(T...)(auto ref T values) { __ctfeWrite(values, "\n"); } * Create RTInfo for type T */ +template RTInfoImpl(size_t[] pointers) +{ + immutable size_t[pointers.length] data = pointers[]; + immutable RTInfoImpl = data.ptr; +} + template RTInfo(T) { - enum RTInfo = null; + enum RTInfo = RTInfoImpl!(__traits(getPointerBitmap, T)); } +/** +* shortcuts for the precise GC, also generated by the compiler +* used instead of the actual pointer bitmap +*/ +enum void* rtinfoNoPointers = null; +enum void* rtinfoHasPointers = cast(void*)1; // Helper functions diff --git a/src/rt/lifetime.d b/src/rt/lifetime.d index d28edf8c58..c5fb9e2548 100644 --- a/src/rt/lifetime.d +++ b/src/rt/lifetime.d @@ -415,7 +415,7 @@ BlkInfo __arrayAlloc(size_t arrsize, const TypeInfo ti, const TypeInfo tinext) n uint attr = (!(tinext.flags & 1) ? BlkAttr.NO_SCAN : 0) | BlkAttr.APPENDABLE; if (typeInfoSize) attr |= BlkAttr.STRUCTFINAL | BlkAttr.FINALIZE; - return GC.qalloc(arrsize + padsize, attr, ti); + return gc_qalloc_emplace(arrsize + padsize, attr, ti); } BlkInfo __arrayAlloc(size_t arrsize, ref BlkInfo info, const TypeInfo ti, const TypeInfo tinext) @@ -423,7 +423,7 @@ BlkInfo __arrayAlloc(size_t arrsize, ref BlkInfo info, const TypeInfo ti, const if (!info.base) return __arrayAlloc(arrsize, ti, tinext); - return GC.qalloc(arrsize + __arrayPad(arrsize, tinext), info.attr, ti); + return gc_qalloc_emplace(arrsize + __arrayPad(arrsize, tinext), info.attr, ti); } /** @@ -701,6 +701,54 @@ void __doPostblit(void *ptr, size_t len, const TypeInfo ti) } } +immutable bool gc_precise; + +shared static this() +{ + import gc.gc; + gc_precise = GC.config.precise; +} + +BlkInfo gc_qalloc_emplace(size_t sz, uint ba, const TypeInfo ti) nothrow pure +{ + if(gc_precise && !(ba & BlkAttr.NO_SCAN)) + { + // an array of classes is in fact an array of pointers + const(TypeInfo) tinext = ti.next.classinfo.name == "TypeInfo_Class" ? typeid(void*) : ti.next; + + if( sz <= PAGESIZE / 2 ) + return GC.qalloc(sz, ba | BlkAttr.REP_RTINFO, tinext); + + // for large arrays, we have to emplace the type info pointer bitmap at offset LARGEPAD + BlkInfo info = GC.qalloc(sz, ba | BlkAttr.NO_RTINFO, tinext); + if(info.base) + { + void* arr = __arrayStart(info); + GC.emplace(arr, info.base + info.size - arr, tinext); + } + return info; + } + else + return GC.qalloc(sz, ba, ti); +} + +BlkInfo gc_qalloc_emplace(size_t sz, const TypeInfo ti) nothrow pure +{ + return gc_qalloc_emplace(sz, !(ti.next.flags & 1) ? BlkAttr.NO_SCAN | BlkAttr.APPENDABLE : BlkAttr.APPENDABLE, ti); +} + +size_t gc_extend_emplace(void* p, size_t mx, size_t sz, size_t oldsz, const TypeInfo ti) +{ + size_t newsz = GC.extend(p, mx, sz, ti); + if(gc_precise && newsz >= PAGESIZE) + { + // an array of classes is in fact an array of pointers + const(TypeInfo) tinext = ti.next.classinfo.name == "TypeInfo_Class" ? typeid(void*) : ti.next; + void* arr = p + LARGEPREFIX; + GC.emplace(arr, newsz - LARGEPAD, tinext); + } + return newsz; +} /** * set the array capacity. If the array capacity isn't currently large enough @@ -1497,7 +1545,7 @@ body { // not enough space, try extending auto extendsize = newsize + offset + LARGEPAD - info.size; - auto u = GC.extend(info.base, extendsize, extendsize); + auto u = gc_extend_emplace(info.base, extendsize, extendsize, info.size, ti); if(u) { // extend worked, now try setting the length @@ -1682,7 +1730,7 @@ body { // not enough space, try extending auto extendsize = newsize + offset + LARGEPAD - info.size; - auto u = GC.extend(info.base, extendsize, extendsize); + auto u = gc_extend_emplace(info.base, extendsize, extendsize, info.size, ti); if(u) { // extend worked, now try setting the length @@ -1923,7 +1971,7 @@ byte[] _d_arrayappendcTX(const TypeInfo ti, ref byte[] px, size_t n) { // not enough space, try extending auto extendoffset = offset + LARGEPAD - info.size; - auto u = GC.extend(info.base, newsize + extendoffset, newcap + extendoffset); + auto u = gc_extend_emplace(info.base, newsize + extendoffset, newcap + extendoffset, info.size, ti); if(u) { // extend worked, now try setting the length diff --git a/src/rt/typeinfo/ti_byte.d b/src/rt/typeinfo/ti_byte.d index 6b0f310851..8638f151bd 100644 --- a/src/rt/typeinfo/ti_byte.d +++ b/src/rt/typeinfo/ti_byte.d @@ -57,4 +57,6 @@ class TypeInfo_g : TypeInfo *cast(byte *)p1 = *cast(byte *)p2; *cast(byte *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_cdouble.d b/src/rt/typeinfo/ti_cdouble.d index 15bec35251..ad6b7d7f4f 100644 --- a/src/rt/typeinfo/ti_cdouble.d +++ b/src/rt/typeinfo/ti_cdouble.d @@ -71,4 +71,6 @@ class TypeInfo_r : TypeInfo arg2 = typeid(double); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_cfloat.d b/src/rt/typeinfo/ti_cfloat.d index 3b82f04d0d..489e4b0945 100644 --- a/src/rt/typeinfo/ti_cfloat.d +++ b/src/rt/typeinfo/ti_cfloat.d @@ -70,4 +70,6 @@ class TypeInfo_q : TypeInfo arg1 = typeid(double); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_char.d b/src/rt/typeinfo/ti_char.d index ba3841792c..23fcd3273d 100644 --- a/src/rt/typeinfo/ti_char.d +++ b/src/rt/typeinfo/ti_char.d @@ -59,4 +59,6 @@ class TypeInfo_a : TypeInfo return (&c)[0 .. 1]; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_creal.d b/src/rt/typeinfo/ti_creal.d index b77429cf89..01f53cec81 100644 --- a/src/rt/typeinfo/ti_creal.d +++ b/src/rt/typeinfo/ti_creal.d @@ -71,4 +71,6 @@ class TypeInfo_c : TypeInfo arg2 = typeid(real); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_dchar.d b/src/rt/typeinfo/ti_dchar.d index 2d4354c18a..534eba00d5 100644 --- a/src/rt/typeinfo/ti_dchar.d +++ b/src/rt/typeinfo/ti_dchar.d @@ -59,4 +59,6 @@ class TypeInfo_w : TypeInfo return (&c)[0 .. 1]; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_delegate.d b/src/rt/typeinfo/ti_delegate.d index fa6b0a21e8..617ff221e1 100644 --- a/src/rt/typeinfo/ti_delegate.d +++ b/src/rt/typeinfo/ti_delegate.d @@ -61,4 +61,6 @@ class TypeInfo_D : TypeInfo { return 1; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return RTInfo!(dg); } } diff --git a/src/rt/typeinfo/ti_double.d b/src/rt/typeinfo/ti_double.d index 8aa281f11b..e524bc7c12 100644 --- a/src/rt/typeinfo/ti_double.d +++ b/src/rt/typeinfo/ti_double.d @@ -65,6 +65,8 @@ class TypeInfo_d : TypeInfo return F.alignof; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } + version (Windows) { } diff --git a/src/rt/typeinfo/ti_float.d b/src/rt/typeinfo/ti_float.d index 60837d1c96..954bd9ab9e 100644 --- a/src/rt/typeinfo/ti_float.d +++ b/src/rt/typeinfo/ti_float.d @@ -60,6 +60,8 @@ class TypeInfo_f : TypeInfo return (&r)[0 .. 1]; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } + version (Windows) { } diff --git a/src/rt/typeinfo/ti_int.d b/src/rt/typeinfo/ti_int.d index b0418c06fc..d0026c2164 100644 --- a/src/rt/typeinfo/ti_int.d +++ b/src/rt/typeinfo/ti_int.d @@ -61,4 +61,6 @@ class TypeInfo_i : TypeInfo *cast(int *)p1 = *cast(int *)p2; *cast(int *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_long.d b/src/rt/typeinfo/ti_long.d index d054022b26..179c757afa 100644 --- a/src/rt/typeinfo/ti_long.d +++ b/src/rt/typeinfo/ti_long.d @@ -68,4 +68,6 @@ class TypeInfo_l : TypeInfo { return long.alignof; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ptr.d b/src/rt/typeinfo/ti_ptr.d index a23511a6fd..1dc5ead949 100644 --- a/src/rt/typeinfo/ti_ptr.d +++ b/src/rt/typeinfo/ti_ptr.d @@ -61,4 +61,6 @@ class TypeInfo_P : TypeInfo } override @property uint flags() nothrow pure const { return 1; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoHasPointers; } } diff --git a/src/rt/typeinfo/ti_real.d b/src/rt/typeinfo/ti_real.d index 77ae126511..7cdfb3c660 100644 --- a/src/rt/typeinfo/ti_real.d +++ b/src/rt/typeinfo/ti_real.d @@ -64,4 +64,6 @@ class TypeInfo_e : TypeInfo { return F.alignof; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_short.d b/src/rt/typeinfo/ti_short.d index f636fc3af3..f14c8b1675 100644 --- a/src/rt/typeinfo/ti_short.d +++ b/src/rt/typeinfo/ti_short.d @@ -57,4 +57,6 @@ class TypeInfo_s : TypeInfo *cast(short *)p1 = *cast(short *)p2; *cast(short *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ubyte.d b/src/rt/typeinfo/ti_ubyte.d index 97b902ab07..fcfcc6e22d 100644 --- a/src/rt/typeinfo/ti_ubyte.d +++ b/src/rt/typeinfo/ti_ubyte.d @@ -57,6 +57,8 @@ class TypeInfo_h : TypeInfo *cast(ubyte *)p1 = *cast(ubyte *)p2; *cast(ubyte *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } class TypeInfo_b : TypeInfo_h diff --git a/src/rt/typeinfo/ti_uint.d b/src/rt/typeinfo/ti_uint.d index 6cd523ba2a..4c4fac18c0 100644 --- a/src/rt/typeinfo/ti_uint.d +++ b/src/rt/typeinfo/ti_uint.d @@ -61,4 +61,6 @@ class TypeInfo_k : TypeInfo *cast(uint *)p1 = *cast(uint *)p2; *cast(uint *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ulong.d b/src/rt/typeinfo/ti_ulong.d index af645a1a42..b05f759177 100644 --- a/src/rt/typeinfo/ti_ulong.d +++ b/src/rt/typeinfo/ti_ulong.d @@ -68,4 +68,6 @@ class TypeInfo_m : TypeInfo { return ulong.alignof; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ushort.d b/src/rt/typeinfo/ti_ushort.d index ae24656432..00c935ddf8 100644 --- a/src/rt/typeinfo/ti_ushort.d +++ b/src/rt/typeinfo/ti_ushort.d @@ -57,4 +57,6 @@ class TypeInfo_t : TypeInfo *cast(ushort *)p1 = *cast(ushort *)p2; *cast(ushort *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_wchar.d b/src/rt/typeinfo/ti_wchar.d index 3e2fba9b3b..a1166a7e21 100644 --- a/src/rt/typeinfo/ti_wchar.d +++ b/src/rt/typeinfo/ti_wchar.d @@ -59,4 +59,6 @@ class TypeInfo_u : TypeInfo return (&c)[0 .. 1]; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } From 7d76528c8eee9c707bb72d2f28c50f10a003f363 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 19 Apr 2015 14:26:22 +0200 Subject: [PATCH 2/9] use typeid pointers instead off TypeInfo classname to detect classes --- src/rt/lifetime.d | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/rt/lifetime.d b/src/rt/lifetime.d index c5fb9e2548..fedb8d2868 100644 --- a/src/rt/lifetime.d +++ b/src/rt/lifetime.d @@ -415,7 +415,7 @@ BlkInfo __arrayAlloc(size_t arrsize, const TypeInfo ti, const TypeInfo tinext) n uint attr = (!(tinext.flags & 1) ? BlkAttr.NO_SCAN : 0) | BlkAttr.APPENDABLE; if (typeInfoSize) attr |= BlkAttr.STRUCTFINAL | BlkAttr.FINALIZE; - return gc_qalloc_emplace(arrsize + padsize, attr, ti); + return gc_qalloc_emplace(arrsize + padsize, attr, ti, tinext); } BlkInfo __arrayAlloc(size_t arrsize, ref BlkInfo info, const TypeInfo ti, const TypeInfo tinext) @@ -423,7 +423,7 @@ BlkInfo __arrayAlloc(size_t arrsize, ref BlkInfo info, const TypeInfo ti, const if (!info.base) return __arrayAlloc(arrsize, ti, tinext); - return gc_qalloc_emplace(arrsize + __arrayPad(arrsize, tinext), info.attr, ti); + return gc_qalloc_emplace(arrsize + __arrayPad(arrsize, tinext), info.attr, ti, tinext); } /** @@ -709,22 +709,22 @@ shared static this() gc_precise = GC.config.precise; } -BlkInfo gc_qalloc_emplace(size_t sz, uint ba, const TypeInfo ti) nothrow pure +BlkInfo gc_qalloc_emplace(size_t sz, uint ba, const TypeInfo ti, const TypeInfo tinext) nothrow pure { if(gc_precise && !(ba & BlkAttr.NO_SCAN)) { // an array of classes is in fact an array of pointers - const(TypeInfo) tinext = ti.next.classinfo.name == "TypeInfo_Class" ? typeid(void*) : ti.next; + const(TypeInfo) tielem = typeid(tinext) is typeid(TypeInfo_Class) ? typeid(void*) : tinext; if( sz <= PAGESIZE / 2 ) - return GC.qalloc(sz, ba | BlkAttr.REP_RTINFO, tinext); + return GC.qalloc(sz, ba | BlkAttr.REP_RTINFO, tielem); // for large arrays, we have to emplace the type info pointer bitmap at offset LARGEPAD - BlkInfo info = GC.qalloc(sz, ba | BlkAttr.NO_RTINFO, tinext); + BlkInfo info = GC.qalloc(sz, ba | BlkAttr.NO_RTINFO, tielem); if(info.base) { void* arr = __arrayStart(info); - GC.emplace(arr, info.base + info.size - arr, tinext); + GC.emplace(arr, info.base + info.size - arr, tielem); } return info; } @@ -732,20 +732,21 @@ BlkInfo gc_qalloc_emplace(size_t sz, uint ba, const TypeInfo ti) nothrow pure return GC.qalloc(sz, ba, ti); } -BlkInfo gc_qalloc_emplace(size_t sz, const TypeInfo ti) nothrow pure +BlkInfo gc_qalloc_emplace(size_t sz, const TypeInfo ti, const TypeInfo tinext) nothrow pure { - return gc_qalloc_emplace(sz, !(ti.next.flags & 1) ? BlkAttr.NO_SCAN | BlkAttr.APPENDABLE : BlkAttr.APPENDABLE, ti); + uint attr = !(tinext.flags & 1) ? BlkAttr.NO_SCAN | BlkAttr.APPENDABLE : BlkAttr.APPENDABLE; + return gc_qalloc_emplace(sz, attr, ti, tinext); } -size_t gc_extend_emplace(void* p, size_t mx, size_t sz, size_t oldsz, const TypeInfo ti) +size_t gc_extend_emplace(void* p, size_t mx, size_t sz, size_t oldsz, const TypeInfo ti, const TypeInfo tinext) { size_t newsz = GC.extend(p, mx, sz, ti); if(gc_precise && newsz >= PAGESIZE) { // an array of classes is in fact an array of pointers - const(TypeInfo) tinext = ti.next.classinfo.name == "TypeInfo_Class" ? typeid(void*) : ti.next; + const(TypeInfo) tielem = typeid(tinext) is typeid(TypeInfo_Class) ? typeid(void*) : tinext; void* arr = p + LARGEPREFIX; - GC.emplace(arr, newsz - LARGEPAD, tinext); + GC.emplace(arr, newsz - LARGEPAD, tielem); } return newsz; } @@ -1545,7 +1546,7 @@ body { // not enough space, try extending auto extendsize = newsize + offset + LARGEPAD - info.size; - auto u = gc_extend_emplace(info.base, extendsize, extendsize, info.size, ti); + auto u = gc_extend_emplace(info.base, extendsize, extendsize, info.size, ti, tinext); if(u) { // extend worked, now try setting the length @@ -1730,7 +1731,7 @@ body { // not enough space, try extending auto extendsize = newsize + offset + LARGEPAD - info.size; - auto u = gc_extend_emplace(info.base, extendsize, extendsize, info.size, ti); + auto u = gc_extend_emplace(info.base, extendsize, extendsize, info.size, ti, tinext); if(u) { // extend worked, now try setting the length @@ -1971,7 +1972,7 @@ byte[] _d_arrayappendcTX(const TypeInfo ti, ref byte[] px, size_t n) { // not enough space, try extending auto extendoffset = offset + LARGEPAD - info.size; - auto u = gc_extend_emplace(info.base, newsize + extendoffset, newcap + extendoffset, info.size, ti); + auto u = gc_extend_emplace(info.base, newsize + extendoffset, newcap + extendoffset, info.size, ti, tinext); if(u) { // extend worked, now try setting the length From 66c8dc187af289d818250af64d8c07e68810db0a Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 19 Apr 2015 20:32:06 +0200 Subject: [PATCH 3/9] move attr evaluation into setPointerBitmap --- src/gc/gc.d | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/gc/gc.d b/src/gc/gc.d index 77b2fe5e05..437c17393e 100644 --- a/src/gc/gc.d +++ b/src/gc/gc.d @@ -675,8 +675,8 @@ struct GC auto newsz = (size + PAGESIZE - 1) / PAGESIZE; if (newsz == psz) { - if(config.precise && !(bits & BlkAttr.NO_RTINFO)) - pool.setPointerBitmap(p, size, newsz * PAGESIZE, ti, (bits & BlkAttr.REP_RTINFO) != 0); + if(config.precise) + pool.setPointerBitmap(p, size, newsz * PAGESIZE, bits, ti); alloc_size = psize; return p; } @@ -712,8 +712,8 @@ struct GC pool.setBits(biti, bits); } alloc_size = newsz * PAGESIZE; - if(config.precise && !(bits & BlkAttr.NO_RTINFO)) - pool.setPointerBitmap(p, size, newsz * PAGESIZE, ti, (bits & BlkAttr.REP_RTINFO) != 0); + if(config.precise) + pool.setPointerBitmap(p, size, newsz * PAGESIZE, bits, ti); return p; } @@ -746,8 +746,8 @@ struct GC else { alloc_size = psize; - if(config.precise && !(bits & BlkAttr.NO_RTINFO)) - pool.setPointerBitmap(p, size, psize, ti, (bits & BlkAttr.REP_RTINFO) != 0); + if(config.precise) + pool.setPointerBitmap(p, size, psize, bits, ti); } } } @@ -1870,10 +1870,9 @@ struct Gcx //debug(PRINTF) printf("\tmalloc => %p\n", p); debug (MEMSTOMP) memset(p, 0xF0, alloc_size); - if (GC.config.precise && !(bits & (BlkAttr.NO_SCAN | BlkAttr.NO_RTINFO))) + if (GC.config.precise) { - bool repeat = (bits & BlkAttr.REP_RTINFO) != 0; - pool.setPointerBitmap(sentinel_add(p), size, alloc_size, ti, repeat); + pool.setPointerBitmap(sentinel_add(p), size, alloc_size, bits, ti); } return p; @@ -1957,8 +1956,8 @@ struct Gcx if (bits) pool.setBits(pn, bits); - if (GC.config.precise && !(bits & (BlkAttr.NO_SCAN | BlkAttr.NO_RTINFO))) - pool.setPointerBitmap(sentinel_add(p), size, alloc_size, ti, (bits & BlkAttr.REP_RTINFO) != 0); + if (GC.config.precise) + pool.setPointerBitmap(sentinel_add(p), size, alloc_size, bits, ti); return p; } @@ -3117,6 +3116,12 @@ struct Pool } } + void setPointerBitmap(void* p, size_t s, size_t allocSize, uint attr, const TypeInfo ti) nothrow + { + if (!(attr & (BlkAttr.NO_SCAN | BlkAttr.NO_RTINFO))) + setPointerBitmap(p, s, allocSize, ti, (attr & BlkAttr.REP_RTINFO) != 0); + } + void setPointerBitmap(void* p, size_t s, size_t allocSize, const TypeInfo ti, bool repeat) nothrow { size_t offset = p - baseAddr; From 0c5dabf0b1b17283b4c721df2a461ec7d8c4d52e Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Thu, 23 Apr 2015 08:36:55 +0200 Subject: [PATCH 4/9] add @nogc to currTicks, tweak SENTINEL for precise GC --- src/gc/gc.d | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/gc/gc.d b/src/gc/gc.d index 437c17393e..bc763f3be6 100644 --- a/src/gc/gc.d +++ b/src/gc/gc.d @@ -1142,7 +1142,11 @@ struct GC BlkInfo info = gcx.getInfo(p, &pool); if(!info.base) return false; - size_t allocSize = info.size - (p - info.base); + + debug(SENTINEL) + size_t allocSize = len; + else + size_t allocSize = info.size - (p - info.base); pool.setPointerBitmap(p, len, allocSize, ti, true); return true; } @@ -1872,7 +1876,10 @@ struct Gcx if (GC.config.precise) { - pool.setPointerBitmap(sentinel_add(p), size, alloc_size, bits, ti); + debug(SENTINEL) + pool.setPointerBitmap(sentinel_add(p), size - SENTINEL_EXTRA, size - SENTINEL_EXTRA, bits, ti); + else + pool.setPointerBitmap(p, size, alloc_size, bits, ti); } return p; @@ -1957,7 +1964,12 @@ struct Gcx if (bits) pool.setBits(pn, bits); if (GC.config.precise) - pool.setPointerBitmap(sentinel_add(p), size, alloc_size, bits, ti); + { + debug(SENTINEL) + pool.setPointerBitmap(sentinel_add(p), size - SENTINEL_EXTRA, size - SENTINEL_EXTRA, bits, ti); + else + pool.setPointerBitmap(p, size, alloc_size, bits, ti); + } return p; } From 3ce92efb1690505ab65f88a1a9ccf54883224b76 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Fri, 24 Apr 2015 20:31:52 +0200 Subject: [PATCH 5/9] adjust new AA for precise scanning fix copyRangeRepeated optimization --- src/gc/bits.d | 13 +++++++--- src/rt/aaA.d | 70 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/gc/bits.d b/src/gc/bits.d index ca8acf1584..65f9048082 100644 --- a/src/gc/bits.d +++ b/src/gc/bits.d @@ -196,8 +196,8 @@ struct GCBits initbits -= sourcelen; } copyRange(target, initbits, source); - target += sourcelen; - destlen -= sourcelen; + target += initbits; + destlen -= initbits; assert((target & BITS_MASK) == 0); size_t tpos = target >> BITS_SHIFT; @@ -208,8 +208,11 @@ struct GCBits tpos++; } - wordtype mask = (BITS_2 << destlen) - 1; - data[tpos] = (data[tpos] & ~mask) | (data[tpos - repwords] & mask); + if (destlen > 0) + { + wordtype mask = (BITS_1 << destlen) - 1; + data[tpos] = (data[tpos] & ~mask) | (data[tpos - repwords] & mask); + } return; } } @@ -364,6 +367,8 @@ struct GCBits testCopyRange(20, 5, 10); // repeating small range spanning two words testCopyRange(40, 21, 7); // repeating medium range testCopyRange(73, 2 * BITS_PER_WORD + 15, 5); // repeating multi-word range + + testCopyRange(2, 3, 166); // failed with assert } void zero() nothrow diff --git a/src/rt/aaA.d b/src/rt/aaA.d index cf8943e1e2..f761567799 100644 --- a/src/rt/aaA.d +++ b/src/rt/aaA.d @@ -250,12 +250,10 @@ TypeInfo_Struct fakeEntryTI(const TypeInfo keyti, const TypeInfo valti) auto kti = unqualify(keyti); auto vti = unqualify(valti); - if (!hasDtor(kti) && !hasDtor(vti)) - return null; // save kti and vti after type info for struct enum sizeti = __traits(classInstanceSize, TypeInfo_Struct); - void* p = GC.malloc(sizeti + 2 * (void*).sizeof); + void* p = GC.malloc(sizeti + 4 * (void*).sizeof); import core.stdc.string : memcpy; memcpy(p, typeid(TypeInfo_Struct).initializer().ptr, sizeti); @@ -268,23 +266,79 @@ TypeInfo_Struct fakeEntryTI(const TypeInfo keyti, const TypeInfo valti) static immutable tiName = __MODULE__ ~ ".Entry!(...)"; ti.name = tiName; + auto rtinfoData = cast(size_t[2]*) (extra + 2); + ti.m_RTInfo = rtinfoEntry(keyti, valti, *rtinfoData); + // we don't expect the Entry objects to be used outside of this module, so we have control // over the non-usage of the callback methods and other entries and can keep these null // xtoHash, xopEquals, xopCmp, xtoString and xpostblit - ti.m_RTInfo = null; immutable entrySize = talign(kti.tsize, vti.talign) + vti.tsize; ti.m_init = (cast(ubyte*) null)[0 .. entrySize]; // init length, but not ptr - // xdtor needs to be built from the dtors of key and value for the GC - ti.xdtorti = &entryDtor; + if (hasDtor(kti) || hasDtor(vti)) + { + // xdtor needs to be built from the dtors of key and value for the GC + ti.xdtorti = &entryDtor; + ti.m_flags = TypeInfo_Struct.StructFlags.hasPointers | TypeInfo_Struct.StructFlags.isDynamicType; + } + else + ti.m_flags = TypeInfo_Struct.StructFlags.hasPointers; - ti.m_flags = TypeInfo_Struct.StructFlags.isDynamicType; - ti.m_flags |= (keyti.flags | valti.flags) & TypeInfo_Struct.StructFlags.hasPointers; ti.m_align = cast(uint) max(kti.talign, vti.talign); return ti; } +// build type info with appropriate RTInfo at runtime +immutable(void)* rtinfoEntry(const TypeInfo keyti, const TypeInfo valti, ref size_t[2] rtinfoData) +{ + static bool isNoClass(const TypeInfo ti) { return ti && typeid(ti) !is typeid(TypeInfo_Class); } + + immutable(size_t)* keyinfo = cast(immutable(size_t)*) (isNoClass(keyti) ? keyti.rtInfo : rtinfoHasPointers); + immutable(size_t)* valinfo = cast(immutable(size_t)*) (isNoClass(valti) ? valti.rtInfo : rtinfoHasPointers); + + enum maxSupportedSize = 8 * (void*).sizeof * (void*).sizeof; + + size_t rtinfosize = 0; + size_t valuesize = valti.tsize(); + + size_t valbits; + if (valinfo is rtinfoNoPointers) + valbits = 0; + else if (rtinfosize + valuesize > maxSupportedSize) + return rtinfoHasPointers; + else if (valinfo is rtinfoHasPointers) + valbits = (1 << (valuesize / (void*).sizeof)) - 1; + else + valbits = valinfo[1]; + + if (valbits != 0) + rtinfosize += valuesize; + + size_t keybits; + size_t keysize = keyti.tsize; + if (keyinfo is rtinfoNoPointers) + keybits = 0; + else if (rtinfosize + keysize > maxSupportedSize) + return rtinfoHasPointers; + else if (keyinfo is rtinfoHasPointers) + keybits = (1 << (keysize / (void*).sizeof)) - 1; + else + keybits = keyinfo[1]; + + if (valbits == 0 && keybits == 0) + return rtinfoNoPointers; + + if (valbits != 0 || keybits != 0) + rtinfosize += keysize; + + rtinfoData[0] = rtinfosize; + + size_t valshift = (keysize + (void*).sizeof - 1) / (void*).sizeof; + rtinfoData[1] = keybits | (valbits << valshift); + + return cast(immutable(void)*) rtinfoData.ptr; +} //============================================================================== // Helper functions //------------------------------------------------------------------------------ From 90336c11ae734254a5146301421b9363c0102b54 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sat, 18 Jul 2015 15:04:58 +0200 Subject: [PATCH 6/9] inline optimizations --- src/gc/bits.d | 158 +++++++++++++++++++++++++++++++------------------- src/gc/gc.d | 16 +++-- 2 files changed, 108 insertions(+), 66 deletions(-) diff --git a/src/gc/bits.d b/src/gc/bits.d index 65f9048082..158015f879 100644 --- a/src/gc/bits.d +++ b/src/gc/bits.d @@ -55,6 +55,7 @@ struct GCBits onOutOfMemoryError(); } + pragma(inline,true) wordtype test(size_t i) const nothrow in { @@ -65,6 +66,7 @@ struct GCBits return core.bitop.bt(data, i); } + pragma(inline,true) int set(size_t i) nothrow in { @@ -75,6 +77,7 @@ struct GCBits return core.bitop.bts(data, i); } + pragma(inline,true) int clear(size_t i) nothrow in { @@ -94,8 +97,36 @@ struct GCBits size_t lastOff = last & BITS_MASK; } + // extract loops to allow inlining the rest + void clearWords(size_t firstWord, size_t lastWord) nothrow + { + for(size_t w = firstWord; w < lastWord; w++) + data[w] = 0; + } + + void setWords(size_t firstWord, size_t lastWord) nothrow + { + for(size_t w = firstWord; w < lastWord; w++) + data[w] = ~0; + } + + void copyWords(size_t firstWord, size_t lastWord, const(wordtype)* source) nothrow + { + for(size_t w = firstWord; w < lastWord; w++) + data[w] = source[w - firstWord]; + } + + void copyWordsShifted(size_t firstWord, size_t cntWords, size_t firstOff, const(wordtype)* source) nothrow + { + wordtype mask = ~BITS_0 << firstOff; + data[firstWord] = (data[firstWord] & ~mask) | (source[0] << firstOff); + for(size_t w = 1; w < cntWords; w++) + data[firstWord + w] = (source[w - 1] >> (BITS_PER_WORD - firstOff)) | (source[w] << firstOff); + } + // target = the biti to start the copy to // destlen = the number of bits to copy from source + pragma(inline,true) void copyRange(size_t target, size_t len, const(wordtype)* source) nothrow { version(bitwise) @@ -108,36 +139,36 @@ struct GCBits } else { - if(len == 0) - return; + if(len > 0) + copyRangeZ(target, len, source); + } + } - mixin RangeVars!(); + pragma(inline,true) + void copyRangeZ(size_t target, size_t len, const(wordtype)* source) nothrow + { + mixin RangeVars!(); - if(firstWord == lastWord) - { - wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; - data[firstWord] = (data[firstWord] & ~mask) | ((source[0] << firstOff) & mask); - } - else if(firstOff == 0) - { - for(size_t w = firstWord; w < lastWord; w++) - data[w] = source[w - firstWord]; + if(firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] = (data[firstWord] & ~mask) | ((source[0] << firstOff) & mask); + } + else if(firstOff == 0) + { + copyWords(firstWord, lastWord, source); - wordtype mask = (BITS_2 << lastOff) - 1; - data[lastWord] = (data[lastWord] & ~mask) | (source[lastWord - firstWord] & mask); - } - else - { - size_t cntWords = lastWord - firstWord; - wordtype mask = ~BITS_0 << firstOff; - data[firstWord] = (data[firstWord] & ~mask) | (source[0] << firstOff); - for(size_t w = 1; w < cntWords; w++) - data[firstWord + w] = (source[w - 1] >> (BITS_PER_WORD - firstOff)) | (source[w] << firstOff); - - wordtype src = (source[cntWords - 1] >> (BITS_PER_WORD - firstOff)) | (source[cntWords] << firstOff); - mask = (BITS_2 << lastOff) - 1; - data[lastWord] = (data[lastWord] & ~mask) | (src & mask); - } + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] = (data[lastWord] & ~mask) | (source[lastWord - firstWord] & mask); + } + else + { + size_t cntWords = lastWord - firstWord; + copyWordsShifted(firstWord, cntWords, firstOff, source); + + wordtype src = (source[cntWords - 1] >> (BITS_PER_WORD - firstOff)) | (source[cntWords] << firstOff); + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] = (data[lastWord] & ~mask) | (src & mask); } } @@ -227,6 +258,7 @@ struct GCBits } } + pragma(inline,true) void setRange(size_t target, size_t len) nothrow { version(bitwise) @@ -236,27 +268,31 @@ struct GCBits } else { - if(len == 0) - return; + if(len > 0) + setRangeZ(target, len); + } + } - mixin RangeVars!(); + pragma(inline,true) + void setRangeZ(size_t target, size_t len) nothrow + { + mixin RangeVars!(); - if(firstWord == lastWord) - { - wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; - data[firstWord] |= mask; - } - else - { - data[firstWord] |= ~BITS_0 << firstOff; - for(size_t w = firstWord + 1; w < lastWord; w++) - data[w] = ~0; - wordtype mask = (BITS_2 << lastOff) - 1; - data[lastWord] |= mask; - } + if(firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] |= mask; + } + else + { + data[firstWord] |= ~BITS_0 << firstOff; + setWords(firstWord + 1, lastWord); + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] |= mask; } } + pragma(inline,true) void clrRange(size_t target, size_t len) nothrow { version(bitwise) @@ -266,24 +302,26 @@ struct GCBits } else { - if(len == 0) - return; - - mixin RangeVars!(); + if(len > 0) + clrRangeZ(target, len); + } + } - if(firstWord == lastWord) - { - wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; - data[firstWord] &= ~mask; - } - else - { - data[firstWord] &= ~(~BITS_0 << firstOff); - for(size_t w = firstWord + 1; w < lastWord; w++) - data[w] = 0; - wordtype mask = (BITS_2 << lastOff) - 1; - data[lastWord] &= ~mask; - } + pragma(inline,true) + void clrRangeZ(size_t target, size_t len) nothrow + { + mixin RangeVars!(); + if(firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] &= ~mask; + } + else + { + data[firstWord] &= ~(~BITS_0 << firstOff); + clearWords(firstWord + 1, lastWord); + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] &= ~mask; } } diff --git a/src/gc/gc.d b/src/gc/gc.d index bc763f3be6..74340bb9f0 100644 --- a/src/gc/gc.d +++ b/src/gc/gc.d @@ -825,7 +825,7 @@ struct GC { auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; if(!pool.noscan.test(biti)) - pool.setPointerBitmap(p, psize + sz * PAGESIZE, psize + sz * PAGESIZE, ti, true); + pool.setPointerBitmap(p, psize + sz * PAGESIZE, psize + sz * PAGESIZE, ti, BlkAttr.REP_RTINFO); } return (psz + sz) * PAGESIZE; @@ -1147,7 +1147,7 @@ struct GC size_t allocSize = len; else size_t allocSize = info.size - (p - info.base); - pool.setPointerBitmap(p, len, allocSize, ti, true); + pool.setPointerBitmap(p, len, allocSize, ti, BlkAttr.REP_RTINFO); return true; } @@ -3128,13 +3128,15 @@ struct Pool } } + pragma(inline,true) void setPointerBitmap(void* p, size_t s, size_t allocSize, uint attr, const TypeInfo ti) nothrow { if (!(attr & (BlkAttr.NO_SCAN | BlkAttr.NO_RTINFO))) - setPointerBitmap(p, s, allocSize, ti, (attr & BlkAttr.REP_RTINFO) != 0); + setPointerBitmap(p, s, allocSize, ti, attr); } - void setPointerBitmap(void* p, size_t s, size_t allocSize, const TypeInfo ti, bool repeat) nothrow + pragma(inline,false) + void setPointerBitmap(void* p, size_t s, size_t allocSize, const TypeInfo ti, uint attr) nothrow { size_t offset = p - baseAddr; //debug(PRINTF) printGCBits(&pool.is_pointer); @@ -3144,7 +3146,7 @@ struct Pool if (ti) { - if (repeat) + if (attr & BlkAttr.REP_RTINFO) s = allocSize; auto rtInfo = cast(const(size_t)*)ti.rtInfo(); @@ -3166,7 +3168,7 @@ struct Pool size_t element_size = * bitmap; bitmap++; size_t tocopy; - if(repeat) + if(attr & BlkAttr.REP_RTINFO) { tocopy = s/(void*).sizeof; is_pointer.copyRangeRepeating(offset/(void*).sizeof, tocopy, bitmap, element_size/(void*).sizeof); @@ -3199,6 +3201,8 @@ struct Pool } else { + // limit pointers to actual size of allocation? might fail for arrays that append + // without notifying the GC s = allocSize; debug(PRINTF) printf("Allocating a block without TypeInfo\n"); From ab74b1563a4dfda50ea128c092004bca9bd4d929 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 8 Nov 2015 08:34:16 +0100 Subject: [PATCH 7/9] remove non-working pragma(inline) --- src/gc/bits.d | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gc/bits.d b/src/gc/bits.d index 158015f879..79ae2aec0c 100644 --- a/src/gc/bits.d +++ b/src/gc/bits.d @@ -144,7 +144,7 @@ struct GCBits } } - pragma(inline,true) + //pragma(inline,true) void copyRangeZ(size_t target, size_t len, const(wordtype)* source) nothrow { mixin RangeVars!(); @@ -258,7 +258,7 @@ struct GCBits } } - pragma(inline,true) + //pragma(inline,true) void setRange(size_t target, size_t len) nothrow { version(bitwise) @@ -273,7 +273,7 @@ struct GCBits } } - pragma(inline,true) + //pragma(inline,true) void setRangeZ(size_t target, size_t len) nothrow { mixin RangeVars!(); @@ -292,7 +292,7 @@ struct GCBits } } - pragma(inline,true) + //pragma(inline,true) void clrRange(size_t target, size_t len) nothrow { version(bitwise) @@ -307,7 +307,7 @@ struct GCBits } } - pragma(inline,true) + //pragma(inline,true) void clrRangeZ(size_t target, size_t len) nothrow { mixin RangeVars!(); From 320b50bbf4e1850aabfdb882017eba7000b07885 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 8 Nov 2015 08:36:20 +0100 Subject: [PATCH 8/9] tweak option help output --- src/gc/config.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gc/config.d b/src/gc/config.d index 89f8ef45c6..52cfc21d2f 100644 --- a/src/gc/config.d +++ b/src/gc/config.d @@ -61,7 +61,7 @@ struct Config string s = "GC options are specified as white space separated assignments: disable:0|1 - start disabled (%d) profile:0|1|2 - enable profiling with summary when terminating program (%d) - precise:0|1 - enable precise scanning (not implemented yet) + precise:0|1 - enable precise heap scanning (%d) concurrent:0|1 - enable concurrent collection (not implemented yet) initReserve:N - initial memory to reserve in MB (%lld) @@ -70,7 +70,7 @@ struct Config incPoolSize:N - pool size increment MB (%lld) heapSizeFactor:N - targeted heap size to used memory ratio (%g) "; - printf(s.ptr, disable, profile, cast(long)initReserve, cast(long)minPoolSize, + printf(s.ptr, disable, profile, precise, cast(long)initReserve, cast(long)minPoolSize, cast(long)maxPoolSize, cast(long)incPoolSize, heapSizeFactor); } From a1c94c96d78a0224eb7f8122211168249b52afbb Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Wed, 23 Mar 2016 23:31:08 +0100 Subject: [PATCH 9/9] fix after rebase --- src/gc/bits.d | 2 +- src/gc/gc.d | 20 +++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/gc/bits.d b/src/gc/bits.d index 79ae2aec0c..42252e8e0e 100644 --- a/src/gc/bits.d +++ b/src/gc/bits.d @@ -66,7 +66,7 @@ struct GCBits return core.bitop.bt(data, i); } - pragma(inline,true) + //pragma(inline,true) int set(size_t i) nothrow in { diff --git a/src/gc/gc.d b/src/gc/gc.d index 74340bb9f0..15cc4ed989 100644 --- a/src/gc/gc.d +++ b/src/gc/gc.d @@ -1122,10 +1122,7 @@ struct GC if (!p) return false; - mixin doLock!(); - bool rc = emplaceNoSync(p, len, ti); - mixin doUnlock!(emplaceTime, numEmplaces); - return rc; + return runLocked!(emplaceNoSync, emplaceTime, numEmplaces)(p, len, ti); } @@ -2132,9 +2129,6 @@ struct Gcx size_t stackPos; ScanRange[FANOUT_LIMIT] stack = void; - auto minAddr = pooltable.minAddr; - auto maxAddr = pooltable.maxAddr; - static if (precise) Pool* p1pool = null; // always starting from a non-heap root else @@ -2195,12 +2189,12 @@ struct Gcx // We don't care abou setting pointsToBase correctly // because it's ignored for small object pools anyhow. auto offsetBase = offset & notbinsize[bin]; - biti = offsetBase >> pool.shiftBy; + biti = offsetBase >> pool.shiftBySmall; base = pool.baseAddr + offsetBase; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = Range(base, base + binsize[bin]); + stack[stackPos++] = ScanRange(base, base + binsize[bin], pool); if (stackPos == stack.length) break; } @@ -2209,7 +2203,7 @@ struct Gcx { auto offsetBase = offset & notbinsize[bin]; base = pool.baseAddr + offsetBase; - biti = offsetBase >> pool.shiftBy; + biti = offsetBase >> pool.shiftByLarge; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); @@ -2222,7 +2216,7 @@ struct Gcx continue; if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE); + stack[stackPos++] = ScanRange(base, base + pool.bPageOffsets[pn] * PAGESIZE, pool); if (stackPos == stack.length) break; } @@ -2231,14 +2225,14 @@ struct Gcx { pn -= pool.bPageOffsets[pn]; base = pool.baseAddr + (pn * PAGESIZE); - biti = pn * (PAGESIZE >> pool.shiftBy); + biti = pn * (PAGESIZE >> pool.shiftByLarge); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); if(pool.nointerior.nbits && pool.nointerior.test(biti)) continue; if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE); + stack[stackPos++] = ScanRange(base, base + pool.bPageOffsets[pn] * PAGESIZE, pool); if (stackPos == stack.length) break; }