diff --git a/mak/COPY b/mak/COPY index 1df2e86788..c74ad6d250 100644 --- a/mak/COPY +++ b/mak/COPY @@ -18,6 +18,7 @@ COPY=\ $(IMPDIR)\core\internal\abort.d \ $(IMPDIR)\core\internal\convert.d \ $(IMPDIR)\core\internal\hash.d \ + $(IMPDIR)\core\internal\spinlock.d \ $(IMPDIR)\core\internal\string.d \ $(IMPDIR)\core\internal\traits.d \ \ diff --git a/mak/SRCS b/mak/SRCS index 9a54c8c41b..65ccba63f5 100644 --- a/mak/SRCS +++ b/mak/SRCS @@ -19,6 +19,7 @@ SRCS=\ src\core\internal\abort.d \ src\core\internal\convert.d \ src\core\internal\hash.d \ + src\core\internal\spinlock.d \ src\core\internal\string.d \ src\core\internal\traits.d \ \ diff --git a/src/core/exception.d b/src/core/exception.d index 7c43d625be..701dff2c30 100644 --- a/src/core/exception.d +++ b/src/core/exception.d @@ -227,7 +227,13 @@ class OutOfMemoryError : Error override string toString() const @trusted { - return msg.length ? (cast()super).toString() : "Memory allocation failed"; + return msg.length ? (cast()this).superToString() : "Memory allocation failed"; + } + + // kludge to call non-const super.toString + private string superToString() @trusted + { + return super.toString(); } } @@ -239,6 +245,7 @@ unittest assert(oome.line == __LINE__ - 2); assert(oome.next is null); assert(oome.msg == "Memory allocation failed"); + assert(oome.toString.length); } { @@ -269,7 +276,13 @@ class InvalidMemoryOperationError : Error override string toString() const @trusted { - return msg.length ? (cast()super).toString() : "Invalid memory operation"; + return msg.length ? (cast()this).superToString() : "Invalid memory operation"; + } + + // kludge to call non-const super.toString + private string superToString() @trusted + { + return super.toString(); } } @@ -281,6 +294,7 @@ unittest assert(oome.line == __LINE__ - 2); assert(oome.next is null); assert(oome.msg == "Invalid memory operation"); + assert(oome.toString.length); } { diff --git a/src/core/internal/spinlock.d b/src/core/internal/spinlock.d new file mode 100644 index 0000000000..fb689dc561 --- /dev/null +++ b/src/core/internal/spinlock.d @@ -0,0 +1,103 @@ +/** + * SpinLock for runtime internal usage. + * + * Copyright: Copyright Digital Mars 2015 -. + * License: $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0). + * Authors: Martin Nowak + * Source: $(DRUNTIMESRC core/internal/_spinlock.d) + */ +module core.internal.spinlock; + +import core.atomic, core.thread; + +shared struct SpinLock +{ + /// for how long is the lock usually contended + enum Contention : ubyte + { + brief, + medium, + lengthy, + } + +@trusted @nogc nothrow: + this(Contention contention) + { + this.contention = contention; + } + + void lock() + { + if (cas(&val, size_t(0), size_t(1))) + return; + // Try to reduce the chance of another cas failure + // TTAS lock (https://en.wikipedia.org/wiki/Test_and_test-and-set) + immutable step = 1 << contention; + while (true) + { + for (size_t n; atomicLoad!(MemoryOrder.raw)(val); n += step) + yield(n); + if (cas(&val, size_t(0), size_t(1))) + return; + } + } + + void unlock() + { + atomicStore!(MemoryOrder.rel)(val, size_t(0)); + } + + /// yield with backoff + void yield(size_t k) + { + if (k < pauseThresh) + return pause(); + else if (k < 32) + return Thread.yield(); + Thread.sleep(1.msecs); + } + +private: + version (D_InlineAsm_X86) + enum X86 = true; + else version (D_InlineAsm_X86_64) + enum X86 = true; + else + enum X86 = false; + + static if (X86) + { + enum pauseThresh = 16; + void pause() + { + asm @trusted @nogc nothrow + { + // pause instruction + rep; + nop; + } + } + } + else + { + enum pauseThresh = 4; + void pause() + { + } + } + + size_t val; + Contention contention; +} + +// aligned to cacheline to avoid false sharing +shared align(64) struct AlignedSpinLock +{ + this(SpinLock.Contention contention) + { + impl = shared(SpinLock)(contention); + } + + SpinLock impl; + alias impl this; +} diff --git a/src/core/memory.d b/src/core/memory.d index c222831ac8..40b3a8f8d7 100644 --- a/src/core/memory.d +++ b/src/core/memory.d @@ -145,6 +145,8 @@ private extern (C) void gc_removeRoot( in void* p ) nothrow; extern (C) void gc_removeRange( in void* p ) nothrow @nogc; extern (C) void gc_runFinalizers( in void[] segment ); + + package extern (C) bool gc_inFinalizer(); } diff --git a/src/core/runtime.d b/src/core/runtime.d index 03fd4b2605..cd9c0fe2ef 100644 --- a/src/core/runtime.d +++ b/src/core/runtime.d @@ -494,6 +494,11 @@ Throwable.TraceInfo defaultTraceHandler( void* ptr = null ) else version( Solaris ) import core.sys.solaris.execinfo; + // avoid recursive GC calls in finalizer, trace handlers should be made @nogc instead + import core.memory : gc_inFinalizer; + if (gc_inFinalizer) + return null; + //printf("runtime.defaultTraceHandler()\n"); static if( __traits( compiles, backtrace ) ) { diff --git a/src/core/thread.d b/src/core/thread.d index 7d31ee3533..043a185321 100644 --- a/src/core/thread.d +++ b/src/core/thread.d @@ -1091,7 +1091,7 @@ class Thread * * ------------------------------------------------------------------------ */ - static void sleep( Duration val ) nothrow + static void sleep( Duration val ) @nogc nothrow in { assert( !val.isNegative ); @@ -1134,7 +1134,7 @@ class Thread if( !nanosleep( &tin, &tout ) ) return; if( errno != EINTR ) - throw new ThreadError( "Unable to sleep for the specified duration" ); + assert(0, "Unable to sleep for the specified duration"); tin = tout; } } @@ -1144,7 +1144,7 @@ class Thread /** * Forces a context switch to occur away from the calling thread. */ - static void yield() nothrow + static void yield() @nogc nothrow { version( Windows ) SwitchToThread(); diff --git a/src/gc/gc.d b/src/gc/gc.d index 9ef6e8ecd5..16aec1e231 100644 --- a/src/gc/gc.d +++ b/src/gc/gc.d @@ -48,7 +48,6 @@ import rt.util.container.treap; import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; import core.stdc.string : memcpy, memset, memmove; import core.bitop; -import core.sync.mutex; import core.thread; static import core.memory; private alias BlkAttr = core.memory.GC.BlkAttr; @@ -136,7 +135,7 @@ private // Declared as an extern instead of importing core.exception // to avoid inlining - see issue 13725. - void onInvalidMemoryOperationError() nothrow; + void onInvalidMemoryOperationError() @nogc nothrow; void onOutOfMemoryErrorNoGC() @nogc nothrow; } @@ -256,20 +255,6 @@ debug (LOGGING) const uint GCVERSION = 1; // increment every time we change interface // to GC. -// This just makes Mutex final to de-virtualize member function calls. -final class GCMutex : Mutex -{ - final override void lock() nothrow @trusted @nogc - { - super.lock_nothrow(); - } - - final override void unlock() nothrow @trusted @nogc - { - super.unlock_nothrow(); - } -} - struct GC { // For passing to debug code (not thread safe) @@ -280,10 +265,17 @@ struct GC Gcx *gcx; // implementation - // We can't allocate a Mutex on the GC heap because we are the GC. - // Store it in the static data segment instead. - __gshared GCMutex gcLock; // global lock - __gshared void[__traits(classInstanceSize, GCMutex)] mutexStorage; + import core.internal.spinlock; + static gcLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy); + static bool inFinalizer; + + // lock GC, throw InvalidMemoryOperationError on recursive locking during finalization + static void lockNR() @nogc nothrow + { + if (inFinalizer) + onInvalidMemoryOperationError(); + gcLock.lock(); + } __gshared Config config; @@ -291,9 +283,6 @@ struct GC { config.initialize(); - mutexStorage[] = typeid(GCMutex).initializer[]; - gcLock = cast(GCMutex) mutexStorage.ptr; - gcLock.__ctor(); gcx = cast(Gcx*)cstdlib.calloc(1, Gcx.sizeof); if (!gcx) onOutOfMemoryErrorNoGC(); @@ -328,10 +317,12 @@ struct GC */ void enable() { - gcLock.lock(); - scope(exit) gcLock.unlock(); - assert(gcx.disabled > 0); - gcx.disabled--; + static void go(Gcx* gcx) nothrow + { + assert(gcx.disabled > 0); + gcx.disabled--; + } + runLocked!(go, otherTime, numOthers)(gcx); } @@ -340,15 +331,38 @@ struct GC */ void disable() { - gcLock.lock(); - scope(exit) gcLock.unlock(); - gcx.disabled++; + static void go(Gcx* gcx) nothrow + { + gcx.disabled++; + } + runLocked!(go, otherTime, numOthers)(gcx); + } + + auto runLocked(alias func, Args...)(auto ref Args args) + { + debug(PROFILE_API) immutable tm = (GC.config.profile > 1 ? currTime.ticks : 0); + lockNR(); + scope (failure) gcLock.unlock(); + debug(PROFILE_API) immutable tm2 = (GC.config.profile > 1 ? currTime.ticks : 0); + + static if (is(typeof(func(args)) == void)) + func(args); + else + auto res = func(args); + + debug(PROFILE_API) if (GC.config.profile > 1) + lockTime += tm2 - tm; + gcLock.unlock(); + + static if (!is(typeof(func(args)) == void)) + return res; } auto runLocked(alias func, alias time, alias count, Args...)(auto ref Args args) { debug(PROFILE_API) immutable tm = (GC.config.profile > 1 ? currTime.ticks : 0); - gcLock.lock(); + lockNR(); + scope (failure) gcLock.unlock(); debug(PROFILE_API) immutable tm2 = (GC.config.profile > 1 ? currTime.ticks : 0); static if (is(typeof(func(args)) == void)) @@ -494,9 +508,6 @@ struct GC assert(gcx); //debug(PRINTF) printf("gcx.self = %x, pthread_self() = %x\n", gcx.self, pthread_self()); - if (gcx.running) - onInvalidMemoryOperationError(); - auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits); if (!p) onOutOfMemoryErrorNoGC(); @@ -564,9 +575,6 @@ struct GC // private void *reallocNoSync(void *p, size_t size, ref uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow { - if (gcx.running) - onInvalidMemoryOperationError(); - if (!size) { if (p) { freeNoSync(p); @@ -728,9 +736,6 @@ struct GC } body { - if (gcx.running) - onInvalidMemoryOperationError(); - //debug(PRINTF) printf("GC::extend(p = %p, minsize = %zu, maxsize = %zu)\n", p, minsize, maxsize); debug (SENTINEL) { @@ -787,10 +792,7 @@ struct GC return 0; } - gcLock.lock(); - auto rc = reserveNoSync(size); - gcLock.unlock(); - return rc; + return runLocked!(reserveNoSync, otherTime, numOthers)(size); } @@ -802,9 +804,6 @@ struct GC assert(size != 0); assert(gcx); - if (gcx.running) - onInvalidMemoryOperationError(); - return gcx.reserve(size); } @@ -831,9 +830,6 @@ struct GC debug(PRINTF) printf("Freeing %p\n", cast(size_t) p); assert (p); - if (gcx.running) - onInvalidMemoryOperationError(); - Pool* pool; size_t pagenum; Bins bin; @@ -1103,10 +1099,11 @@ struct GC private auto rootIterImpl(scope int delegate(ref Root) nothrow dg) nothrow { - gcLock.lock(); - auto res = gcx.roots.opApply(dg); - gcLock.unlock(); - return res; + static int go(ref Treap!(Root) roots, scope int delegate(ref Root) nothrow dg) nothrow + { + return roots.opApply(dg); + } + return runLocked!(go, otherTime, numOthers)(gcx.roots, dg); } /** @@ -1167,10 +1164,11 @@ struct GC private auto rangeIterImpl(scope int delegate(ref Range) nothrow dg) nothrow { - gcLock.lock(); - auto res = gcx.ranges.opApply(dg); - gcLock.unlock(); - return res; + static int go(ref Treap!(Range) ranges, scope int delegate(ref Range) nothrow dg) nothrow + { + return ranges.opApply(dg); + } + return runLocked!(go, otherTime, numOthers)(gcx.ranges, dg); } /** @@ -1189,15 +1187,14 @@ struct GC size_t fullCollect() nothrow { debug(PRINTF) printf("GC.fullCollect()\n"); - size_t result; // Since a finalizer could launch a new thread, we always need to lock // when collecting. + static size_t go(Gcx* gcx) nothrow { - gcLock.lock(); - result = gcx.fullcollect(); - gcLock.unlock(); + return gcx.fullcollect(); } + immutable result = runLocked!go(gcx); version (none) { @@ -1220,11 +1217,11 @@ struct GC { // Since a finalizer could launch a new thread, we always need to lock // when collecting. + static size_t go(Gcx* gcx) nothrow { - gcLock.lock(); - gcx.fullcollect(true); - gcLock.unlock(); + return gcx.fullcollect(true); } + runLocked!go(gcx); } @@ -1233,9 +1230,11 @@ struct GC */ void minimize() nothrow { - gcLock.lock(); - gcx.minimize(); - gcLock.unlock(); + static void go(Gcx* gcx) nothrow + { + gcx.minimize(); + } + runLocked!(go, otherTime, numOthers)(gcx); } @@ -1245,9 +1244,7 @@ struct GC */ void getStats(out GCStats stats) nothrow { - gcLock.lock(); - getStatsNoSync(stats); - gcLock.unlock(); + return runLocked!(getStatsNoSync, otherTime, numOthers)(stats); } @@ -1372,7 +1369,6 @@ struct Gcx bool log; // turn on logging debug(INVARIANT) bool initialized; - bool running; uint disabled; // turn off collections if >0 import gc.pooltable; @@ -1543,6 +1539,9 @@ struct Gcx */ void runFinalizers(in void[] segment) nothrow { + GC.inFinalizer = true; + scope (failure) GC.inFinalizer = false; + foreach (pool; pooltable[0 .. npools]) { if (!pool.finals.nbits) continue; @@ -1558,6 +1557,7 @@ struct Gcx spool.runFinalizers(segment); } } + GC.inFinalizer = false; } Pool* findPool(void* p) pure nothrow @@ -2417,10 +2417,6 @@ struct Gcx debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n"); //printf("\tpool address range = %p .. %p\n", minAddr, maxAddr); - if (running) - onInvalidMemoryOperationError(); - running = true; - thread_suspendAll(); prepare(); @@ -2447,7 +2443,13 @@ struct Gcx start = stop; } - immutable freedLargePages = sweep(); + GC.inFinalizer = true; + size_t freedLargePages=void; + { + scope (failure) GC.inFinalizer = false; + freedLargePages = sweep(); + GC.inFinalizer = false; + } if (GC.config.profile) { @@ -2465,8 +2467,6 @@ struct Gcx ++numCollections; } - running = false; // only clear on success - updateCollectThresholds(); return freedLargePages + freedSmallPages; diff --git a/src/gc/proxy.d b/src/gc/proxy.d index e8d96cdc3f..eb0f7ee275 100644 --- a/src/gc/proxy.d +++ b/src/gc/proxy.d @@ -60,6 +60,8 @@ private void function(void*) gc_removeRoot; void function(void*) gc_removeRange; void function(in void[]) gc_runFinalizers; + + bool function() gc_inFinalizer; } } @@ -96,6 +98,8 @@ private pthis.gc_removeRoot = &gc_removeRoot; pthis.gc_removeRange = &gc_removeRange; pthis.gc_runFinalizers = &gc_runFinalizers; + + pthis.gc_inFinalizer = &gc_inFinalizer; } } @@ -307,6 +311,13 @@ extern (C) return proxy.gc_runFinalizers( segment ); } + bool gc_inFinalizer() nothrow + { + if( proxy is null ) + return _gc.inFinalizer; + return proxy.gc_inFinalizer(); + } + Proxy* gc_getProxy() nothrow { return &pthis; diff --git a/src/gcstub/gc.d b/src/gcstub/gc.d index 534e9bf7a7..b357a8074e 100644 --- a/src/gcstub/gc.d +++ b/src/gcstub/gc.d @@ -67,6 +67,8 @@ private extern (C) void function(void*) gc_removeRoot; extern (C) void function(void*) gc_removeRange; extern (C) void function(in void[]) gc_runFinalizers; + + extern (C) bool function() gc_inFinalizer; } __gshared Proxy pthis; @@ -102,6 +104,8 @@ private pthis.gc_removeRoot = &gc_removeRoot; pthis.gc_removeRange = &gc_removeRange; pthis.gc_runFinalizers = &gc_runFinalizers; + + pthis.gc_inFinalizer = &gc_inFinalizer; } __gshared void** roots = null; @@ -349,6 +353,13 @@ extern (C) void gc_runFinalizers( in void[] segment ) proxy.gc_runFinalizers( segment ); } +extern (C) bool gc_inFinalizer() +{ + if( proxy !is null ) + return proxy.gc_inFinalizer(); + return false; +} + extern (C) Proxy* gc_getProxy() { return &pthis; diff --git a/win32.mak b/win32.mak index 098ea6bd35..358668b7e1 100644 --- a/win32.mak +++ b/win32.mak @@ -268,6 +268,9 @@ $(IMPDIR)\core\internal\convert.d : src\core\internal\convert.d $(IMPDIR)\core\internal\hash.d : src\core\internal\hash.d copy $** $@ +$(IMPDIR)\core\internal\spinlock.d : src\core\internal\spinlock.d + copy $** $@ + $(IMPDIR)\core\internal\string.d : src\core\internal\string.d copy $** $@ diff --git a/win64.mak b/win64.mak index 02b6af9f82..69f3ba701d 100644 --- a/win64.mak +++ b/win64.mak @@ -276,6 +276,9 @@ $(IMPDIR)\core\internal\convert.d : src\core\internal\convert.d $(IMPDIR)\core\internal\hash.d : src\core\internal\hash.d copy $** $@ +$(IMPDIR)\core\internal\spinlock.d : src\core\internal\spinlock.d + copy $** $@ + $(IMPDIR)\core\internal\string.d : src\core\internal\string.d copy $** $@