diff --git a/CODEOWNERS b/CODEOWNERS index 50ee7e8455..bcdb157e21 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -15,7 +15,7 @@ src/checkedint.d @redstar @andralex @JackStouffer -src/core/atomic.d @WalterBright @ibuclaw +src/core/atomic.d @WalterBright @ibuclaw @TurkeyMan src/core/attribute.d @jacob-carlborg src/core/bitop.d @schveiguy @tsbockman @Geod24 src/core/cpuid.d @WalterBright @ibuclaw @JackStouffer @@ -26,7 +26,7 @@ src/core/math.d @ibuclaw @redstar src/core/runtime.d @MartinNowak @Abscissa src/core/simd.d @WalterBright @MartinNowak src/core/stdc/* @schveiguy @ibuclaw -src/core/stdcpp/* @WalterBright @Darredevil +src/core/stdcpp/* @WalterBright @Darredevil @TurkeyMan src/core/sync/* @MartinNowak @Geod24 @WalterBright @ZombineDev src/core/sys/bionic/* @joakim-noah src/core/sys/darwin/* @jacob-carlborg @klickverbot @etcimon @MartinNowak diff --git a/mak/COPY b/mak/COPY index 1a8342116b..2962e567e4 100644 --- a/mak/COPY +++ b/mak/COPY @@ -23,6 +23,7 @@ COPY=\ \ $(IMPDIR)\core\internal\abort.d \ $(IMPDIR)\core\internal\arrayop.d \ + $(IMPDIR)\core\internal\atomic.d \ $(IMPDIR)\core\internal\attributes.d \ $(IMPDIR)\core\internal\convert.d \ $(IMPDIR)\core\internal\dassert.d \ diff --git a/mak/SRCS b/mak/SRCS index cc0e925154..e744d0c992 100644 --- a/mak/SRCS +++ b/mak/SRCS @@ -23,6 +23,7 @@ SRCS=\ \ src\core\internal\abort.d \ src\core\internal\arrayop.d \ + src\core\internal\atomic.d \ src\core\internal\convert.d \ src\core\internal\dassert.d \ src\core\internal\hash.d \ diff --git a/mak/WINDOWS b/mak/WINDOWS index 4cd551976a..cccb5cc480 100644 --- a/mak/WINDOWS +++ b/mak/WINDOWS @@ -123,6 +123,9 @@ $(IMPDIR)\core\internal\abort.d : src\core\internal\abort.d $(IMPDIR)\core\internal\arrayop.d : src\core\internal\arrayop.d copy $** $@ +$(IMPDIR)\core\internal\atomic.d : src\core\internal\atomic.d + copy $** $@ + $(IMPDIR)\core\internal\attributes.d : src\core\internal\attributes.d copy $** $@ diff --git a/src/core/atomic.d b/src/core/atomic.d index 6b54587199..fcbf899868 100644 --- a/src/core/atomic.d +++ b/src/core/atomic.d @@ -10,2100 +10,587 @@ module core.atomic; +import core.internal.atomic; import core.internal.attributes : betterC; +import core.internal.traits : hasUnsharedIndirections; -version (D_InlineAsm_X86) -{ - version = AsmX86; - version = AsmX86_32; - enum has64BitXCHG = false; - enum has64BitCAS = true; - enum has128BitCAS = false; -} -else version (D_InlineAsm_X86_64) -{ - version = AsmX86; - version = AsmX86_64; - enum has64BitXCHG = true; - enum has64BitCAS = true; - enum has128BitCAS = true; -} -else +/** + * Specifies the memory ordering semantics of an atomic operation. + * + * See_Also: + * $(HTTP en.cppreference.com/w/cpp/atomic/memory_order) + */ +enum MemoryOrder { - enum has64BitXCHG = false; - enum has64BitCAS = false; - enum has128BitCAS = false; + /** + * Not sequenced. + * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#monotonic, LLVM AtomicOrdering.Monotonic) + * and C++11/C11 `memory_order_relaxed`. + */ + raw, + /** + * Hoist-load + hoist-store barrier. + * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#acquire, LLVM AtomicOrdering.Acquire) + * and C++11/C11 `memory_order_acquire`. + */ + acq, + /** + * Sink-load + sink-store barrier. + * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#release, LLVM AtomicOrdering.Release) + * and C++11/C11 `memory_order_release`. + */ + rel, + /** + * Acquire + release barrier. + * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#acquirerelease, LLVM AtomicOrdering.AcquireRelease) + * and C++11/C11 `memory_order_acq_rel`. + */ + acq_rel, + /** + * Fully sequenced (acquire + release). Corresponds to + * $(LINK2 https://llvm.org/docs/Atomics.html#sequentiallyconsistent, LLVM AtomicOrdering.SequentiallyConsistent) + * and C++11/C11 `memory_order_seq_cst`. + */ + seq, } -private +/** + * Loads 'val' from memory and returns it. The memory barrier specified + * by 'ms' is applied to the operation, which is fully sequenced by + * default. Valid memory orders are MemoryOrder.raw, MemoryOrder.acq, + * and MemoryOrder.seq. + * + * Params: + * val = The target variable. + * + * Returns: + * The value of 'val'. + */ +T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref T val ) pure nothrow @nogc @trusted + if ( !is( T == shared U, U ) && !is( T == shared inout U, U ) && !is( T == shared const U, U ) ) { - /* Construct a type with a shared tail, and if possible with an unshared - head. */ - template TailShared(U) if (!is(U == shared)) - { - alias TailShared = .TailShared!(shared U); - } - template TailShared(S) if (is(S == shared)) + static if ( __traits(isFloating, T) ) { - // Get the unshared variant of S. - static if (is(S U == shared U)) {} - else static assert(false, "Should never be triggered. The `static " ~ - "if` declares `U` as the unshared version of the shared type " ~ - "`S`. `S` is explicitly declared as shared, so getting `U` " ~ - "should always work."); - - static if (is(S : U)) - alias TailShared = U; - else static if (is(S == struct)) - { - enum implName = () { - /* Start with "_impl". If S has a field with that name, append - underscores until the clash is resolved. */ - string name = "_impl"; - string[] fieldNames; - static foreach (alias field; S.tupleof) - { - fieldNames ~= __traits(identifier, field); - } - static bool canFind(string[] haystack, string needle) - { - foreach (candidate; haystack) - { - if (candidate == needle) return true; - } - return false; - } - while (canFind(fieldNames, name)) name ~= "_"; - return name; - } (); - struct TailShared - { - static foreach (i, alias field; S.tupleof) - { - /* On @trusted: This is casting the field from shared(Foo) - to TailShared!Foo. The cast is safe because the field has - been loaded and is not shared anymore. */ - mixin(" - @trusted @property - ref " ~ __traits(identifier, field) ~ "() - { - alias R = TailShared!(typeof(field)); - return * cast(R*) &" ~ implName ~ ".tupleof[i]; - } - "); - } - mixin(" - S " ~ implName ~ "; - alias " ~ implName ~ " this; - "); - } - } - else - alias TailShared = S; + alias IntTy = IntForFloat!T; + IntTy r = core.internal.atomic.atomicLoad!ms(cast(IntTy*)&val); + return *cast(T*)&r; } - @safe unittest - { - // No tail (no indirections) -> fully unshared. - - static assert(is(TailShared!int == int)); - static assert(is(TailShared!(shared int) == int)); - - static struct NoIndir { int i; } - static assert(is(TailShared!NoIndir == NoIndir)); - static assert(is(TailShared!(shared NoIndir) == NoIndir)); - - // Tail can be independently shared or is already -> tail-shared. - - static assert(is(TailShared!(int*) == shared(int)*)); - static assert(is(TailShared!(shared int*) == shared(int)*)); - static assert(is(TailShared!(shared(int)*) == shared(int)*)); - - static assert(is(TailShared!(int[]) == shared(int)[])); - static assert(is(TailShared!(shared int[]) == shared(int)[])); - static assert(is(TailShared!(shared(int)[]) == shared(int)[])); - - static struct S1 { shared int* p; } - static assert(is(TailShared!S1 == S1)); - static assert(is(TailShared!(shared S1) == S1)); - - static struct S2 { shared(int)* p; } - static assert(is(TailShared!S2 == S2)); - static assert(is(TailShared!(shared S2) == S2)); + else + return core.internal.atomic.atomicLoad!ms(&val); +} - // Tail follows shared-ness of head -> fully shared. +/// Ditto +T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref shared T val ) pure nothrow @nogc @trusted + if ( !hasUnsharedIndirections!T ) +{ + import core.internal.traits : hasUnsharedIndirections; + static assert(!hasUnsharedIndirections!T, "Copying `shared " ~ T.stringof ~ "` would violate shared."); - static class C { int i; } - static assert(is(TailShared!C == shared C)); - static assert(is(TailShared!(shared C) == shared C)); + return atomicLoad!ms(*cast(T*)&val); +} - /* However, structs get a wrapper that has getters which cast to - TailShared. */ +/// Ditto +TailShared!T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref shared T val ) pure nothrow @nogc @trusted + if ( hasUnsharedIndirections!T ) +{ + // HACK: DEPRECATE THIS FUNCTION, IT IS INVALID TO DO ATOMIC LOAD OF SHARED CLASS + // this is here because code exists in the wild that does this... - static struct S3 { int* p; int _impl; int _impl_; int _impl__; } - static assert(!is(TailShared!S3 : S3)); - static assert(is(TailShared!S3 : shared S3)); - static assert(is(TailShared!(shared S3) == TailShared!S3)); + import core.lifetime : move; - static struct S4 { shared(int)** p; } - static assert(!is(TailShared!S4 : S4)); - static assert(is(TailShared!S4 : shared S4)); - static assert(is(TailShared!(shared S4) == TailShared!S4)); - } + T r = core.internal.atomic.atomicLoad!ms(cast(T*)&val); + return move(*cast(TailShared!T*)&r); } - -version (AsmX86) +/** + * Writes 'newval' into 'val'. The memory barrier specified by 'ms' is + * applied to the operation, which is fully sequenced by default. + * Valid memory orders are MemoryOrder.raw, MemoryOrder.rel, and + * MemoryOrder.seq. + * + * Params: + * val = The target variable. + * newval = The value to store. + */ +void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V)( ref T val, V newval ) pure nothrow @nogc @trusted + if ( __traits( compiles, { val = newval; } ) && !is(T == shared S, S) && !is(V == shared U, U) ) { - // NOTE: Strictly speaking, the x86 supports atomic operations on - // unaligned values. However, this is far slower than the - // common case, so such behavior should be prohibited. - private bool atomicValueIsProperlyAligned(T)( ref T val ) pure nothrow @nogc @trusted - { - return atomicPtrIsProperlyAligned(&val); - } - - private bool atomicPtrIsProperlyAligned(T)( T* ptr ) pure nothrow @nogc @safe + static if ( __traits(isFloating, T) ) { - // NOTE: 32 bit x86 systems support 8 byte CAS, which only requires - // 4 byte alignment, so use size_t as the align type here. - static if ( T.sizeof > size_t.sizeof ) - return cast(size_t)ptr % size_t.sizeof == 0; - else - return cast(size_t)ptr % T.sizeof == 0; + static assert ( __traits(isFloating, V) && V.sizeof == T.sizeof, "Mismatching argument types." ); + alias IntTy = IntForFloat!T; + core.internal.atomic.atomicStore!ms(cast(IntTy*)&val, *cast(IntTy*)&newval); } + else + core.internal.atomic.atomicStore!ms(&val, newval); } - -version (CoreDdoc) +/// Ditto +void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V)( ref shared T val, V newval ) pure nothrow @nogc @trusted + if ( __traits( compiles, { val = newval; } ) && !is( T == class ) ) { - /** - * Performs the binary operation 'op' on val using 'mod' as the modifier. - * - * Params: - * val = The target variable. - * mod = The modifier to apply. - * - * Returns: - * The result of the operation. - */ - TailShared!T atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe - if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) + static if ( is ( V == shared U, U ) ) + alias Thunk = U; + else { - return TailShared!T.init; + import core.internal.traits : hasUnsharedIndirections; + static assert(!hasUnsharedIndirections!V, "Copying unshared argument `newval` to shared `val` would violate shared."); + alias Thunk = V; } + atomicStore!ms(*cast(T*)&val, *cast(Thunk*)&newval); +} - /** - * Atomically adds `mod` to the value referenced by `val` and returns the value `val` held previously. - * This operation is both lock-free and atomic. - * - * Params: - * val = Reference to the value to modify. - * mod = The value to add. - * - * Returns: - * The value held previously by `val`. - */ - TailShared!(T) atomicFetchAdd(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe; - - /** - * Atomically subtracts `mod` from the value referenced by `val` and returns the value `val` held previously. - * This operation is both lock-free and atomic. - * - * Params: - * val = Reference to the value to modify. - * mod = The value to subtract. - * - * Returns: - * The value held previously by `val`. - */ - TailShared!(T) atomicFetchSub(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe; - - /** - * Exchange `exchangeWith` with the memory referenced by `here`. - * This operation is both lock-free and atomic. - * - * Params: - * here = The address of the destination variable. - * exchangeWith = The value to exchange. - * - * Returns: - * The value held previously by `here`. - */ - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, V exchangeWith ) pure nothrow @nogc @safe - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ); - - /// Ditto - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V) exchangeWith ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = exchangeWith; } ) ); - - /// Ditto - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V)* exchangeWith ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ); - - /** - * Stores 'writeThis' to the memory referenced by 'here' if the value - * referenced by 'here' is equal to 'ifThis'. This operation is both - * lock-free and atomic. - * - * Params: - * here = The address of the destination variable. - * writeThis = The value to store. - * ifThis = The comparison value. - * - * Returns: - * true if the store occurred, false if not. - */ - bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ); +/// Ditto +void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V)( ref shared T val, shared V newval ) pure nothrow @nogc @trusted + if ( is( T == class ) ) +{ + static assert ( is ( V : T ), "Can't assign `newval` of type `shared " ~ V.stringof ~ "` to `shared " ~ T.stringof ~ "`."); - /// Ditto - bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ); + core.internal.atomic.atomicStore!ms(cast(T*)&val, cast(V)newval); +} - /// Ditto - bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ); +/** + * Atomically adds `mod` to the value referenced by `val` and returns the value `val` held previously. + * This operation is both lock-free and atomic. + * + * Params: + * val = Reference to the value to modify. + * mod = The value to add. + * + * Returns: + * The value held previously by `val`. + */ +TailShared!T atomicFetchAdd(MemoryOrder ms = MemoryOrder.seq, T)( ref shared T val, size_t mod ) pure nothrow @nogc @trusted + if ( __traits(isIntegral, T) ) +in ( atomicValueIsProperlyAligned(val) ) +{ + return core.internal.atomic.atomicFetchAdd!ms( &val, cast(T)mod ); +} - /** - * Stores 'writeThis' to the memory referenced by 'here' if the value - * referenced by 'here' is equal to the value referenced by 'ifThis'. - * The prior value referenced by 'here' is written to `ifThis` and - * returned to the user. This operation is both lock-free and atomic. - * - * Params: - * here = The address of the destination variable. - * writeThis = The value to store. - * ifThis = The address of the value to compare, and receives the prior value of `here` as output. - * - * Returns: - * true if the store occurred, false if not. - */ - bool cas(T,V1,V2)( shared(T)* here, V1* ifThis, V2 writeThis ) pure nothrow @nogc @safe - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ); - - /// Ditto - bool cas(T,V1,V2)( shared(T)* here, shared(V1)* ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ); - - /// Ditto - bool cas(T,V1,V2)( shared(T)* here, shared(V1)** ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ); +/** + * Atomically subtracts `mod` from the value referenced by `val` and returns the value `val` held previously. + * This operation is both lock-free and atomic. + * + * Params: + * val = Reference to the value to modify. + * mod = The value to subtract. + * + * Returns: + * The value held previously by `val`. + */ +TailShared!T atomicFetchSub(MemoryOrder ms = MemoryOrder.seq, T)( ref shared T val, size_t mod ) pure nothrow @nogc @trusted + if ( __traits(isIntegral, T) ) +in ( atomicValueIsProperlyAligned(val) ) +{ + return core.internal.atomic.atomicFetchSub!ms( &val, cast(T)mod ); +} - /** - * Loads 'val' from memory and returns it. The memory barrier specified - * by 'ms' is applied to the operation, which is fully sequenced by - * default. Valid memory orders are MemoryOrder.raw, MemoryOrder.acq, - * and MemoryOrder.seq. - * - * Params: - * val = The target variable. - * - * Returns: - * The value of 'val'. - */ - TailShared!T atomicLoad(MemoryOrder ms = MemoryOrder.seq,T)( ref const shared T val ) pure nothrow @nogc @safe +/** + * Exchange `exchangeWith` with the memory referenced by `here`. + * This operation is both lock-free and atomic. + * + * Params: + * here = The address of the destination variable. + * exchangeWith = The value to exchange. + * + * Returns: + * The value held previously by `here`. + */ +shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, V exchangeWith ) pure nothrow @nogc @trusted + if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + static if ( __traits(isFloating, T) ) { - return TailShared!T.init; + static assert ( __traits(isFloating, V) && V.sizeof == T.sizeof, "Mismatching argument types." ); + alias IntTy = IntForFloat!T; + IntTy r = core.internal.atomic.atomicExchange!ms(cast(IntTy*)here, *cast(IntTy*)&exchangeWith); + return *cast(shared(T)*)&r; } + else + return core.internal.atomic.atomicExchange!ms(here, exchangeWith); +} +/// Ditto +shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V) exchangeWith ) pure nothrow @nogc @safe + if ( is(T == class) && __traits( compiles, { *here = exchangeWith; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + return core.internal.atomic.atomicExchange!ms(here, exchangeWith); +} - /** - * Writes 'newval' into 'val'. The memory barrier specified by 'ms' is - * applied to the operation, which is fully sequenced by default. - * Valid memory orders are MemoryOrder.raw, MemoryOrder.rel, and - * MemoryOrder.seq. - * - * Params: - * val = The target variable. - * newval = The value to store. - */ - void atomicStore(MemoryOrder ms = MemoryOrder.seq,T,V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe - if ( __traits( compiles, { val = newval; } ) ) - { +/// Ditto +shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V)* exchangeWith ) pure nothrow @nogc @safe + if ( is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + return core.internal.atomic.atomicExchange!ms(here, exchangeWith); +} +/** + * Stores 'writeThis' to the memory referenced by 'here' if the value + * referenced by 'here' is equal to 'ifThis'. This operation is both + * lock-free and atomic. + * + * Params: + * here = The address of the destination variable. + * writeThis = The value to store. + * ifThis = The comparison value. + * + * Returns: + * true if the store occurred, false if not. + */ +bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted + if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + static if ( __traits(isFloating, T) ) + { + static assert ( __traits(isFloating, V1) && V1.sizeof == T.sizeof, "Mismatching argument types." ); + static assert ( __traits(isFloating, V2) && V2.sizeof == T.sizeof, "Mismatching argument types." ); + alias IntTy = IntForFloat!T; + return atomicCompareExchangeStrongNoResult( cast(IntTy*)here, *cast(IntTy*)&ifThis, *cast(IntTy*)&writeThis ); } + else + return atomicCompareExchangeStrongNoResult!( MemoryOrder.seq, MemoryOrder.seq, T )( cast(T*)here, cast()ifThis, cast()writeThis ); +} +/// Ditto +bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe + if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + return atomicCompareExchangeStrongNoResult( here, ifThis, writeThis ); +} - /** - * Specifies the memory ordering semantics of an atomic operation. - * - * See_Also: - * $(HTTP en.cppreference.com/w/cpp/atomic/memory_order) - */ - enum MemoryOrder +/// Ditto +bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe + if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + return atomicCompareExchangeStrongNoResult( here, ifThis, writeThis ); +} + +/** + * Stores 'writeThis' to the memory referenced by 'here' if the value + * referenced by 'here' is equal to the value referenced by 'ifThis'. + * The prior value referenced by 'here' is written to `ifThis` and + * returned to the user. This operation is both lock-free and atomic. + * + * Params: + * here = The address of the destination variable. + * writeThis = The value to store. + * ifThis = The address of the value to compare, and receives the prior value of `here` as output. + * + * Returns: + * true if the store occurred, false if not. + */ +bool cas(T,V)( shared(T)* here, shared(T)* ifThis, V writeThis ) pure nothrow @nogc @trusted + if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; *ifThis = *here; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + static if ( __traits(isFloating, T) ) { - /++ - Not sequenced. - Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#monotonic, LLVM AtomicOrdering.Monotonic) - and C++11/C11 `memory_order_relaxed`. - +/ - raw, - /++ - Hoist-load + hoist-store barrier. - Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#acquire, LLVM AtomicOrdering.Acquire) - and C++11/C11 `memory_order_acquire`. - +/ - acq, - /++ - Sink-load + sink-store barrier. - Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#release, LLVM AtomicOrdering.Release) - and C++11/C11 `memory_order_release`. - +/ - rel, - /++ - Fully sequenced (acquire + release). Corresponds to - $(LINK2 https://llvm.org/docs/Atomics.html#sequentiallyconsistent, LLVM AtomicOrdering.SequentiallyConsistent) - and C++11/C11 `memory_order_seq_cst`. - +/ - seq, + static assert ( __traits(isFloating, V) && V.sizeof == T.sizeof, "Mismatching argument types." ); + alias IntTy = IntForFloat!T; + return atomicCompareExchangeStrong( cast(IntTy*)here, cast(IntTy*)ifThis, *cast(IntTy*)&writeThis ); } + else + return atomicCompareExchangeStrong!( MemoryOrder.seq, MemoryOrder.seq, T )( cast(T*)here, cast(T*)ifThis, cast()writeThis ); +} - /** - * Inserts a full load/store memory fence (on platforms that need it). This ensures - * that all loads and stores before a call to this function are executed before any - * loads and stores after the call. - */ - void atomicFence() nothrow @nogc; +/// Ditto +bool cas(T,V)( shared(T)* here, shared(T)* ifThis, shared(V) writeThis ) pure nothrow @nogc @trusted + if ( is(T == class) && __traits( compiles, { *here = writeThis; *ifThis = *here; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) +{ + return atomicCompareExchangeStrong( cast(T*)here, cast(T*)ifThis, cast()writeThis ); } -else version (AsmX86_32) + +/// Ditto +bool cas(T,V)( shared(T)* here, shared(T)* ifThis, shared(V)* writeThis ) pure nothrow @nogc @trusted + if ( is(T U : U*) && __traits( compiles, { *here = writeThis; *ifThis = *here; } ) ) +in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) { - // Uses specialized asm for fast fetch and add operations - TailShared!(T) atomicFetchAdd(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe - if ( T.sizeof <= 4 ) - { - asm pure nothrow @nogc @trusted - { - mov EAX, mod; - mov EDX, val; - } - static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[EDX], AL; } - else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[EDX], AX; } - else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[EDX], EAX; } - } + return atomicCompareExchangeStrong!( MemoryOrder.seq, MemoryOrder.seq, T )( cast(T*)here, cast(T*)ifThis, writeThis ); +} - TailShared!(T) atomicFetchSub(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe - if ( T.sizeof <= 4) - { - return atomicFetchAdd(val, -mod); - } +/** + * Inserts a full load/store memory fence (on platforms that need it). This ensures + * that all loads and stores before a call to this function are executed before any + * loads and stores after the call. + */ +void atomicFence() nothrow @nogc @safe +{ + core.internal.atomic.atomicFence(); +} - TailShared!T atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc - if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) - in - { - assert(atomicValueIsProperlyAligned(val)); +/** + * Performs the binary operation 'op' on val using 'mod' as the modifier. + * + * Params: + * val = The target variable. + * mod = The modifier to apply. + * + * Returns: + * The result of the operation. + */ +TailShared!T atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe + if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) +in ( atomicValueIsProperlyAligned( val ) ) +{ + // binary operators + // + // + - * / % ^^ & + // | ^ << >> >>> ~ in + // == != < <= > >= + static if ( op == "+" || op == "-" || op == "*" || op == "/" || + op == "%" || op == "^^" || op == "&" || op == "|" || + op == "^" || op == "<<" || op == ">>" || op == ">>>" || + op == "~" || // skip "in" + op == "==" || op == "!=" || op == "<" || op == "<=" || + op == ">" || op == ">=" ) + { + TailShared!T get = atomicLoad!(MemoryOrder.raw)( val ); + mixin( "return get " ~ op ~ " mod;" ); } - do + else + // assignment operators + // + // += -= *= /= %= ^^= &= + // |= ^= <<= >>= >>>= ~= + static if ( op == "+=" && __traits(isIntegral, T) && __traits(isIntegral, V1) && T.sizeof <= size_t.sizeof && V1.sizeof <= size_t.sizeof) { - // binary operators - // - // + - * / % ^^ & - // | ^ << >> >>> ~ in - // == != < <= > >= - static if (op == "+" || op == "-" || op == "*" || op == "/" || - op == "%" || op == "^^" || op == "&" || op == "|" || - op == "^" || op == "<<" || op == ">>" || op == ">>>" || - op == "~" || // skip "in" - op == "==" || op == "!=" || op == "<" || op == "<=" || - op == ">" || op == ">=") - { - TailShared!T get = atomicLoad!(MemoryOrder.raw)( val ); - mixin( "return get " ~ op ~ " mod;" ); - } - else - // assignment operators - // - // += -= *= /= %= ^^= &= - // |= ^= <<= >>= >>>= ~= - static if ( op == "+=" && __traits(isIntegral, T) && T.sizeof <= 4 && V1.sizeof <= 4) - { - return cast(T)(atomicFetchAdd!(T)(val, mod) + mod); - } - else static if ( op == "-=" && __traits(isIntegral, T) && T.sizeof <= 4 && V1.sizeof <= 4) - { - return cast(T)(atomicFetchSub!(T)(val, mod) - mod); - } - else static if ( op == "+=" || op == "-=" || op == "*=" || op == "/=" || - op == "%=" || op == "^^=" || op == "&=" || op == "|=" || - op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=" ) // skip "~=" - { - TailShared!T get, set; - - do - { - get = set = atomicLoad!(MemoryOrder.raw)( val ); - mixin( "set " ~ op ~ " mod;" ); - } while ( !casByRef( val, get, set ) ); - return set; - } - else - { - static assert( false, "Operation not supported." ); - } - } - - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, V exchangeWith ) pure nothrow @nogc @safe - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ) - { - return atomicExchangeImpl(here, exchangeWith); - } - - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V) exchangeWith ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = exchangeWith; } ) ) - { - return atomicExchangeImpl(here, exchangeWith); - } - - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V)* exchangeWith ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ) - { - return atomicExchangeImpl(here, exchangeWith); - } - - private shared(T) atomicExchangeImpl(T,V)( shared(T)* here, V exchangeWith ) pure nothrow @nogc @safe - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - static if ( T.sizeof == byte.sizeof ) - { - asm pure nothrow @nogc @trusted - { - mov AL, exchangeWith; - mov ECX, here; - xchg [ECX], AL; - } - } - else static if ( T.sizeof == short.sizeof ) - { - asm pure nothrow @nogc @trusted - { - mov AX, exchangeWith; - mov ECX, here; - xchg [ECX], AX; - } - } - else static if ( T.sizeof == int.sizeof ) - { - asm pure nothrow @nogc @trusted - { - mov EAX, exchangeWith; - mov ECX, here; - xchg [ECX], EAX; - } - static if ( __traits(isFloating, T) ) - { - asm pure nothrow @nogc @trusted - { - mov exchangeWith, EAX; - } - return exchangeWith; - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted - { - return cas(&value, ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - { - return casImplNoResult(here, ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) - { - return casImplNoResult(here, ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - { - return casImplNoResult(here, ifThis, writeThis); - } - - private bool casImplNoResult(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe - in - { - assert( atomicPtrIsProperlyAligned( here ) ); - } - do - { - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte CAS - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - mov DL, writeThis; - mov AL, ifThis; - mov ECX, here; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], DL; - setz AL; - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte CAS - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - mov DX, writeThis; - mov AX, ifThis; - mov ECX, here; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], DX; - setz AL; - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte CAS - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - mov EDX, writeThis; - mov EAX, ifThis; - mov ECX, here; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], EDX; - setz AL; - } - } - else static if ( T.sizeof == long.sizeof && has64BitCAS ) - { - - ////////////////////////////////////////////////////////////////// - // 8 Byte CAS on a 32-Bit Processor - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - push EBX; - lea EDI, writeThis; - mov EBX, [EDI]; - mov ECX, 4[EDI]; - lea EDI, ifThis; - mov EAX, [EDI]; - mov EDX, 4[EDI]; - mov EDI, here; - lock; // lock always needed to make this op atomic - cmpxchg8b [EDI]; - setz AL; - pop EBX; - pop EDI; - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - bool cas(T,V1,V2)( shared(T)* here, V1* ifThis, V2 writeThis ) pure nothrow @nogc @safe - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - { - return casImplWithResult(here, *ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, shared(V1)* ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) - { - return casImplWithResult(here, *ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, shared(V1*)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - { - return casImplWithResult(here, *ifThis, writeThis); - } - - private bool casImplWithResult(T,V1,V2)( shared(T)* here, ref V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe - in - { - assert( atomicPtrIsProperlyAligned( here ) ); - } - do - { - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte CAS - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - mov DL, writeThis; - mov EDI, ifThis; - mov AL, [EDI]; - mov ECX, here; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], DL; - mov [EDI], AL; - setz AL; - pop EDI; - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte CAS - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - mov DX, writeThis; - mov EDI, ifThis; - mov AX, [EDI]; - mov ECX, here; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], DX; - mov [EDI], AX; - setz AL; - pop EDI; - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte CAS - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - mov EDX, writeThis; - mov EDI, ifThis; - mov EAX, [EDI]; - mov ECX, here; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], EDX; - mov [EDI], EAX; - setz AL; - pop EDI; - } - } - else static if ( T.sizeof == long.sizeof && has64BitCAS ) - { - - ////////////////////////////////////////////////////////////////// - // 8 Byte CAS on a 32-Bit Processor - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - push EBX; - lea EDI, writeThis; - mov EBX, [EDI]; - mov ECX, 4[EDI]; - mov EDI, ifThis; - mov EAX, [EDI]; - mov EDX, 4[EDI]; - mov EDI, here; - lock; // lock always needed to make this op atomic - cmpxchg8b [EDI]; - mov EDI, ifThis; - mov [EDI], EAX; - mov 4[EDI], EDX; - setz AL; - pop EBX; - pop EDI; - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - - enum MemoryOrder - { - raw, - acq, - rel, - seq, - } - - - private - { - // NOTE: x86 loads implicitly have acquire semantics so a memory - // barrier is only necessary on releases. - template needsLoadBarrier( MemoryOrder ms ) - { - enum bool needsLoadBarrier = ms == MemoryOrder.seq; - } - - - // NOTE: x86 stores implicitly have release semantics so a memory - // barrier is only necessary on acquires. - template needsStoreBarrier( MemoryOrder ms ) - { - enum bool needsStoreBarrier = ms == MemoryOrder.seq; - } - } - - - TailShared!T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @safe - if (!__traits(isFloating, T)) - { - static assert( ms != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()" ); - static assert( __traits(isPOD, T), "argument to atomicLoad() must be POD" ); - - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte Load - ////////////////////////////////////////////////////////////////// - - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov DL, 0; - mov AL, 0; - mov ECX, val; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], DL; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov AL, [EAX]; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte Load - ////////////////////////////////////////////////////////////////// - - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov DX, 0; - mov AX, 0; - mov ECX, val; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], DX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov AX, [EAX]; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte Load - ////////////////////////////////////////////////////////////////// - - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov EDX, 0; - mov EAX, 0; - mov ECX, val; - lock; // lock always needed to make this op atomic - cmpxchg [ECX], EDX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov EAX, [EAX]; - } - } - } - else static if ( T.sizeof == long.sizeof && has64BitCAS ) - { - ////////////////////////////////////////////////////////////////// - // 8 Byte Load on a 32-Bit Processor - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - push EBX; - mov EBX, 0; - mov ECX, 0; - mov EAX, 0; - mov EDX, 0; - mov EDI, val; - lock; // lock always needed to make this op atomic - cmpxchg8b [EDI]; - pop EBX; - pop EDI; - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe - if ( __traits( compiles, { val = newval; } ) ) - { - static assert( ms != MemoryOrder.acq, "invalid MemoryOrder for atomicStore()" ); - static assert( __traits(isPOD, T), "argument to atomicStore() must be POD" ); - - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte Store - ////////////////////////////////////////////////////////////////// - - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov DL, newval; - lock; - xchg [EAX], DL; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov DL, newval; - mov [EAX], DL; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte Store - ////////////////////////////////////////////////////////////////// - - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov DX, newval; - lock; - xchg [EAX], DX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov DX, newval; - mov [EAX], DX; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte Store - ////////////////////////////////////////////////////////////////// - - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov EDX, newval; - lock; - xchg [EAX], EDX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov EAX, val; - mov EDX, newval; - mov [EAX], EDX; - } - } - } - else static if ( T.sizeof == long.sizeof && has64BitCAS ) - { - ////////////////////////////////////////////////////////////////// - // 8 Byte Store on a 32-Bit Processor - ////////////////////////////////////////////////////////////////// - - asm pure nothrow @nogc @trusted - { - push EDI; - push EBX; - lea EDI, newval; - mov EBX, [EDI]; - mov ECX, 4[EDI]; - mov EDI, val; - mov EAX, [EDI]; - mov EDX, 4[EDI]; - L1: lock; // lock always needed to make this op atomic - cmpxchg8b [EDI]; - jne L1; - pop EBX; - pop EDI; - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - - void atomicFence() nothrow @nogc @safe - { - import core.cpuid; - - asm pure nothrow @nogc @trusted - { - naked; - - call sse2; - test AL, AL; - jne Lcpuid; - - // Fast path: We have SSE2, so just use mfence. - mfence; - jmp Lend; - - Lcpuid: - - // Slow path: We use cpuid to serialize. This is - // significantly slower than mfence, but is the - // only serialization facility we have available - // on older non-SSE2 chips. - push EBX; - - mov EAX, 0; - cpuid; - - pop EBX; - - Lend: - - ret; - } - } -} -else version (AsmX86_64) -{ - // Uses specialized asm for fast fetch and add operations - TailShared!(T) atomicFetchAdd(T)( ref shared T val, size_t mod ) pure nothrow @nogc @trusted - if ( __traits(isIntegral, T) ) - in ( atomicValueIsProperlyAligned(val) ) - { - return atomicFetchAddImpl( val, mod ); - } - TailShared!(T) atomicFetchAddImpl(T)( ref shared T val, size_t mod ) pure nothrow @nogc @trusted - { - asm pure nothrow @nogc @trusted { naked; } - version (Windows) - { - asm pure nothrow @nogc @trusted { mov RAX, RCX; } - static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[RDX], AL; } - else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[RDX], AX; } - else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[RDX], EAX; } - else static if (T.sizeof == 8) asm pure nothrow @nogc @trusted { lock; xadd[RDX], RAX; } - } - else - { - asm pure nothrow @nogc @trusted { mov RAX, RDI; } - static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[RSI], AL; } - else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[RSI], AX; } - else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[RSI], EAX; } - else static if (T.sizeof == 8) asm pure nothrow @nogc @trusted { lock; xadd[RSI], RAX; } - } - asm pure nothrow @nogc @trusted { ret; } - } - - TailShared!(T) atomicFetchSub(T)( ref shared T val, size_t mod ) pure nothrow @nogc @safe - if ( __traits(isIntegral, T) ) - in ( atomicValueIsProperlyAligned(val) ) - { - return atomicFetchAddImpl(val, -mod); - } - - TailShared!T atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc - if ( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) ) - in - { - assert( atomicValueIsProperlyAligned(val)); - } - do - { - // binary operators - // - // + - * / % ^^ & - // | ^ << >> >>> ~ in - // == != < <= > >= - static if ( op == "+" || op == "-" || op == "*" || op == "/" || - op == "%" || op == "^^" || op == "&" || op == "|" || - op == "^" || op == "<<" || op == ">>" || op == ">>>" || - op == "~" || // skip "in" - op == "==" || op == "!=" || op == "<" || op == "<=" || - op == ">" || op == ">=" ) - { - TailShared!T get = atomicLoad!(MemoryOrder.raw)( val ); - mixin( "return get " ~ op ~ " mod;" ); - } - else - // assignment operators - // - // += -= *= /= %= ^^= &= - // |= ^= <<= >>= >>>= ~= - static if ( op == "+=" && __traits(isIntegral, T) && __traits(isIntegral, V1)) - { - return cast(T)(atomicFetchAdd!(T)(val, mod) + mod); - } - else static if ( op == "-=" && __traits(isIntegral, T) && __traits(isIntegral, V1)) - { - return cast(T)(atomicFetchSub!(T)(val, mod) - mod); - } - else static if ( op == "+=" || op == "-=" || op == "*=" || op == "/=" || - op == "%=" || op == "^^=" || op == "&=" || op == "|=" || - op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=" ) // skip "~=" - { - TailShared!T get, set; - - do - { - get = set = atomicLoad!(MemoryOrder.raw)( val ); - mixin( "set " ~ op ~ " mod;" ); - } while ( !casByRef( val, get, set ) ); - return set; - } - else - { - static assert( false, "Operation not supported." ); - } - } - - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, V exchangeWith ) pure nothrow @nogc @trusted - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - static if ( __traits(isFloating, V) ) - { - static if ( V.sizeof == 4 ) - alias I = uint; - else static if ( V.sizeof == 8 ) - alias I = ulong; - else - static assert( false, "Float type " ~ V.stringof ~ " not supported."); - I r = atomicExchangeImpl(cast(shared(I)*)here, *cast(I*)&exchangeWith); - return *cast(shared(T)*)&r; - } - else - return atomicExchangeImpl(here, exchangeWith); - } - - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V) exchangeWith ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = exchangeWith; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - return atomicExchangeImpl(here, exchangeWith); - } - - shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)( shared(T)* here, shared(V)* exchangeWith ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = exchangeWith; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - return atomicExchangeImpl(here, exchangeWith); - } - - private shared(T) atomicExchangeImpl(T,V)( shared(T)* here, V exchangeWith ) pure nothrow @nogc @safe - { - // Windows: here = RDX, exchangeWith = RCX - // Posix: here = RSI, exchangeWith = RDI - static if ( T.sizeof == byte.sizeof ) - { - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RDX], CL; - mov AL, CL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RSI], DIL; - mov AL, DIL; - ret; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RDX], CX; - mov AX, CX; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RSI], DI; - mov AX, DI; - ret; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RDX], ECX; - mov EAX, ECX; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RSI], EDI; - mov EAX, EDI; - ret; - } - } - } - else static if ( T.sizeof == long.sizeof ) - { - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RDX], RCX; - mov RAX, RCX; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - xchg [RSI], RDI; - mov RAX, RDI; - ret; - } - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted - { - return cas(&value, ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - static assert (V1.sizeof == V2.sizeof, "Mismatching argument sizes"); - static if ( V2.sizeof == 4 && __traits(isFloating, V2) ) - { - uint cmp = *cast(uint*)&ifThis; - uint arg = *cast(uint*)&writeThis; - } - else static if ( V2.sizeof == 8 && __traits(isFloating, V2) ) - { - ulong cmp = *cast(ulong*)&ifThis; - ulong arg = *cast(ulong*)&writeThis; - } - else - { - alias cmp = ifThis; - alias arg = writeThis; - } - return casImplNoResult(here, cmp, arg); - } - - bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - return casImplNoResult(here, ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - return casImplNoResult(here, ifThis, writeThis); - } - - private bool casImplNoResult(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe - { - // Windows: here = *R8, ifThis = RDX, writeThis = RCX - // Posix: here = *RDX, ifThis = RSI, writeThis = RDI - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte CAS - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov AL, DL; - lock; cmpxchg [R8], CL; - setz AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov AL, SIL; - lock; cmpxchg [RDX], DIL; - setz AL; - ret; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte CAS - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov AX, DX; - lock; cmpxchg [R8], CX; - setz AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov AX, SI; - lock; cmpxchg [RDX], DI; - setz AL; - ret; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte CAS - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov EAX, EDX; - lock; cmpxchg [R8], ECX; - setz AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov EAX, ESI; - lock; cmpxchg [RDX], EDI; - setz AL; - ret; - } - } - } - else static if ( T.sizeof == long.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 8 Byte CAS on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov RAX, RDX; - lock; cmpxchg [R8], RCX; - setz AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov RAX, RSI; - lock; cmpxchg [RDX], RDI; - setz AL; - ret; - } - } - } - else static if ( T.sizeof == long.sizeof*2 && has128BitCAS) - { - ////////////////////////////////////////////////////////////////// - // 16 Byte CAS on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// - - // Windows: here = *R8, ifThis = *RDX, writeThis = *RCX - // Posix: here = *R8, ifThis = RCX:RDX, writeThis = RSI:RDI - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - push RBX; - mov RAX, [RDX]; - mov RDX, 8[RDX]; - mov RBX, [RCX]; - mov RCX, 8[RCX]; - lock; cmpxchg16b [R8]; - setz AL; - pop RBX; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - push RBX; - mov RAX, RDX; - mov RDX, RCX; - mov RBX, RDI; - mov RCX, RSI; - lock; cmpxchg16b [R8]; - setz AL; - pop RBX; - ret; - } - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } - - bool cas(T,V1,V2)( shared(T)* here, V1* ifThis, V2 writeThis ) pure nothrow @nogc @trusted - if ( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - static if ( V2.sizeof == 4 && __traits(isFloating, V2) ) - uint arg = *cast(uint*)&writeThis; - else static if ( V2.sizeof == 8 && __traits(isFloating, V2) ) - ulong arg = *cast(ulong*)&writeThis; - else - alias arg = writeThis; - return casImplWithResult(here, *ifThis, arg); - } - - bool cas(T,V1,V2)( shared(T)* here, shared(V1)* ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe - if ( is(T == class) && __traits( compiles, { *here = writeThis; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - return casImplWithResult(here, *ifThis, writeThis); - } - - bool cas(T,V1,V2)( shared(T)* here, shared(V1*)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe - if ( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) ) - in ( atomicPtrIsProperlyAligned( here ), "Argument `here` is not properly aligned" ) - { - return casImplWithResult(here, *ifThis, writeThis); - } - - private bool casImplWithResult(T,V1,V2)( shared(T)* here, ref V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe - { - // Windows: here = *R8, ifThis = *RDX, writeThis = RCX - // Posix: here = *RDX, ifThis = *RSI, writeThis = RDI - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte CAS - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov AL, [RDX]; - lock; cmpxchg [R8], CL; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RDX], AL; - xor AL, AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov AL, [RSI]; - lock; cmpxchg [RDX], DIL; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RSI], AL; - xor AL, AL; - ret; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte CAS - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov AX, [RDX]; - lock; cmpxchg [R8], CX; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RDX], AX; - xor AL, AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov AX, [RSI]; - lock; cmpxchg [RDX], DI; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RSI], AX; - xor AL, AL; - ret; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte CAS - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov EAX, [RDX]; - lock; cmpxchg [R8], ECX; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RDX], EAX; - xor AL, AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov EAX, [RSI]; - lock; cmpxchg [RDX], EDI; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RSI], EAX; - xor AL, AL; - ret; - } - } - } - else static if ( T.sizeof == long.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 8 Byte CAS on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - mov RAX, [RDX]; - lock; cmpxchg [R8], RCX; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RDX], RAX; - xor AL, AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - mov RAX, [RSI]; - lock; cmpxchg [RDX], RDI; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [RSI], RAX; - xor AL, AL; - ret; - } - } - } - else static if ( T.sizeof == long.sizeof*2 && has128BitCAS) - { - ////////////////////////////////////////////////////////////////// - // 16 Byte CAS on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// + return cast(T)( atomicFetchAdd!(MemoryOrder.seq, T)( val, mod ) + mod ); + } + else static if ( op == "-=" && __traits(isIntegral, T) && __traits(isIntegral, V1) && T.sizeof <= size_t.sizeof && V1.sizeof <= size_t.sizeof) + { + return cast(T)( atomicFetchSub!(MemoryOrder.seq, T)( val, mod ) - mod ); + } + else static if ( op == "+=" || op == "-=" || op == "*=" || op == "/=" || + op == "%=" || op == "^^=" || op == "&=" || op == "|=" || + op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=" ) // skip "~=" + { + TailShared!T get, set; - // Windows: here = *R8, ifThis = *RDX, writeThis = *RCX - // Posix: here = *RCX, ifThis = *RDX, writeThis = RSI:RDI - version (Windows) - { - asm pure nothrow @nogc @trusted - { - naked; - push RBX; - mov R9, RDX; - mov RAX, [RDX]; - mov RDX, 8[RDX]; - mov RBX, [RCX]; - mov RCX, 8[RCX]; - lock; cmpxchg16b [R8]; - pop RBX; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [R9], RAX; - mov 8[R9], RDX; - xor AL, AL; - ret; - } - } - else - { - asm pure nothrow @nogc @trusted - { - naked; - push RBX; - mov R8, RCX; - mov R9, RDX; - mov RAX, [RDX]; - mov RDX, 8[RDX]; - mov RBX, RDI; - mov RCX, RSI; - lock; cmpxchg16b [R8]; - pop RBX; - jne compare_fail; - mov AL, 1; - ret; - compare_fail: - mov [R9], RAX; - mov 8[R9], RDX; - xor AL, AL; - ret; - } - } - } - else + do { - static assert( false, "Invalid template type specified." ); - } + get = set = atomicLoad!(MemoryOrder.raw)( val ); + mixin( "set " ~ op ~ " mod;" ); + } while ( !casByRef( val, get, set ) ); + return set; } - - - enum MemoryOrder + else { - raw, - acq, - rel, - seq, + static assert( false, "Operation not supported." ); } +} - private +version (X86) +{ + version = IsX86; + enum has64BitXCHG = false; + enum has64BitCAS = true; + enum has128BitCAS = false; +} +else version (X86_64) +{ + version = IsX86; + enum has64BitXCHG = true; + enum has64BitCAS = true; + enum has128BitCAS = true; +} +else +{ + enum has64BitXCHG = false; + enum has64BitCAS = false; + enum has128BitCAS = false; +} + +private +{ + version (IsX86) { - // NOTE: x86 loads implicitly have acquire semantics so a memory - // barrier is only necessary on releases. - template needsLoadBarrier( MemoryOrder ms ) + // NOTE: Strictly speaking, the x86 supports atomic operations on + // unaligned values. However, this is far slower than the + // common case, so such behavior should be prohibited. + bool atomicValueIsProperlyAligned(T)( ref T val ) pure nothrow @nogc @trusted { - enum bool needsLoadBarrier = ms == MemoryOrder.seq; + return atomicPtrIsProperlyAligned(&val); } - - // NOTE: x86 stores implicitly have release semantics so a memory - // barrier is only necessary on acquires. - template needsStoreBarrier( MemoryOrder ms ) + bool atomicPtrIsProperlyAligned(T)( T* ptr ) pure nothrow @nogc @safe { - enum bool needsStoreBarrier = ms == MemoryOrder.seq; + // NOTE: 32 bit x86 systems support 8 byte CAS, which only requires + // 4 byte alignment, so use size_t as the align type here. + static if ( T.sizeof > size_t.sizeof ) + return cast(size_t)ptr % size_t.sizeof == 0; + else + return cast(size_t)ptr % T.sizeof == 0; } } + template IntForFloat(F) + if (__traits(isFloating, F)) + { + static if ( F.sizeof == 4 ) + alias IntForFloat = uint; + else static if ( F.sizeof == 8 ) + alias IntForFloat = ulong; + else + static assert ( false, "Invalid floating point type: " ~ F.stringof ~ ", only support `float` and `double`." ); + } + + template IntForStruct(S) + if (is(S == struct)) + { + static if ( S.sizeof == 1 ) + alias IntForFloat = ubyte; + else static if ( F.sizeof == 2 ) + alias IntForFloat = ushort; + else static if ( F.sizeof == 4 ) + alias IntForFloat = uint; + else static if ( F.sizeof == 8 ) + alias IntForFloat = ulong; + else static if ( F.sizeof == 16 ) + alias IntForFloat = ulong[2]; // TODO: what's the best type here? slice/delegates pass in registers... + else + static assert (ValidateStruct!S); + } - TailShared!T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @safe - if (!__traits(isFloating, T)) + template ValidateStruct(S) + if (is(S == struct)) { - static assert( ms != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()" ); - static assert( __traits(isPOD, T), "argument to atomicLoad() must be POD" ); + import core.internal.traits : hasElaborateAssign; - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte Load - ////////////////////////////////////////////////////////////////// + static assert (S.sizeof <= size_t*2 && (S.sizeof & (S.sizeof - 1)) == 0, S.stringof ~ " has invalid size for atomic operations."); + static assert (!hasElaborateAssign!S, S.stringof ~ " may not have an elaborate assignment when used with atomic operations."); - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov DL, 0; - mov AL, 0; - mov RCX, val; - lock; // lock always needed to make this op atomic - cmpxchg [RCX], DL; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov AL, [RAX]; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte Load - ////////////////////////////////////////////////////////////////// + enum ValidateStruct = true; + } - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov DX, 0; - mov AX, 0; - mov RCX, val; - lock; // lock always needed to make this op atomic - cmpxchg [RCX], DX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov AX, [RAX]; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte Load - ////////////////////////////////////////////////////////////////// + // TODO: it'd be nice if we had @trusted scopes; we could remove this... + bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted + { + return cas( &value, ifThis, writeThis ); + } - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov EDX, 0; - mov EAX, 0; - mov RCX, val; - lock; // lock always needed to make this op atomic - cmpxchg [RCX], EDX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov EAX, [RAX]; - } - } - } - else static if ( T.sizeof == long.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 8 Byte Load - ////////////////////////////////////////////////////////////////// + /* Construct a type with a shared tail, and if possible with an unshared + head. */ + template TailShared(U) if (!is(U == shared)) + { + alias TailShared = .TailShared!(shared U); + } + template TailShared(S) if (is(S == shared)) + { + // Get the unshared variant of S. + static if (is(S U == shared U)) {} + else static assert(false, "Should never be triggered. The `static " ~ + "if` declares `U` as the unshared version of the shared type " ~ + "`S`. `S` is explicitly declared as shared, so getting `U` " ~ + "should always work."); - static if ( needsLoadBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov RDX, 0; - mov RAX, 0; - mov RCX, val; - lock; // lock always needed to make this op atomic - cmpxchg [RCX], RDX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov RAX, [RAX]; - } - } - } - else static if ( T.sizeof == long.sizeof*2 && has128BitCAS ) + static if (is(S : U)) + alias TailShared = U; + else static if (is(S == struct)) { - ////////////////////////////////////////////////////////////////// - // 16 Byte Load on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// - version (Win64){ - size_t[2] retVal; - asm pure nothrow @nogc @trusted + enum implName = () { + /* Start with "_impl". If S has a field with that name, append + underscores until the clash is resolved. */ + string name = "_impl"; + string[] fieldNames; + static foreach (alias field; S.tupleof) { - push RDI; - push RBX; - mov RDI, val; - mov RBX, 0; - mov RCX, 0; - mov RAX, 0; - mov RDX, 0; - lock; // lock always needed to make this op atomic - cmpxchg16b [RDI]; - lea RDI, retVal; - mov [RDI], RAX; - mov 8[RDI], RDX; - pop RBX; - pop RDI; + fieldNames ~= __traits(identifier, field); } - - static if (is(T:U[], U)) + static bool canFind(string[] haystack, string needle) { - pragma(inline, true) - static typeof(return) toTrusted(size_t[2] retVal) @trusted + foreach (candidate; haystack) { - return *(cast(typeof(return)*) retVal.ptr); + if (candidate == needle) return true; } - - return toTrusted(retVal); - } - else - { - return cast(typeof(return)) retVal; + return false; } - }else{ - asm pure nothrow @nogc @trusted + while (canFind(fieldNames, name)) name ~= "_"; + return name; + } (); + struct TailShared + { + static foreach (i, alias field; S.tupleof) { - push RDI; - push RBX; - mov RBX, 0; - mov RCX, 0; - mov RAX, 0; - mov RDX, 0; - mov RDI, val; - lock; // lock always needed to make this op atomic - cmpxchg16b [RDI]; - pop RBX; - pop RDI; + /* On @trusted: This is casting the field from shared(Foo) + to TailShared!Foo. The cast is safe because the field has + been loaded and is not shared anymore. */ + mixin(" + @trusted @property + ref " ~ __traits(identifier, field) ~ "() + { + alias R = TailShared!(typeof(field)); + return * cast(R*) &" ~ implName ~ ".tupleof[i]; + } + "); } + mixin(" + S " ~ implName ~ "; + alias " ~ implName ~ " this; + "); } } else - { - static assert( false, "Invalid template type specified." ); - } + alias TailShared = S; } + @safe unittest + { + // No tail (no indirections) -> fully unshared. + static assert(is(TailShared!int == int)); + static assert(is(TailShared!(shared int) == int)); - void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe - if ( __traits( compiles, { val = newval; } ) ) - { - static assert( ms != MemoryOrder.acq, "invalid MemoryOrder for atomicStore()" ); - static assert( __traits(isPOD, T), "argument to atomicStore() must be POD" ); + static struct NoIndir { int i; } + static assert(is(TailShared!NoIndir == NoIndir)); + static assert(is(TailShared!(shared NoIndir) == NoIndir)); - static if ( T.sizeof == byte.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 1 Byte Store - ////////////////////////////////////////////////////////////////// + // Tail can be independently shared or is already -> tail-shared. - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov DL, newval; - lock; - xchg [RAX], DL; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov DL, newval; - mov [RAX], DL; - } - } - } - else static if ( T.sizeof == short.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 2 Byte Store - ////////////////////////////////////////////////////////////////// + static assert(is(TailShared!(int*) == shared(int)*)); + static assert(is(TailShared!(shared int*) == shared(int)*)); + static assert(is(TailShared!(shared(int)*) == shared(int)*)); - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov DX, newval; - lock; - xchg [RAX], DX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov DX, newval; - mov [RAX], DX; - } - } - } - else static if ( T.sizeof == int.sizeof ) - { - ////////////////////////////////////////////////////////////////// - // 4 Byte Store - ////////////////////////////////////////////////////////////////// + static assert(is(TailShared!(int[]) == shared(int)[])); + static assert(is(TailShared!(shared int[]) == shared(int)[])); + static assert(is(TailShared!(shared(int)[]) == shared(int)[])); - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov EDX, newval; - lock; - xchg [RAX], EDX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov EDX, newval; - mov [RAX], EDX; - } - } - } - else static if ( T.sizeof == long.sizeof && has64BitCAS ) - { - ////////////////////////////////////////////////////////////////// - // 8 Byte Store on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// + static struct S1 { shared int* p; } + static assert(is(TailShared!S1 == S1)); + static assert(is(TailShared!(shared S1) == S1)); - static if ( needsStoreBarrier!(ms) ) - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov RDX, newval; - lock; - xchg [RAX], RDX; - } - } - else - { - asm pure nothrow @nogc @trusted - { - mov RAX, val; - mov RDX, newval; - mov [RAX], RDX; - } - } - } - else static if ( T.sizeof == long.sizeof*2 && has128BitCAS ) - { - ////////////////////////////////////////////////////////////////// - // 16 Byte Store on a 64-Bit Processor - ////////////////////////////////////////////////////////////////// - version (Win64){ - asm pure nothrow @nogc @trusted - { - push RDI; - push RBX; - mov R9, val; - mov R10, newval; - - mov RDI, R10; - mov RBX, [RDI]; - mov RCX, 8[RDI]; - - mov RDI, R9; - mov RAX, [RDI]; - mov RDX, 8[RDI]; - - L1: lock; // lock always needed to make this op atomic - cmpxchg16b [RDI]; - jne L1; - pop RBX; - pop RDI; - } - }else{ - asm pure nothrow @nogc @trusted - { - push RDI; - push RBX; - lea RDI, newval; - mov RBX, [RDI]; - mov RCX, 8[RDI]; - mov RDI, val; - mov RAX, [RDI]; - mov RDX, 8[RDI]; - L1: lock; // lock always needed to make this op atomic - cmpxchg16b [RDI]; - jne L1; - pop RBX; - pop RDI; - } - } - } - else - { - static assert( false, "Invalid template type specified." ); - } - } + static struct S2 { shared(int)* p; } + static assert(is(TailShared!S2 == S2)); + static assert(is(TailShared!(shared S2) == S2)); + // Tail follows shared-ness of head -> fully shared. - void atomicFence() nothrow @nogc @safe - { - // SSE2 is always present in 64-bit x86 chips. - asm nothrow @nogc @trusted - { - naked; + static class C { int i; } + static assert(is(TailShared!C == shared C)); + static assert(is(TailShared!(shared C) == shared C)); - mfence; - ret; - } - } -} + /* However, structs get a wrapper that has getters which cast to + TailShared. */ -// This is an ABI adapter that works on all architectures. It type puns -// floats and doubles to ints and longs, atomically loads them, then puns -// them back. This is necessary so that they get returned in floating -// point instead of integer registers. -TailShared!T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @trusted -if (__traits(isFloating, T)) -{ - static if (T.sizeof == int.sizeof) - { - static assert(is(T : float)); - auto ptr = cast(const shared int*) &val; - auto asInt = atomicLoad!(ms)(*ptr); - return *(cast(typeof(return)*) &asInt); - } - else static if (T.sizeof == long.sizeof) - { - static assert(is(T : double)); - auto ptr = cast(const shared long*) &val; - auto asLong = atomicLoad!(ms)(*ptr); - return *(cast(typeof(return)*) &asLong); - } - else - { - static assert(0, "Cannot atomically load 80-bit reals."); + static struct S3 { int* p; int _impl; int _impl_; int _impl__; } + static assert(!is(TailShared!S3 : S3)); + static assert(is(TailShared!S3 : shared S3)); + static assert(is(TailShared!(shared S3) == TailShared!S3)); + + static struct S4 { shared(int)** p; } + static assert(!is(TailShared!S4 : S4)); + static assert(is(TailShared!S4 : shared S4)); + static assert(is(TailShared!(shared S4) == TailShared!S4)); } } + //////////////////////////////////////////////////////////////////////////////// // Unit Tests //////////////////////////////////////////////////////////////////////////////// @@ -2148,7 +635,7 @@ version (unittest) atom = cast(shared(T))null; - T arg = base; + shared(T) arg = base; assert( cas( &atom, &arg, val ), T.stringof ); assert( arg is base, T.stringof ); assert( atom is val, T.stringof ); @@ -2218,7 +705,7 @@ version (unittest) { () @trusted { - struct Big { long a, b; } + align(16) struct Big { long a, b; } shared(Big) atom; shared(Big) base; @@ -2380,7 +867,7 @@ version (unittest) assert(atomicOp!"+="(i8, 8) == 13); assert(atomicOp!"+="(i16, 8) == 14); assert(atomicOp!"+="(i32, 8) == 15); - version (AsmX86_64) + version (D_LP64) { shared ulong u64 = 4; shared long i64 = 8; @@ -2404,7 +891,7 @@ version (unittest) assert(atomicOp!"-="(i8, 1) == 4); assert(atomicOp!"-="(i16, 1) == 5); assert(atomicOp!"-="(i32, 1) == 6); - version (AsmX86_64) + version (D_LP64) { shared ulong u64 = 4; shared long i64 = 8; diff --git a/src/core/internal/atomic.d b/src/core/internal/atomic.d new file mode 100644 index 0000000000..b8e10c8271 --- /dev/null +++ b/src/core/internal/atomic.d @@ -0,0 +1,734 @@ +/** +* The core.internal.atomic module comtains the low-level atomic features available in hardware. +* This module may be a routing layer for compiler intrinsics. +* +* Copyright: Copyright Manu Evans 2019. +* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) +* Authors: Sean Kelly, Alex Rønne Petersen, Manu Evans +* Source: $(DRUNTIMESRC core/internal/_atomic.d) +*/ + +module core.internal.atomic; + +import core.atomic : MemoryOrder; + +private +{ + enum : int + { + AX, BX, CX, DX, DI, SI, R8, R9 + } + + immutable string[4][8] registerNames = [ + [ "AL", "AX", "EAX", "RAX" ], + [ "BL", "BX", "EBX", "RBX" ], + [ "CL", "CX", "ECX", "RCX" ], + [ "DL", "DX", "EDX", "RDX" ], + [ "DIL", "DI", "EDI", "RDI" ], + [ "SIL", "SI", "ESI", "RSI" ], + [ "R8B", "R8W", "R8D", "R8" ], + [ "R9B", "R9W", "R9D", "R9" ], + ]; + + template RegIndex(T) + { + static if (T.sizeof == 1) + enum RegIndex = 0; + else static if (T.sizeof == 2) + enum RegIndex = 1; + else static if (T.sizeof == 4) + enum RegIndex = 2; + else static if (T.sizeof == 8) + enum RegIndex = 3; + else + static assert(false, "Invalid type"); + } + + enum SizedReg(int reg, T = size_t) = registerNames[reg][RegIndex!T]; +} + +inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted + if (CanCAS!T) +{ + static assert(order != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()"); + + static if (T.sizeof == size_t.sizeof * 2) + { + version (D_InlineAsm_X86) + { + asm pure nothrow @nogc @trusted + { + push EDI; + push EBX; + mov EBX, 0; + mov ECX, 0; + mov EAX, 0; + mov EDX, 0; + mov EDI, src; + lock; cmpxchg8b [EDI]; + pop EBX; + pop EDI; + } + } + else version (D_InlineAsm_X86_64) + { + version (Windows) + { + static if (RegisterReturn!T) + { + enum SrcPtr = SizedReg!CX; + enum RetPtr = null; + } + else + { + enum SrcPtr = SizedReg!DX; + enum RetPtr = SizedReg!CX; + } + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov R8, %0; +?1 mov R9, %1; + mov RBX, 0; + mov RCX, 0; + mov RAX, 0; + mov RDX, 0; + lock; cmpxchg16b [R8]; +?1 mov [R9], RAX; +?1 mov 8[R9], RDX; + pop RBX; + ret; + } + }, SrcPtr, RetPtr)); + } + else + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov RBX, 0; + mov RCX, 0; + mov RAX, 0; + mov RDX, 0; + lock; cmpxchg16b [RDI]; + pop RBX; + ret; + } + } + } + } + else static if (needsLoadBarrier!order) + { + version (D_InlineAsm_X86) + { + enum SrcReg = SizedReg!CX; + enum ZeroReg = SizedReg!(DX, T); + enum ResReg = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + mov %1, 0; + mov %2, 0; + mov %0, src; + lock; cmpxchg [%0], %1; + } + }, SrcReg, ZeroReg, ResReg)); + } + else version (D_InlineAsm_X86_64) + { + version (Windows) + enum SrcReg = SizedReg!CX; + else + enum SrcReg = SizedReg!DI; + enum ZeroReg = SizedReg!(DX, T); + enum ResReg = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + naked; + mov %1, 0; + mov %2, 0; + lock; cmpxchg [%0], %1; + ret; + } + }, SrcReg, ZeroReg, ResReg)); + } + } + else + return *src; +} + +void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @safe + if (CanCAS!T) +{ + static assert(order != MemoryOrder.acq, "Invalid MemoryOrder for atomicStore()"); + + static if (T.sizeof == size_t.sizeof * 2) + { + version (D_InlineAsm_X86) + { + asm pure nothrow @nogc @trusted + { + push EDI; + push EBX; + lea EDI, value; + mov EBX, [EDI]; + mov ECX, 4[EDI]; + mov EDI, dest; + mov EAX, [EDI]; + mov EDX, 4[EDI]; + L1: lock; cmpxchg8b [EDI]; + jne L1; + pop EBX; + pop EDI; + } + } + else version (D_InlineAsm_X86_64) + { + version (Windows) + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov R8, RDX; + mov RAX, [RDX]; + mov RDX, 8[RDX]; + mov RBX, [RCX]; + mov RCX, 8[RCX]; + L1: lock; cmpxchg16b [R8]; + jne L1; + pop RBX; + ret; + } + } + else + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov RBX, RDI; + mov RCX, RSI; + mov RDI, RDX; + mov RAX, [RDX]; + mov RDX, 8[RDX]; + L1: lock; cmpxchg16b [RDI]; + jne L1; + pop RBX; + ret; + } + } + } + } + else static if (needsStoreBarrier!order) + atomicExchange!(order, false)(dest, value); + else + *dest = value; +} + +T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @safe + if (is(T : ulong)) +{ + version (D_InlineAsm_X86) + { + static assert(T.sizeof <= 4, "64bit atomicFetchAdd not supported on 32bit target." ); + + enum DestReg = SizedReg!DX; + enum ValReg = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + mov %1, value; + mov %0, dest; + lock; xadd[%0], %1; + } + }, DestReg, ValReg)); + } + else version (D_InlineAsm_X86_64) + { + version (Windows) + { + enum DestReg = SizedReg!DX; + enum ValReg = SizedReg!(CX, T); + } + else + { + enum DestReg = SizedReg!SI; + enum ValReg = SizedReg!(DI, T); + } + enum ResReg = result ? SizedReg!(AX, T) : null; + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + naked; + lock; xadd[%0], %1; +?2 mov %2, %1; + ret; + } + }, DestReg, ValReg, ResReg)); + } + else + static assert (false, "Unsupported architecture."); +} + +T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @safe + if (is(T : ulong)) +{ + return atomicFetchAdd(dest, cast(T)-cast(IntOrLong!T)value); +} + +T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @safe + if (is(T : ulong) || is(T == class) || is(T U : U*)) +{ + version (D_InlineAsm_X86) + { + static assert(T.sizeof <= 4, "64bit atomicExchange not supported on 32bit target." ); + + enum DestReg = SizedReg!CX; + enum ValReg = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + mov %1, value; + mov %0, dest; + xchg [%0], %1; + } + }, DestReg, ValReg)); + } + else version (D_InlineAsm_X86_64) + { + version (Windows) + { + enum DestReg = SizedReg!DX; + enum ValReg = SizedReg!(CX, T); + } + else + { + enum DestReg = SizedReg!SI; + enum ValReg = SizedReg!(DI, T); + } + enum ResReg = result ? SizedReg!(AX, T) : null; + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + naked; + xchg [%0], %1; +?2 mov %2, %1; + ret; + } + }, DestReg, ValReg, ResReg)); + } + else + static assert (false, "Unsupported architecture."); +} + +alias atomicCompareExchangeWeak = atomicCompareExchangeStrong; + +bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @safe + if (CanCAS!T) +{ + version (D_InlineAsm_X86) + { + static if (T.sizeof <= 4) + { + enum DestAddr = SizedReg!CX; + enum CmpAddr = SizedReg!DI; + enum Val = SizedReg!(DX, T); + enum Cmp = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + push %1; + mov %2, value; + mov %1, compare; + mov %3, [%1]; + mov %0, dest; + lock; cmpxchg [%0], %2; + mov [%1], %3; + setz AL; + pop %1; + } + }, DestAddr, CmpAddr, Val, Cmp)); + } + else static if (T.sizeof == 8) + { + asm pure nothrow @nogc @trusted + { + push EDI; + push EBX; + lea EDI, value; + mov EBX, [EDI]; + mov ECX, 4[EDI]; + mov EDI, compare; + mov EAX, [EDI]; + mov EDX, 4[EDI]; + mov EDI, dest; + lock; cmpxchg8b [EDI]; + mov EDI, compare; + mov [EDI], EAX; + mov 4[EDI], EDX; + setz AL; + pop EBX; + pop EDI; + } + } + else + static assert(T.sizeof <= 8, "128bit atomicCompareExchangeStrong not supported on 32bit target." ); + } + else version (D_InlineAsm_X86_64) + { + static if (T.sizeof <= 8) + { + version (Windows) + { + enum DestAddr = SizedReg!R8; + enum CmpAddr = SizedReg!DX; + enum Val = SizedReg!(CX, T); + } + else + { + enum DestAddr = SizedReg!DX; + enum CmpAddr = SizedReg!SI; + enum Val = SizedReg!(DI, T); + } + enum Res = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + naked; + mov %3, [%1]; + lock; cmpxchg [%0], %2; + jne compare_fail; + mov AL, 1; + ret; + compare_fail: + mov [%1], %3; + xor AL, AL; + ret; + } + }, DestAddr, CmpAddr, Val, Res)); + } + else + { + version (Windows) + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov R9, RDX; + mov RAX, [RDX]; + mov RDX, 8[RDX]; + mov RBX, [RCX]; + mov RCX, 8[RCX]; + lock; cmpxchg16b [R8]; + pop RBX; + jne compare_fail; + mov AL, 1; + ret; + compare_fail: + mov [R9], RAX; + mov 8[R9], RDX; + xor AL, AL; + ret; + } + } + else + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov R8, RCX; + mov R9, RDX; + mov RAX, [RDX]; + mov RDX, 8[RDX]; + mov RBX, RDI; + mov RCX, RSI; + lock; cmpxchg16b [R8]; + pop RBX; + jne compare_fail; + mov AL, 1; + ret; + compare_fail: + mov [R9], RAX; + mov 8[R9], RDX; + xor AL, AL; + ret; + } + } + } + } + else + static assert (false, "Unsupported architecture."); +} + +bool atomicCompareExchangeStrongNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @safe + if (CanCAS!T) +{ + version (D_InlineAsm_X86) + { + static if (T.sizeof <= 4) + { + enum DestAddr = SizedReg!CX; + enum Cmp = SizedReg!(AX, T); + enum Val = SizedReg!(DX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + mov %2, value; + mov %1, compare; + mov %0, dest; + lock; cmpxchg [%0], %2; + setz AL; + } + }, DestAddr, Cmp, Val)); + } + else static if (T.sizeof == 8) + { + asm pure nothrow @nogc @trusted + { + push EDI; + push EBX; + lea EDI, value; + mov EBX, [EDI]; + mov ECX, 4[EDI]; + lea EDI, compare; + mov EAX, [EDI]; + mov EDX, 4[EDI]; + mov EDI, dest; + lock; cmpxchg8b [EDI]; + setz AL; + pop EBX; + pop EDI; + } + } + else + static assert(T.sizeof <= 8, "128bit atomicCompareExchangeStrong not supported on 32bit target." ); + } + else version (D_InlineAsm_X86_64) + { + static if (T.sizeof <= 8) + { + version (Windows) + { + enum DestAddr = SizedReg!R8; + enum Cmp = SizedReg!(DX, T); + enum Val = SizedReg!(CX, T); + } + else + { + enum DestAddr = SizedReg!DX; + enum Cmp = SizedReg!(SI, T); + enum Val = SizedReg!(DI, T); + } + enum AXReg = SizedReg!(AX, T); + + mixin (simpleFormat(q{ + asm pure nothrow @nogc @trusted + { + naked; + mov %3, %1; + lock; cmpxchg [%0], %2; + setz AL; + ret; + } + }, DestAddr, Cmp, Val, AXReg)); + } + else + { + version (Windows) + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov RAX, [RDX]; + mov RDX, 8[RDX]; + mov RBX, [RCX]; + mov RCX, 8[RCX]; + lock; cmpxchg16b [R8]; + setz AL; + pop RBX; + ret; + } + } + else + { + asm pure nothrow @nogc @trusted + { + naked; + push RBX; + mov RAX, RDX; + mov RDX, RCX; + mov RBX, RDI; + mov RCX, RSI; + lock; cmpxchg16b [R8]; + setz AL; + pop RBX; + ret; + } + } + } + } + else + static assert (false, "Unsupported architecture."); +} + +void atomicFence(MemoryOrder order = MemoryOrder.seq)() nothrow @nogc @safe +{ + // TODO: `mfence` should only be required for seq_cst operations, but this depends on + // the compiler's backend knowledge to not reorder code inappropriately, + // so we'll apply it conservatively. + static if (order != MemoryOrder.raw) + { + version (D_InlineAsm_X86) + { + import core.cpuid; + + // TODO: review this implementation; it seems way overly complicated + asm pure nothrow @nogc @trusted + { + naked; + + call sse2; + test AL, AL; + jne Lcpuid; + + // Fast path: We have SSE2, so just use mfence. + mfence; + jmp Lend; + + Lcpuid: + + // Slow path: We use cpuid to serialize. This is + // significantly slower than mfence, but is the + // only serialization facility we have available + // on older non-SSE2 chips. + push EBX; + + mov EAX, 0; + cpuid; + + pop EBX; + + Lend: + + ret; + } + } + else version (D_InlineAsm_X86_64) + { + asm nothrow @nogc @trusted + { + naked; + mfence; + ret; + } + } + } + else + static assert (false, "Unsupported architecture."); +} + + +private: + +version (Windows) +{ + enum RegisterReturn(T) = is(T : U[], U) || is(T : R delegate(A), R, A...); +} + +enum CanCAS(T) = is(T : ulong) || + is(T == class) || + is(T : U*, U) || + is(T : U[], U) || + is(T : R delegate(A), R, A...) || + (is(T == struct) && __traits(isPOD, T) && + T.sizeof <= size_t.sizeof*2 && // no more than 2 words + (T.sizeof & (T.sizeof - 1)) == 0 // is power of 2 + ); + +template IntOrLong(T) +{ + static if (T.sizeof > 4) + alias IntOrLong = long; + else + alias IntOrLong = int; +} + +// NOTE: x86 loads implicitly have acquire semantics so a memory +// barrier is only necessary on releases. +template needsLoadBarrier( MemoryOrder ms ) +{ + enum bool needsLoadBarrier = ms == MemoryOrder.seq; +} + + +// NOTE: x86 stores implicitly have release semantics so a memory +// barrier is only necessary on acquires. +template needsStoreBarrier( MemoryOrder ms ) +{ + enum bool needsStoreBarrier = ms == MemoryOrder.seq; +} + +// this is a helper to build asm blocks +string simpleFormat(string format, string[] args...) +{ + string result; + outer: while (format.length) + { + foreach (i; 0 .. format.length) + { + if (format[i] == '%' || format[i] == '?') + { + bool isQ = format[i] == '?'; + result ~= format[0 .. i++]; + assert (i < format.length, "Invalid format string"); + if (format[i] == '%' || format[i] == '?') + { + assert(!isQ, "Invalid format string"); + result ~= format[i++]; + } + else + { + int index = 0; + assert (format[i] >= '0' && format[i] <= '9', "Invalid format string"); + while (i < format.length && format[i] >= '0' && format[i] <= '9') + index = index * 10 + (ubyte(format[i++]) - ubyte('0')); + if (!isQ) + result ~= args[index]; + else if (!args[index]) + { + size_t j = i; + for (; j < format.length;) + { + if (format[j++] == '\n') + break; + } + i = j; + } + } + format = format[i .. $]; + continue outer; + } + } + result ~= format; + break; + } + return result; +} diff --git a/src/core/internal/traits.d b/src/core/internal/traits.d index bccf1ad356..30519dc276 100644 --- a/src/core/internal/traits.d +++ b/src/core/internal/traits.d @@ -8,23 +8,22 @@ */ module core.internal.traits; -/// taken from std.typetuple.TypeTuple -template TypeTuple(TList...) -{ - alias TypeTuple = TList; -} -alias AliasSeq = TypeTuple; -template FieldTypeTuple(T) +// TODO: deprecate these old names...? +alias TypeTuple = AliasSeq; +alias FieldTypeTuple = Fields; + + +alias AliasSeq(TList...) = TList; + +template Fields(T) { static if (is(T == struct) || is(T == union)) - alias FieldTypeTuple = typeof(T.tupleof[0 .. $ - __traits(isNested, T)]); + alias Fields = typeof(T.tupleof[0 .. $ - __traits(isNested, T)]); else static if (is(T == class)) - alias FieldTypeTuple = typeof(T.tupleof); + alias Fields = typeof(T.tupleof); else - { - alias FieldTypeTuple = TypeTuple!T; - } + alias Fields = AliasSeq!T; } T trustedCast(T, U)(auto ref U u) @trusted pure nothrow @@ -66,6 +65,20 @@ template Unqual(T) } } +// [For internal use] +package template ModifyTypePreservingTQ(alias Modifier, T) +{ + static if (is(T U == immutable U)) alias ModifyTypePreservingTQ = immutable Modifier!U; + else static if (is(T U == shared inout const U)) alias ModifyTypePreservingTQ = shared inout const Modifier!U; + else static if (is(T U == shared inout U)) alias ModifyTypePreservingTQ = shared inout Modifier!U; + else static if (is(T U == shared const U)) alias ModifyTypePreservingTQ = shared const Modifier!U; + else static if (is(T U == shared U)) alias ModifyTypePreservingTQ = shared Modifier!U; + else static if (is(T U == inout const U)) alias ModifyTypePreservingTQ = inout const Modifier!U; + else static if (is(T U == inout U)) alias ModifyTypePreservingTQ = inout Modifier!U; + else static if (is(T U == const U)) alias ModifyTypePreservingTQ = const Modifier!U; + else alias ModifyTypePreservingTQ = Modifier!T; +} + // Substitute all `inout` qualifiers that appears in T to `const` template substInout(T) { @@ -187,12 +200,12 @@ template allSatisfy(alias F, T...) } // taken from std.meta.anySatisfy -template anySatisfy(alias F, T...) +template anySatisfy(alias F, Ts...) { - static foreach (Ti; T) + static foreach (T; Ts) { static if (!is(typeof(anySatisfy) == bool) && // not yet defined - F!(Ti)) + F!T) { enum anySatisfy = true; } @@ -220,17 +233,6 @@ template maxAlignment(U...) } } -// std.traits.Fields -template Fields(T) -{ - static if (is(T == struct) || is(T == union)) - alias Fields = typeof(T.tupleof[0 .. $ - __traits(isNested, T)]); - else static if (is(T == class)) - alias Fields = typeof(T.tupleof); - else - alias Fields = TypeTuple!T; -} - /// See $(REF hasElaborateMove, std,traits) template hasElaborateMove(S) { @@ -303,6 +305,110 @@ template hasElaborateAssign(S) } } +template hasIndirections(T) +{ + static if (is(T == struct) || is(T == union)) + enum hasIndirections = anySatisfy!(.hasIndirections, Fields!T); + else static if (__traits(isStaticArray, T) && is(T : E[N], E, size_t N)) + enum hasIndirections = is(E == void) ? true : hasIndirections!E; + else static if (isFunctionPointer!T) + enum hasIndirections = false; + else + enum hasIndirections = isPointer!T || isDelegate!T || isDynamicArray!T || + __traits(isAssociativeArray, T) || is (T == class) || is(T == interface); +} + +template hasUnsharedIndirections(T) +{ + static if (is(T == struct) || is(T == union)) + enum hasUnsharedIndirections = anySatisfy!(.hasUnsharedIndirections, Fields!T); + else static if (is(T : E[N], E, size_t N)) + enum hasUnsharedIndirections = is(E == void) ? false : hasUnsharedIndirections!E; + else static if (isFunctionPointer!T) + enum hasUnsharedIndirections = false; + else static if (isPointer!T) + enum hasUnsharedIndirections = !is(T : shared(U)*, U); + else static if (isDynamicArray!T) + enum hasUnsharedIndirections = !is(T : shared(V)[], V); + else static if (is(T == class) || is(T == interface)) + enum hasUnsharedIndirections = !is(T : shared(W), W); + else + enum hasUnsharedIndirections = isDelegate!T || __traits(isAssociativeArray, T); // TODO: how to handle these? +} + +enum bool isAggregateType(T) = is(T == struct) || is(T == union) || + is(T == class) || is(T == interface); + +enum bool isPointer(T) = is(T == U*, U) && !isAggregateType!T; + +enum bool isDynamicArray(T) = is(DynamicArrayTypeOf!T) && !isAggregateType!T; + +template OriginalType(T) +{ + template Impl(T) + { + static if (is(T U == enum)) alias Impl = OriginalType!U; + else alias Impl = T; + } + + alias OriginalType = ModifyTypePreservingTQ!(Impl, T); +} + +template DynamicArrayTypeOf(T) +{ + static if (is(AliasThisTypeOf!T AT) && !is(AT[] == AT)) + alias X = DynamicArrayTypeOf!AT; + else + alias X = OriginalType!T; + + static if (is(Unqual!X : E[], E) && !is(typeof({ enum n = X.length; }))) + alias DynamicArrayTypeOf = X; + else + static assert(0, T.stringof ~ " is not a dynamic array"); +} + +private template AliasThisTypeOf(T) + if (isAggregateType!T) +{ + alias members = __traits(getAliasThis, T); + + static if (members.length == 1) + alias AliasThisTypeOf = typeof(__traits(getMember, T.init, members[0])); + else + static assert(0, T.stringof~" does not have alias this type"); +} + +template isFunctionPointer(T...) + if (T.length == 1) +{ + static if (is(T[0] U) || is(typeof(T[0]) U)) + { + static if (is(U F : F*) && is(F == function)) + enum bool isFunctionPointer = true; + else + enum bool isFunctionPointer = false; + } + else + enum bool isFunctionPointer = false; +} + +template isDelegate(T...) + if (T.length == 1) +{ + static if (is(typeof(& T[0]) U : U*) && is(typeof(& T[0]) U == delegate)) + { + // T is a (nested) function symbol. + enum bool isDelegate = true; + } + else static if (is(T[0] W) || is(typeof(T[0]) W)) + { + // T is an expression or a type. Take the type of it and examine. + enum bool isDelegate = is(W == delegate); + } + else + enum bool isDelegate = false; +} + // std.meta.Filter template Filter(alias pred, TList...) {