Skip to content

More precise writebarrier for regions #98485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MichalStrehovsky wants to merge 1 commit into from
Closed

More precise writebarrier for regions #98485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,9 @@ EXTERN g_highest_address : QWORD
EXTERN g_ephemeral_low : QWORD
EXTERN g_ephemeral_high : QWORD
EXTERN g_card_table : QWORD
EXTERN g_region_shr : BYTE
EXTERN g_region_use_bitwise_write_barrier : BYTE
EXTERN g_region_to_generation_table : QWORD

ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
EXTERN g_card_bundle_table : QWORD
Expand Down
103 changes: 103 additions & 0 deletions src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,55 @@ endif
cmp REFREG, [g_ephemeral_high]
jae &BASENAME&_NoBarrierRequired_&REFREG&

mov r10, REFREG
mov r11, rcx

mov cl, [g_region_shr]
test cl, cl
je &BASENAME&_SkipCheck_&REFREG&

; check if the source is in gen 2 - then it's not an ephemeral pointer
shr r10, cl
add r10, [g_region_to_generation_table]
cmp byte ptr [r10], 82h
je &BASENAME&_NoBarrierRequired_&REFREG&

; check if the destination happens to be in gen 0
mov r10, r11
shr r10, cl
add r10, [g_region_to_generation_table]
cmp byte ptr [r10], 0
je &BASENAME&_NoBarrierRequired_&REFREG&
&BASENAME&_SkipCheck_&REFREG&:

cmp [g_region_use_bitwise_write_barrier], 0
mov rcx, r11
je &BASENAME&_CheckCardTableByte_&REFREG&

; compute card table bit
mov r10b, 1
shr rcx, 8
and cl, 7
shl r10b, cl

; Check if we need to update the card table
; Calc pCardByte
mov rcx, r11
shr rcx, 0Bh
add rcx, [g_card_table]

; Check if this card table bit is already set
test byte ptr [rcx], r10b
jne &BASENAME&_NoBarrierRequired_&REFREG&

lock or byte ptr [rcx], r10b

mov rcx, r11
shr rcx, 15h
jmp &BASENAME&_CheckCardBundle_&REFREG&

&BASENAME&_CheckCardTableByte_&REFREG&:

;; We have a location on the GC heap being updated with a reference to an ephemeral object so we must
;; track this write. The location address is translated into an offset in the card table bitmap. We set
;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
Expand All @@ -137,11 +186,14 @@ endif
ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0x0B already)
shr rcx, 0Ah
&BASENAME&_CheckCardBundle_&REFREG&:
add rcx, [g_card_bundle_table]
cmp byte ptr [rcx], 0FFh
je &BASENAME&_NoBarrierRequired_&REFREG&

mov byte ptr [rcx], 0FFh
else
&BASENAME&_CheckCardBundle_&REFREG&:
endif

&BASENAME&_NoBarrierRequired_&REFREG&:
Expand Down Expand Up @@ -317,6 +369,54 @@ RhpByRefAssignRef_CheckCardTable:
cmp rcx, [g_ephemeral_high]
jae RhpByRefAssignRef_NoBarrierRequired

mov r10, rcx

mov cl, [g_region_shr]
test cl, cl
je RhpByRefAssignRef_SkipCheck

; check if the source is in gen 2 - then it's not an ephemeral pointer
shr r10, cl
add r10, [g_region_to_generation_table]
cmp byte ptr [r10], 82h
je RhpByRefAssignRef_NoBarrierRequired

; check if the destination happens to be in gen 0
mov r10, rdi
shr r10, cl
add r10, [g_region_to_generation_table]
cmp byte ptr [r10], 0
je RhpByRefAssignRef_NoBarrierRequired
RhpByRefAssignRef_SkipCheck:

cmp [g_region_use_bitwise_write_barrier], 0
je RhpByRefAssignRef_CheckCardTableByte

; compute card table bit
mov rcx, rdi
mov r10b, 1
shr rcx, 8
and cl, 7
shl r10b, cl

; Check if we need to update the card table
; Calc pCardByte
mov rcx, rdi
shr rcx, 0Bh
add rcx, [g_card_table]

; Check if this card table bit is already set
test byte ptr [rcx], r10b
jne RhpByRefAssignRef_NoBarrierRequired

lock or byte ptr [rcx], r10b

mov rcx, rdi
shr rcx, 15h
jmp RhpByRefAssignRef_CheckCardBundle

RhpByRefAssignRef_CheckCardTableByte:

;; move current rdi value into rcx, we need to keep rdi and eventually increment by 8
mov rcx, rdi

Expand All @@ -335,11 +435,14 @@ RhpByRefAssignRef_CheckCardTable:
ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0Bh already)
shr rcx, 0Ah
RhpByRefAssignRef_CheckCardBundle:
add rcx, [g_card_bundle_table]
cmp byte ptr [rcx], 0FFh
je RhpByRefAssignRef_NoBarrierRequired

mov byte ptr [rcx], 0FFh
else
RhpByRefAssignRef_CheckCardBundle:
endif

RhpByRefAssignRef_NoBarrierRequired:
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/nativeaot/Runtime/gcenv.ee.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,9 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
assert(args->ephemeral_high != nullptr);
g_ephemeral_low = args->ephemeral_low;
g_ephemeral_high = args->ephemeral_high;
g_region_to_generation_table = args->region_to_generation_table;
g_region_shr = args->region_shr;
g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier;
return;
case WriteBarrierOp::Initialize:
// This operation should only be invoked once, upon initialization.
Expand Down Expand Up @@ -442,6 +445,9 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)

g_lowest_address = args->lowest_address;
g_highest_address = args->highest_address;
g_region_to_generation_table = args->region_to_generation_table;
g_region_shr = args->region_shr;
g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier;
g_ephemeral_low = args->ephemeral_low;
g_ephemeral_high = args->ephemeral_high;
return;
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/nativeaot/Runtime/gcheaputilities.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ GPTR_IMPL_INIT(uint8_t, g_highest_address, nullptr);
GVAL_IMPL_INIT(GCHeapType, g_heap_type, GC_HEAP_INVALID);
uint8_t* g_ephemeral_low = (uint8_t*)1;
uint8_t* g_ephemeral_high = (uint8_t*)~0;
uint8_t* g_region_to_generation_table = nullptr;
uint8_t g_region_shr = 0;
bool g_region_use_bitwise_write_barrier = false;

#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
uint32_t* g_card_bundle_table = nullptr;
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/nativeaot/Runtime/gcheaputilities.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ extern "C" uint32_t* g_card_bundle_table;

extern "C" uint8_t* g_ephemeral_low;
extern "C" uint8_t* g_ephemeral_high;
extern "C" uint8_t* g_region_to_generation_table;
extern "C" uint8_t g_region_shr;
extern "C" bool g_region_use_bitwise_write_barrier;

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
extern "C" bool g_sw_ww_enabled_for_gc_heap;
Expand Down