From 8aab1ccc0ccfe13435c939ec50717a4356319c54 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Tue, 2 Apr 2024 12:16:36 -0700 Subject: [PATCH 01/17] draft of uoh changes --- src/coreclr/gc/gc.cpp | 59 ++++++++++++++++++++++++----------------- src/coreclr/gc/gcpriv.h | 10 ++++--- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 40cb8694fd4d52..2e207540a1d469 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2657,13 +2657,12 @@ size_t gc_heap::end_loh_size = 0; size_t gc_heap::bgc_begin_poh_size = 0; size_t gc_heap::end_poh_size = 0; -#ifdef BGC_SERVO_TUNING uint64_t gc_heap::loh_a_no_bgc = 0; - uint64_t gc_heap::loh_a_bgc_marking = 0; - uint64_t gc_heap::loh_a_bgc_planning = 0; - +uint64_t gc_heap::poh_a_bgc_marking = 0; +uint64_t gc_heap::poh_a_bgc_planning = 0; +#ifdef BGC_SERVO_TUNING size_t gc_heap::bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING @@ -2794,9 +2793,9 @@ FinalizerWorkItem* gc_heap::finalizer_work; BOOL gc_heap::proceed_with_gc_p = FALSE; GCSpinLock gc_heap::gc_lock; -#ifdef BGC_SERVO_TUNING +#ifdef BACKGROUND_GC uint64_t gc_heap::total_loh_a_last_bgc = 0; -#endif //BGC_SERVO_TUNING +#endif //BACKGROUND_GC #ifdef USE_REGIONS region_free_list gc_heap::global_regions_to_decommit[count_free_region_kinds]; @@ -15039,10 +15038,12 @@ gc_heap::init_gc_heap (int h_number) make_mark_stack(arr); #ifdef BACKGROUND_GC -#ifdef BGC_SERVO_TUNING loh_a_no_bgc = 0; loh_a_bgc_marking = 0; loh_a_bgc_planning = 0; + poh_a_bgc_marking = 0; + poh_a_bgc_planning = 0; +#ifdef BGC_SERVO_TUNING bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING freeable_soh_segment = 0; @@ -18448,24 +18449,34 @@ allocation_state gc_heap::allocate_uoh (int gen_number, #ifdef BACKGROUND_GC if (gc_heap::background_running_p()) { -#ifdef BGC_SERVO_TUNING - bool planning_p = (current_c_gc_state == c_gc_state_planning); -#endif //BGC_SERVO_TUNING - background_uoh_alloc_count++; - //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0) + + //!!! add helper + if (current_c_gc_state == c_gc_state_planning) { -#ifdef BGC_SERVO_TUNING - if (planning_p) + if (gen_number == loh_generation) { loh_a_bgc_planning += size; } else + { + poh_a_bgc_planning += size; + } + } + else + { + if (gen_number == loh_generation) { loh_a_bgc_marking += size; } -#endif //BGC_SERVO_TUNING + else + { + poh_a_bgc_marking += size; + } + } + //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0) + { int spin_for_allocation = (gen_number == loh_generation) ? bgc_loh_allocate_spin() : bgc_poh_allocate_spin(); @@ -18491,12 +18502,10 @@ allocation_state gc_heap::allocate_uoh (int gen_number, } } } -#ifdef BGC_SERVO_TUNING else { loh_a_no_bgc += size; } -#endif //BGC_SERVO_TUNING #endif //BACKGROUND_GC gc_reason gr = reason_oos_loh; @@ -45890,9 +45899,7 @@ void gc_heap::background_sweep() if (heap_number == 0) { -#ifdef BGC_SERVO_TUNING get_and_reset_loh_alloc_info(); -#endif //BGC_SERVO_TUNING uint64_t suspended_end_ts = GetHighPrecisionTimeStamp(); last_bgc_info[last_bgc_info_index].pause_durations[1] = (size_t)(suspended_end_ts - suspended_start_time); total_suspended_time += last_bgc_info[last_bgc_info_index].pause_durations[1]; @@ -50238,12 +50245,10 @@ void gc_heap::check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, } } } +#endif //BGC_SERVO_TUNING void gc_heap::get_and_reset_loh_alloc_info() { - if (!bgc_tuning::enable_fl_tuning) - return; - total_loh_a_last_bgc = 0; uint64_t total_loh_a_no_bgc = 0; @@ -50257,11 +50262,16 @@ void gc_heap::get_and_reset_loh_alloc_info() { gc_heap* hp = pGenGCHeap; #endif //MULTIPLE_HEAPS + + gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); + current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_marking + hp->loh_a_bgc_planning; + current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_marking + hp->poh_a_bgc_planning; + total_loh_a_no_bgc += hp->loh_a_no_bgc; hp->loh_a_no_bgc = 0; - total_loh_a_bgc_marking += hp->loh_a_bgc_marking; + total_loh_a_bgc_marking += hp->loh_a_bgc_marking + hp->poh_a_bgc_marking; hp->loh_a_bgc_marking = 0; - total_loh_a_bgc_planning += hp->loh_a_bgc_planning; + total_loh_a_bgc_planning += hp->loh_a_bgc_planning + hp->loh_a_bgc_planning; hp->loh_a_bgc_planning = 0; } dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd", @@ -50271,7 +50281,6 @@ void gc_heap::get_and_reset_loh_alloc_info() total_loh_a_last_bgc = total_loh_a_no_bgc + total_loh_a_bgc_marking + total_loh_a_bgc_planning; } -#endif //BGC_SERVO_TUNING bool gc_heap::is_pm_ratio_exceeded() { diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 788cbff9f5e507..0e9c19e4b6a326 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -1997,8 +1997,9 @@ class gc_heap }; PER_HEAP_ISOLATED_METHOD void check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, ptrdiff_t virtual_fl_size); - PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info(); #endif //BGC_SERVO_TUNING + //!!! move? + PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info(); #ifndef USE_REGIONS PER_HEAP_METHOD BOOL expand_soh_with_minimal_gc(); @@ -3460,10 +3461,13 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC snoop_stats_data snoop_stat; #endif //SNOOP_STATS -#ifdef BGC_SERVO_TUNING + //!!! move these?? PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_no_bgc; PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_marking; PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_planning; + PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_marking; + PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_planning; +#ifdef BGC_SERVO_TUNING PER_HEAP_FIELD_SINGLE_GC size_t bgc_maxgen_end_fl_size; #endif //BGC_SERVO_TUNING #endif //BACKGROUND_GC @@ -4097,11 +4101,9 @@ class gc_heap PER_HEAP_ISOLATED_FIELD_SINGLE_GC GCEvent bgc_start_event; -#ifdef BGC_SERVO_TUNING // Total allocated last BGC's plan + between last and this bgc + // this bgc's mark PER_HEAP_ISOLATED_FIELD_SINGLE_GC uint64_t total_loh_a_last_bgc; -#endif //BGC_SERVO_TUNING #endif //BACKGROUND_GC #ifdef USE_REGIONS From 7a0ab00c6b020794f2b1ddab4b3a1ee1d2bb91bc Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 5 Apr 2024 11:27:28 -0700 Subject: [PATCH 02/17] comments --- src/coreclr/gc/gc.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 2e207540a1d469..57372a50ecd86c 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -45818,7 +45818,6 @@ void gc_heap::background_sweep() concurrent_print_time_delta ("Sw"); dprintf (2, ("---- (GC%zu)Background Sweep Phase ----", VolatileLoad(&settings.gc_index))); - //block concurrent allocation for large objects dprintf (3, ("lh state: planning")); for (int i = 0; i <= max_generation; i++) @@ -45869,6 +45868,7 @@ void gc_heap::background_sweep() sweep_ro_segments(); #endif //FEATURE_BASICFREEZE + // Multiple threads will reach here. This conditional avoids multiple volatile writes. if (current_c_gc_state != c_gc_state_planning) { current_c_gc_state = c_gc_state_planning; @@ -46228,6 +46228,7 @@ void gc_heap::background_sweep() concurrent_print_time_delta ("Swe SOH"); FIRE_EVENT(BGC1stSweepEnd, 0); + //block concurrent allocation for large objects enter_spin_lock (&more_space_lock_uoh); add_saved_spinlock_info (true, me_acquire, mt_bgc_uoh_sweep, msl_entered); From 97c78921546fc5f0cc3fdd3c9d6c6fe59639cccc Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 5 Apr 2024 14:18:18 -0700 Subject: [PATCH 03/17] comments --- src/coreclr/gc/gc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 57372a50ecd86c..1e5006e00501bc 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -45818,8 +45818,6 @@ void gc_heap::background_sweep() concurrent_print_time_delta ("Sw"); dprintf (2, ("---- (GC%zu)Background Sweep Phase ----", VolatileLoad(&settings.gc_index))); - dprintf (3, ("lh state: planning")); - for (int i = 0; i <= max_generation; i++) { generation* gen_to_reset = generation_of (i); @@ -45868,7 +45866,9 @@ void gc_heap::background_sweep() sweep_ro_segments(); #endif //FEATURE_BASICFREEZE - // Multiple threads will reach here. This conditional avoids multiple volatile writes. + dprintf (3, ("lh state: planning")); + + // Multiple threads may reach here. This conditional partially avoids multiple volatile writes. if (current_c_gc_state != c_gc_state_planning) { current_c_gc_state = c_gc_state_planning; From 808b278024c553357037be634f739d6e8adb472b Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 5 Apr 2024 14:18:30 -0700 Subject: [PATCH 04/17] update size_before in the correct places --- src/coreclr/gc/gc.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 1e5006e00501bc..a54eb08b6372e0 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -46284,6 +46284,14 @@ void gc_heap::background_sweep() // be accurate. compute_new_dynamic_data (max_generation); + // We also need to adjust size_before for UOH allocations that occurred during sweeping. + gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); + assert(hp->loh_a_bgc_marking == 0); + assert(hp->poh_a_bgc_marking == 0); + assert(hp->loh_a_no_bgc == 0); + current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_planning; + current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_planning; + #ifdef DOUBLY_LINKED_FL current_bgc_state = bgc_not_in_process; @@ -50264,9 +50272,12 @@ void gc_heap::get_and_reset_loh_alloc_info() gc_heap* hp = pGenGCHeap; #endif //MULTIPLE_HEAPS + // We need to adjust size_before for UOH allocations that occurred during marking + // before we lose the values here. gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); - current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_marking + hp->loh_a_bgc_planning; - current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_marking + hp->poh_a_bgc_planning; + // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before. + current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_marking; + current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_marking; total_loh_a_no_bgc += hp->loh_a_no_bgc; hp->loh_a_no_bgc = 0; From dde17e21e40aeef2dc603401bf51f588d5f57a74 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 5 Apr 2024 14:21:08 -0700 Subject: [PATCH 05/17] this --- src/coreclr/gc/gc.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index a54eb08b6372e0..6a27c7aa83789e 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -46286,11 +46286,11 @@ void gc_heap::background_sweep() // We also need to adjust size_before for UOH allocations that occurred during sweeping. gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); - assert(hp->loh_a_bgc_marking == 0); - assert(hp->poh_a_bgc_marking == 0); - assert(hp->loh_a_no_bgc == 0); - current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_planning; - current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_planning; + assert(loh_a_bgc_marking == 0); + assert(poh_a_bgc_marking == 0); + assert(loh_a_no_bgc == 0); + current_gc_data_per_heap->gen_data[loh_generation].size_before += loh_a_bgc_planning; + current_gc_data_per_heap->gen_data[poh_generation].size_before += poh_a_bgc_planning; #ifdef DOUBLY_LINKED_FL current_bgc_state = bgc_not_in_process; From ec0d46852d1557cdf944c0c890da9bd1518fa0eb Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 5 Apr 2024 15:55:17 -0700 Subject: [PATCH 06/17] add helper, move ifdefs --- src/coreclr/gc/gc.cpp | 59 ++++++++++++++++++++++++----------------- src/coreclr/gc/gcpriv.h | 19 +++++++------ 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 6a27c7aa83789e..e9e321de11f232 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -18425,33 +18425,14 @@ bool gc_heap::should_retry_other_heap (int gen_number, size_t size) } } -allocation_state gc_heap::allocate_uoh (int gen_number, - size_t size, - alloc_context* acontext, - uint32_t flags, - int align_const) +void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size) { - enter_msl_status msl_status = msl_entered; - - // No variable values should be "carried over" from one state to the other. - // That's why there are local variable for each state - allocation_state uoh_alloc_state = a_state_start; - -#ifdef SPINLOCK_HISTORY - current_uoh_alloc_state = uoh_alloc_state; -#endif //SPINLOCK_HISTORY + assert(gen_number == loh_generation || gen_number == poh_generation); -#ifdef RECORD_LOH_STATE - EEThreadId current_thread_id; - current_thread_id.SetToCurrentThread (); -#endif //RECORD_LOH_STATE - -#ifdef BACKGROUND_GC if (gc_heap::background_running_p()) { background_uoh_alloc_count++; - //!!! add helper if (current_c_gc_state == c_gc_state_planning) { if (gen_number == loh_generation) @@ -18474,7 +18455,39 @@ allocation_state gc_heap::allocate_uoh (int gen_number, poh_a_bgc_marking += size; } } + } + else + { + loh_a_no_bgc += size; + } +} +allocation_state gc_heap::allocate_uoh (int gen_number, + size_t size, + alloc_context* acontext, + uint32_t flags, + int align_const) +{ + enter_msl_status msl_status = msl_entered; + + // No variable values should be "carried over" from one state to the other. + // That's why there are local variable for each state + allocation_state uoh_alloc_state = a_state_start; + +#ifdef SPINLOCK_HISTORY + current_uoh_alloc_state = uoh_alloc_state; +#endif //SPINLOCK_HISTORY + +#ifdef RECORD_LOH_STATE + EEThreadId current_thread_id; + current_thread_id.SetToCurrentThread (); +#endif //RECORD_LOH_STATE + +#ifdef BACKGROUND_GC + bgc_record_uoh_allocation(gen_number, size); + + if (gc_heap::background_running_p()) + { //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0) { int spin_for_allocation = (gen_number == loh_generation) ? @@ -18502,10 +18515,6 @@ allocation_state gc_heap::allocate_uoh (int gen_number, } } } - else - { - loh_a_no_bgc += size; - } #endif //BACKGROUND_GC gc_reason gr = reason_oos_loh; diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 0e9c19e4b6a326..14f9afead8a683 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -1752,6 +1752,8 @@ class gc_heap PER_HEAP_ISOLATED_METHOD void add_to_history(); + PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info(); + #ifdef BGC_SERVO_TUNING // Currently BGC servo tuning is an experimental feature. class bgc_tuning @@ -1998,8 +2000,6 @@ class gc_heap PER_HEAP_ISOLATED_METHOD void check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, ptrdiff_t virtual_fl_size); #endif //BGC_SERVO_TUNING - //!!! move? - PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info(); #ifndef USE_REGIONS PER_HEAP_METHOD BOOL expand_soh_with_minimal_gc(); @@ -2231,6 +2231,8 @@ class gc_heap PER_HEAP_METHOD BOOL bgc_loh_allocate_spin(); PER_HEAP_METHOD BOOL bgc_poh_allocate_spin(); + + PER_HEAP_METHOD void bgc_record_uoh_allocation(int gen_number, size_t size); #endif //BACKGROUND_GC PER_HEAP_METHOD void add_saved_spinlock_info ( @@ -3437,6 +3439,13 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC uint8_t* next_sweep_obj; PER_HEAP_FIELD_SINGLE_GC uint8_t* current_sweep_pos; + + PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_no_bgc; + PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_marking; + PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_planning; + PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_marking; + PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_planning; + #ifdef DOUBLY_LINKED_FL PER_HEAP_FIELD_SINGLE_GC heap_segment* current_sweep_seg; #endif //DOUBLY_LINKED_FL @@ -3461,12 +3470,6 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC snoop_stats_data snoop_stat; #endif //SNOOP_STATS - //!!! move these?? - PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_no_bgc; - PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_marking; - PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_planning; - PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_marking; - PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_planning; #ifdef BGC_SERVO_TUNING PER_HEAP_FIELD_SINGLE_GC size_t bgc_maxgen_end_fl_size; #endif //BGC_SERVO_TUNING From 4541a8bdb18586812c014607abe32b9bd6ab4f4f Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 5 Apr 2024 16:06:42 -0700 Subject: [PATCH 07/17] fix poh --- src/coreclr/gc/gc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index e9e321de11f232..34c82fefe73feb 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -50292,8 +50292,10 @@ void gc_heap::get_and_reset_loh_alloc_info() hp->loh_a_no_bgc = 0; total_loh_a_bgc_marking += hp->loh_a_bgc_marking + hp->poh_a_bgc_marking; hp->loh_a_bgc_marking = 0; - total_loh_a_bgc_planning += hp->loh_a_bgc_planning + hp->loh_a_bgc_planning; + hp->poh_a_bgc_marking = 0; + total_loh_a_bgc_planning += hp->loh_a_bgc_planning + hp->poh_a_bgc_planning; hp->loh_a_bgc_planning = 0; + hp->poh_a_bgc_planning = 0; } dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd", total_loh_a_no_bgc, From 8451ebf5b950f40df330c2d36088429591f0e130 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Thu, 11 Apr 2024 10:19:17 -0700 Subject: [PATCH 08/17] more size_t --- src/coreclr/gc/gc.cpp | 10 +++++----- src/coreclr/gc/gcpriv.h | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 34c82fefe73feb..93be75c38f1b83 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2657,11 +2657,11 @@ size_t gc_heap::end_loh_size = 0; size_t gc_heap::bgc_begin_poh_size = 0; size_t gc_heap::end_poh_size = 0; -uint64_t gc_heap::loh_a_no_bgc = 0; -uint64_t gc_heap::loh_a_bgc_marking = 0; -uint64_t gc_heap::loh_a_bgc_planning = 0; -uint64_t gc_heap::poh_a_bgc_marking = 0; -uint64_t gc_heap::poh_a_bgc_planning = 0; +size_t gc_heap::loh_a_no_bgc = 0; +size_t gc_heap::loh_a_bgc_marking = 0; +size_t gc_heap::loh_a_bgc_planning = 0; +size_t gc_heap::poh_a_bgc_marking = 0; +size_t gc_heap::poh_a_bgc_planning = 0; #ifdef BGC_SERVO_TUNING size_t gc_heap::bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 14f9afead8a683..0a394be7f19ead 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -3440,11 +3440,11 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC uint8_t* next_sweep_obj; PER_HEAP_FIELD_SINGLE_GC uint8_t* current_sweep_pos; - PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_no_bgc; - PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_marking; - PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_planning; - PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_marking; - PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_planning; + PER_HEAP_FIELD_SINGLE_GC size_t loh_a_no_bgc; + PER_HEAP_FIELD_SINGLE_GC size_t loh_a_bgc_marking; + PER_HEAP_FIELD_SINGLE_GC size_t loh_a_bgc_planning; + PER_HEAP_FIELD_SINGLE_GC size_t poh_a_bgc_marking; + PER_HEAP_FIELD_SINGLE_GC size_t poh_a_bgc_planning; #ifdef DOUBLY_LINKED_FL PER_HEAP_FIELD_SINGLE_GC heap_segment* current_sweep_seg; From 621befa6e89c7e7b53dc7637cec491a1e120b87d Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Thu, 11 Apr 2024 15:11:10 -0700 Subject: [PATCH 09/17] UOH names --- src/coreclr/gc/gc.cpp | 10 +++++----- src/coreclr/gc/gcpriv.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 93be75c38f1b83..b42971d668f2a5 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2794,7 +2794,7 @@ BOOL gc_heap::proceed_with_gc_p = FALSE; GCSpinLock gc_heap::gc_lock; #ifdef BACKGROUND_GC -uint64_t gc_heap::total_loh_a_last_bgc = 0; +uint64_t gc_heap::total_uoh_a_last_bgc = 0; #endif //BACKGROUND_GC #ifdef USE_REGIONS @@ -40601,7 +40601,7 @@ void gc_heap::bgc_tuning::record_and_adjust_bgc_end() calculate_tuning (max_generation, true); - if (total_loh_a_last_bgc > 0) + if (total_uoh_a_last_bgc > 0) { calculate_tuning (loh_generation, true); } @@ -46237,7 +46237,7 @@ void gc_heap::background_sweep() concurrent_print_time_delta ("Swe SOH"); FIRE_EVENT(BGC1stSweepEnd, 0); - //block concurrent allocation for large objects + //block concurrent allocation for UOH objects enter_spin_lock (&more_space_lock_uoh); add_saved_spinlock_info (true, me_acquire, mt_bgc_uoh_sweep, msl_entered); @@ -50267,7 +50267,7 @@ void gc_heap::check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, void gc_heap::get_and_reset_loh_alloc_info() { - total_loh_a_last_bgc = 0; + total_uoh_a_last_bgc = 0; uint64_t total_loh_a_no_bgc = 0; uint64_t total_loh_a_bgc_marking = 0; @@ -50302,7 +50302,7 @@ void gc_heap::get_and_reset_loh_alloc_info() total_loh_a_bgc_marking, total_loh_a_bgc_planning)); - total_loh_a_last_bgc = total_loh_a_no_bgc + total_loh_a_bgc_marking + total_loh_a_bgc_planning; + total_uoh_a_last_bgc = total_loh_a_no_bgc + total_loh_a_bgc_marking + total_loh_a_bgc_planning; } bool gc_heap::is_pm_ratio_exceeded() diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 0a394be7f19ead..71dcbf7a4ca3f7 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -4106,7 +4106,7 @@ class gc_heap // Total allocated last BGC's plan + between last and this bgc + // this bgc's mark - PER_HEAP_ISOLATED_FIELD_SINGLE_GC uint64_t total_loh_a_last_bgc; + PER_HEAP_ISOLATED_FIELD_SINGLE_GC uint64_t total_uoh_a_last_bgc; #endif //BACKGROUND_GC #ifdef USE_REGIONS From 532256c9fd6d9e7a9be809b5fbf5b034d2753fbd Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Thu, 11 Apr 2024 17:02:50 -0700 Subject: [PATCH 10/17] UOH names --- src/coreclr/gc/gc.cpp | 69 ++++++++++++++++------------------------- src/coreclr/gc/gcpriv.h | 8 ++--- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index b42971d668f2a5..f1eebd1adda97f 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2657,11 +2657,9 @@ size_t gc_heap::end_loh_size = 0; size_t gc_heap::bgc_begin_poh_size = 0; size_t gc_heap::end_poh_size = 0; -size_t gc_heap::loh_a_no_bgc = 0; -size_t gc_heap::loh_a_bgc_marking = 0; -size_t gc_heap::loh_a_bgc_planning = 0; -size_t gc_heap::poh_a_bgc_marking = 0; -size_t gc_heap::poh_a_bgc_planning = 0; +size_t gc_heap::uoh_a_no_bgc = 0; +size_t gc_heap::uoh_a_bgc_marking[2] = {}; +size_t gc_heap::uoh_a_bgc_planning[2] = {}; #ifdef BGC_SERVO_TUNING size_t gc_heap::bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING @@ -15038,11 +15036,9 @@ gc_heap::init_gc_heap (int h_number) make_mark_stack(arr); #ifdef BACKGROUND_GC - loh_a_no_bgc = 0; - loh_a_bgc_marking = 0; - loh_a_bgc_planning = 0; - poh_a_bgc_marking = 0; - poh_a_bgc_planning = 0; + uoh_a_no_bgc = 0; + uoh_a_bgc_marking[0] = uoh_a_bgc_marking[1] = 0; + uoh_a_bgc_planning[0] = uoh_a_bgc_planning[1] = 0; #ifdef BGC_SERVO_TUNING bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING @@ -18433,32 +18429,19 @@ void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size) { background_uoh_alloc_count++; + static_assert(poh_generation == (loh_generation + 1)); if (current_c_gc_state == c_gc_state_planning) { - if (gen_number == loh_generation) - { - loh_a_bgc_planning += size; - } - else - { - poh_a_bgc_planning += size; - } + uoh_a_bgc_planning[gen_number - loh_generation] += size; } else { - if (gen_number == loh_generation) - { - loh_a_bgc_marking += size; - } - else - { - poh_a_bgc_marking += size; - } + uoh_a_bgc_marking[gen_number - loh_generation] += size; } } else { - loh_a_no_bgc += size; + uoh_a_no_bgc += size; } } @@ -46295,11 +46278,11 @@ void gc_heap::background_sweep() // We also need to adjust size_before for UOH allocations that occurred during sweeping. gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); - assert(loh_a_bgc_marking == 0); - assert(poh_a_bgc_marking == 0); - assert(loh_a_no_bgc == 0); - current_gc_data_per_heap->gen_data[loh_generation].size_before += loh_a_bgc_planning; - current_gc_data_per_heap->gen_data[poh_generation].size_before += poh_a_bgc_planning; + assert(uoh_a_bgc_marking[0] == 0); + assert(uoh_a_bgc_marking[1] == 0); + assert(uoh_a_no_bgc == 0); + current_gc_data_per_heap->gen_data[loh_generation].size_before += uoh_a_bgc_planning[0]; + current_gc_data_per_heap->gen_data[poh_generation].size_before += uoh_a_bgc_planning[1]; #ifdef DOUBLY_LINKED_FL current_bgc_state = bgc_not_in_process; @@ -50285,17 +50268,17 @@ void gc_heap::get_and_reset_loh_alloc_info() // before we lose the values here. gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before. - current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_marking; - current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_marking; - - total_loh_a_no_bgc += hp->loh_a_no_bgc; - hp->loh_a_no_bgc = 0; - total_loh_a_bgc_marking += hp->loh_a_bgc_marking + hp->poh_a_bgc_marking; - hp->loh_a_bgc_marking = 0; - hp->poh_a_bgc_marking = 0; - total_loh_a_bgc_planning += hp->loh_a_bgc_planning + hp->poh_a_bgc_planning; - hp->loh_a_bgc_planning = 0; - hp->poh_a_bgc_planning = 0; + current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->uoh_a_bgc_marking[0]; + current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->uoh_a_bgc_marking[1]; + + total_loh_a_no_bgc += hp->uoh_a_no_bgc; + hp->uoh_a_no_bgc = 0; + total_loh_a_bgc_marking += hp->uoh_a_bgc_marking[0] + hp->uoh_a_bgc_marking[1]; + hp->uoh_a_bgc_marking[0] = 0; + hp->uoh_a_bgc_marking[1] = 0; + total_loh_a_bgc_planning += hp->uoh_a_bgc_planning[0] + hp->uoh_a_bgc_planning[1]; + hp->uoh_a_bgc_planning[0] = 0; + hp->uoh_a_bgc_planning[1] = 0; } dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd", total_loh_a_no_bgc, diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 71dcbf7a4ca3f7..69ac14866fd642 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -3440,11 +3440,9 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC uint8_t* next_sweep_obj; PER_HEAP_FIELD_SINGLE_GC uint8_t* current_sweep_pos; - PER_HEAP_FIELD_SINGLE_GC size_t loh_a_no_bgc; - PER_HEAP_FIELD_SINGLE_GC size_t loh_a_bgc_marking; - PER_HEAP_FIELD_SINGLE_GC size_t loh_a_bgc_planning; - PER_HEAP_FIELD_SINGLE_GC size_t poh_a_bgc_marking; - PER_HEAP_FIELD_SINGLE_GC size_t poh_a_bgc_planning; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_no_bgc; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_marking[2]; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_planning[2]; #ifdef DOUBLY_LINKED_FL PER_HEAP_FIELD_SINGLE_GC heap_segment* current_sweep_seg; From df470c22f43f5ae5c24a871f176bd27d7b20f92e Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 12 Apr 2024 10:20:08 -0700 Subject: [PATCH 11/17] UOH names, symbolic constants --- src/coreclr/gc/gc.cpp | 51 +++++++++++++++++++++-------------------- src/coreclr/gc/gcpriv.h | 2 +- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index f1eebd1adda97f..cc4ae0361b9ecb 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -388,7 +388,7 @@ int relative_index_power2_free_space (size_t power2) #ifdef BACKGROUND_GC uint32_t bgc_alloc_spin_count = 140; -uint32_t bgc_alloc_spin_count_loh = 16; +uint32_t bgc_alloc_spin_count_uoh = 16; uint32_t bgc_alloc_spin = 2; inline @@ -15037,6 +15037,7 @@ gc_heap::init_gc_heap (int h_number) #ifdef BACKGROUND_GC uoh_a_no_bgc = 0; + static_assert((total_generation_count - uoh_start_generation) == 2); uoh_a_bgc_marking[0] = uoh_a_bgc_marking[1] = 0; uoh_a_bgc_planning[0] = uoh_a_bgc_planning[1] = 0; #ifdef BGC_SERVO_TUNING @@ -18429,14 +18430,14 @@ void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size) { background_uoh_alloc_count++; - static_assert(poh_generation == (loh_generation + 1)); + static_assert((total_generation_count - uoh_start_generation) == 2); if (current_c_gc_state == c_gc_state_planning) { - uoh_a_bgc_planning[gen_number - loh_generation] += size; + uoh_a_bgc_planning[gen_number - uoh_start_generation] += size; } else { - uoh_a_bgc_marking[gen_number - loh_generation] += size; + uoh_a_bgc_marking[gen_number - uoh_start_generation] += size; } } else @@ -18471,7 +18472,7 @@ allocation_state gc_heap::allocate_uoh (int gen_number, if (gc_heap::background_running_p()) { - //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0) + //if ((background_uoh_alloc_count % bgc_alloc_spin_count_uoh) == 0) { int spin_for_allocation = (gen_number == loh_generation) ? bgc_loh_allocate_spin() : @@ -45891,7 +45892,7 @@ void gc_heap::background_sweep() if (heap_number == 0) { - get_and_reset_loh_alloc_info(); + get_and_reset_uoh_alloc_info(); uint64_t suspended_end_ts = GetHighPrecisionTimeStamp(); last_bgc_info[last_bgc_info_index].pause_durations[1] = (size_t)(suspended_end_ts - suspended_start_time); total_suspended_time += last_bgc_info[last_bgc_info_index].pause_durations[1]; @@ -46278,11 +46279,12 @@ void gc_heap::background_sweep() // We also need to adjust size_before for UOH allocations that occurred during sweeping. gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); + static_assert((total_generation_count - uoh_start_generation) == 2); assert(uoh_a_bgc_marking[0] == 0); assert(uoh_a_bgc_marking[1] == 0); assert(uoh_a_no_bgc == 0); - current_gc_data_per_heap->gen_data[loh_generation].size_before += uoh_a_bgc_planning[0]; - current_gc_data_per_heap->gen_data[poh_generation].size_before += uoh_a_bgc_planning[1]; + current_gc_data_per_heap->gen_data[loh_generation].size_before += uoh_a_bgc_planning[loh_generation - uoh_start_generation]; + current_gc_data_per_heap->gen_data[poh_generation].size_before += uoh_a_bgc_planning[poh_generation - uoh_start_generation]; #ifdef DOUBLY_LINKED_FL current_bgc_state = bgc_not_in_process; @@ -50248,13 +50250,13 @@ void gc_heap::check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, } #endif //BGC_SERVO_TUNING -void gc_heap::get_and_reset_loh_alloc_info() +void gc_heap::get_and_reset_uoh_alloc_info() { total_uoh_a_last_bgc = 0; - uint64_t total_loh_a_no_bgc = 0; - uint64_t total_loh_a_bgc_marking = 0; - uint64_t total_loh_a_bgc_planning = 0; + uint64_t total_uoh_a_no_bgc = 0; + uint64_t total_uoh_a_bgc_marking = 0; + uint64_t total_uoh_a_bgc_planning = 0; #ifdef MULTIPLE_HEAPS for (int i = 0; i < gc_heap::n_heaps; i++) { @@ -50268,24 +50270,23 @@ void gc_heap::get_and_reset_loh_alloc_info() // before we lose the values here. gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before. - current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->uoh_a_bgc_marking[0]; - current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->uoh_a_bgc_marking[1]; + static_assert((total_generation_count - uoh_start_generation) == 2); + current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->uoh_a_bgc_marking[loh_generation - uoh_start_generation]; + current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->uoh_a_bgc_marking[poh_generation - uoh_start_generation]; - total_loh_a_no_bgc += hp->uoh_a_no_bgc; + total_uoh_a_no_bgc += hp->uoh_a_no_bgc; hp->uoh_a_no_bgc = 0; - total_loh_a_bgc_marking += hp->uoh_a_bgc_marking[0] + hp->uoh_a_bgc_marking[1]; - hp->uoh_a_bgc_marking[0] = 0; - hp->uoh_a_bgc_marking[1] = 0; - total_loh_a_bgc_planning += hp->uoh_a_bgc_planning[0] + hp->uoh_a_bgc_planning[1]; - hp->uoh_a_bgc_planning[0] = 0; - hp->uoh_a_bgc_planning[1] = 0; + total_uoh_a_bgc_marking += hp->uoh_a_bgc_marking[0] + hp->uoh_a_bgc_marking[1]; + hp->uoh_a_bgc_marking[0] = hp->uoh_a_bgc_marking[1] = 0; + total_uoh_a_bgc_planning += hp->uoh_a_bgc_planning[0] + hp->uoh_a_bgc_planning[1]; + hp->uoh_a_bgc_planning[0] = hp->uoh_a_bgc_planning[1] = 0; } dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd", - total_loh_a_no_bgc, - total_loh_a_bgc_marking, - total_loh_a_bgc_planning)); + total_uoh_a_no_bgc, + total_uoh_a_bgc_marking, + total_uoh_a_bgc_planning)); - total_uoh_a_last_bgc = total_loh_a_no_bgc + total_loh_a_bgc_marking + total_loh_a_bgc_planning; + total_uoh_a_last_bgc = total_uoh_a_no_bgc + total_uoh_a_bgc_marking + total_uoh_a_bgc_planning; } bool gc_heap::is_pm_ratio_exceeded() diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 69ac14866fd642..a973c1ab527be0 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -1752,7 +1752,7 @@ class gc_heap PER_HEAP_ISOLATED_METHOD void add_to_history(); - PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info(); + PER_HEAP_ISOLATED_METHOD void get_and_reset_uoh_alloc_info(); #ifdef BGC_SERVO_TUNING // Currently BGC servo tuning is an experimental feature. From f20e9b1d1d3dc771ef2477ac8b697c5048732473 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Fri, 12 Apr 2024 10:29:00 -0700 Subject: [PATCH 12/17] another use of 2 --- src/coreclr/gc/gc.cpp | 4 ++-- src/coreclr/gc/gcpriv.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index cc4ae0361b9ecb..e07748d3c23f47 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2658,8 +2658,8 @@ size_t gc_heap::bgc_begin_poh_size = 0; size_t gc_heap::end_poh_size = 0; size_t gc_heap::uoh_a_no_bgc = 0; -size_t gc_heap::uoh_a_bgc_marking[2] = {}; -size_t gc_heap::uoh_a_bgc_planning[2] = {}; +size_t gc_heap::uoh_a_bgc_marking[total_generation_count - uoh_start_generation] = {}; +size_t gc_heap::uoh_a_bgc_planning[total_generation_count - uoh_start_generation] = {}; #ifdef BGC_SERVO_TUNING size_t gc_heap::bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index a973c1ab527be0..31724d7aef36e5 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -3441,8 +3441,8 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC uint8_t* current_sweep_pos; PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_no_bgc; - PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_marking[2]; - PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_planning[2]; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_marking[total_generation_count - uoh_start_generation]; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_planning[total_generation_count - uoh_start_generation]; #ifdef DOUBLY_LINKED_FL PER_HEAP_FIELD_SINGLE_GC heap_segment* current_sweep_seg; From cde20fbbcc7848c4860046d575843f744573ae0b Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Sat, 13 Apr 2024 10:42:20 -0700 Subject: [PATCH 13/17] static_assert needs message --- src/coreclr/gc/gc.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index e07748d3c23f47..1adae2222bd727 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -15037,7 +15037,7 @@ gc_heap::init_gc_heap (int h_number) #ifdef BACKGROUND_GC uoh_a_no_bgc = 0; - static_assert((total_generation_count - uoh_start_generation) == 2); + static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); uoh_a_bgc_marking[0] = uoh_a_bgc_marking[1] = 0; uoh_a_bgc_planning[0] = uoh_a_bgc_planning[1] = 0; #ifdef BGC_SERVO_TUNING @@ -18430,7 +18430,7 @@ void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size) { background_uoh_alloc_count++; - static_assert((total_generation_count - uoh_start_generation) == 2); + static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); if (current_c_gc_state == c_gc_state_planning) { uoh_a_bgc_planning[gen_number - uoh_start_generation] += size; @@ -46279,7 +46279,7 @@ void gc_heap::background_sweep() // We also need to adjust size_before for UOH allocations that occurred during sweeping. gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); - static_assert((total_generation_count - uoh_start_generation) == 2); + static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); assert(uoh_a_bgc_marking[0] == 0); assert(uoh_a_bgc_marking[1] == 0); assert(uoh_a_no_bgc == 0); @@ -50270,7 +50270,7 @@ void gc_heap::get_and_reset_uoh_alloc_info() // before we lose the values here. gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before. - static_assert((total_generation_count - uoh_start_generation) == 2); + static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->uoh_a_bgc_marking[loh_generation - uoh_start_generation]; current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->uoh_a_bgc_marking[poh_generation - uoh_start_generation]; From 8fc75edfff484b61964081dded8b4b2074b9f860 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Wed, 17 Apr 2024 10:00:53 -0700 Subject: [PATCH 14/17] loops --- src/coreclr/gc/gc.cpp | 56 ++++++++++++++++++++++------------------- src/coreclr/gc/gc.h | 5 +++- src/coreclr/gc/gcpriv.h | 6 ++--- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 1adae2222bd727..5e089337758706 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2657,9 +2657,9 @@ size_t gc_heap::end_loh_size = 0; size_t gc_heap::bgc_begin_poh_size = 0; size_t gc_heap::end_poh_size = 0; -size_t gc_heap::uoh_a_no_bgc = 0; -size_t gc_heap::uoh_a_bgc_marking[total_generation_count - uoh_start_generation] = {}; -size_t gc_heap::uoh_a_bgc_planning[total_generation_count - uoh_start_generation] = {}; +size_t gc_heap::uoh_a_no_bgc[uoh_generation_count] = {}; +size_t gc_heap::uoh_a_bgc_marking[uoh_generation_count] = {}; +size_t gc_heap::uoh_a_bgc_planning[uoh_generation_count] = {}; #ifdef BGC_SERVO_TUNING size_t gc_heap::bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING @@ -15036,10 +15036,12 @@ gc_heap::init_gc_heap (int h_number) make_mark_stack(arr); #ifdef BACKGROUND_GC - uoh_a_no_bgc = 0; - static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); - uoh_a_bgc_marking[0] = uoh_a_bgc_marking[1] = 0; - uoh_a_bgc_planning[0] = uoh_a_bgc_planning[1] = 0; + for (int gen_num = uoh_start_generation; gen_num < total_generation_count; gen_num++) + { + uoh_a_no_bgc[gen_num - uoh_start_generation] = 0; + uoh_a_bgc_marking[gen_num - uoh_start_generation] = 0; + uoh_a_bgc_planning[gen_num - uoh_start_generation] = 0; + } #ifdef BGC_SERVO_TUNING bgc_maxgen_end_fl_size = 0; #endif //BGC_SERVO_TUNING @@ -18424,13 +18426,12 @@ bool gc_heap::should_retry_other_heap (int gen_number, size_t size) void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size) { - assert(gen_number == loh_generation || gen_number == poh_generation); + assert((gen_number >= uoh_start_generation) && (gen_number < total_generation_count)); if (gc_heap::background_running_p()) { background_uoh_alloc_count++; - static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); if (current_c_gc_state == c_gc_state_planning) { uoh_a_bgc_planning[gen_number - uoh_start_generation] += size; @@ -18442,7 +18443,7 @@ void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size) } else { - uoh_a_no_bgc += size; + uoh_a_no_bgc[gen_number - uoh_start_generation] += size; } } @@ -46279,12 +46280,12 @@ void gc_heap::background_sweep() // We also need to adjust size_before for UOH allocations that occurred during sweeping. gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); - static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); - assert(uoh_a_bgc_marking[0] == 0); - assert(uoh_a_bgc_marking[1] == 0); - assert(uoh_a_no_bgc == 0); - current_gc_data_per_heap->gen_data[loh_generation].size_before += uoh_a_bgc_planning[loh_generation - uoh_start_generation]; - current_gc_data_per_heap->gen_data[poh_generation].size_before += uoh_a_bgc_planning[poh_generation - uoh_start_generation]; + for (int gen_num = uoh_start_generation; gen_num < total_generation_count; gen_num++) + { + assert(uoh_a_bgc_marking[gen_num - uoh_start_generation] == 0); + assert(uoh_a_no_bgc[gen_num - uoh_start_generation] == 0); + current_gc_data_per_heap->gen_data[gen_num].size_before += uoh_a_bgc_planning[gen_num - uoh_start_generation]; + } #ifdef DOUBLY_LINKED_FL current_bgc_state = bgc_not_in_process; @@ -50270,16 +50271,19 @@ void gc_heap::get_and_reset_uoh_alloc_info() // before we lose the values here. gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before. - static_assert((total_generation_count - uoh_start_generation) == 2, "code depends on 2 UOH generations"); - current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->uoh_a_bgc_marking[loh_generation - uoh_start_generation]; - current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->uoh_a_bgc_marking[poh_generation - uoh_start_generation]; - - total_uoh_a_no_bgc += hp->uoh_a_no_bgc; - hp->uoh_a_no_bgc = 0; - total_uoh_a_bgc_marking += hp->uoh_a_bgc_marking[0] + hp->uoh_a_bgc_marking[1]; - hp->uoh_a_bgc_marking[0] = hp->uoh_a_bgc_marking[1] = 0; - total_uoh_a_bgc_planning += hp->uoh_a_bgc_planning[0] + hp->uoh_a_bgc_planning[1]; - hp->uoh_a_bgc_planning[0] = hp->uoh_a_bgc_planning[1] = 0; + for (int gen_num = uoh_start_generation; gen_num < total_generation_count; gen_num++) + { + current_gc_data_per_heap->gen_data[gen_num].size_before += hp->uoh_a_bgc_marking[gen_num - uoh_start_generation]; + + total_uoh_a_no_bgc += hp->uoh_a_no_bgc[gen_num - uoh_start_generation]; + hp->uoh_a_no_bgc[gen_num - uoh_start_generation] = 0; + + total_uoh_a_bgc_marking += hp->uoh_a_bgc_marking[gen_num - uoh_start_generation]; + hp->uoh_a_bgc_marking[gen_num - uoh_start_generation] = 0; + + total_uoh_a_bgc_planning += hp->uoh_a_bgc_planning[gen_num - uoh_start_generation]; + hp->uoh_a_bgc_planning[gen_num - uoh_start_generation] = 0; + } } dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd", total_uoh_a_no_bgc, diff --git a/src/coreclr/gc/gc.h b/src/coreclr/gc/gc.h index a1093b5e76ceca..df6dc8a42f6c19 100644 --- a/src/coreclr/gc/gc.h +++ b/src/coreclr/gc/gc.h @@ -124,7 +124,10 @@ enum gc_generation_num ephemeral_generation_count = max_generation, // number of all generations - total_generation_count = poh_generation + 1 + total_generation_count = poh_generation + 1, + + // number of uoh generations + uoh_generation_count = total_generation_count - ephemeral_generation_count }; #ifdef GC_CONFIG_DRIVEN diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 31724d7aef36e5..04cb36ed2784f8 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -3440,9 +3440,9 @@ class gc_heap PER_HEAP_FIELD_SINGLE_GC uint8_t* next_sweep_obj; PER_HEAP_FIELD_SINGLE_GC uint8_t* current_sweep_pos; - PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_no_bgc; - PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_marking[total_generation_count - uoh_start_generation]; - PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_planning[total_generation_count - uoh_start_generation]; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_no_bgc[uoh_generation_count]; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_marking[uoh_generation_count]; + PER_HEAP_FIELD_SINGLE_GC size_t uoh_a_bgc_planning[uoh_generation_count]; #ifdef DOUBLY_LINKED_FL PER_HEAP_FIELD_SINGLE_GC heap_segment* current_sweep_seg; From 6547c2c6662c7d216e26359ea84ec404bf185086 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Wed, 17 Apr 2024 15:33:43 -0700 Subject: [PATCH 15/17] loops -> i --- src/coreclr/gc/gc.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index ef893b319c0132..acd313d47916c6 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -15036,11 +15036,11 @@ gc_heap::init_gc_heap (int h_number) make_mark_stack(arr); #ifdef BACKGROUND_GC - for (int gen_num = uoh_start_generation; gen_num < total_generation_count; gen_num++) + for (int i = uoh_start_generation; i < total_generation_count; i++) { - uoh_a_no_bgc[gen_num - uoh_start_generation] = 0; - uoh_a_bgc_marking[gen_num - uoh_start_generation] = 0; - uoh_a_bgc_planning[gen_num - uoh_start_generation] = 0; + uoh_a_no_bgc[i - uoh_start_generation] = 0; + uoh_a_bgc_marking[i - uoh_start_generation] = 0; + uoh_a_bgc_planning[i - uoh_start_generation] = 0; } #ifdef BGC_SERVO_TUNING bgc_maxgen_end_fl_size = 0; @@ -46306,11 +46306,11 @@ void gc_heap::background_sweep() // We also need to adjust size_before for UOH allocations that occurred during sweeping. gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap(); - for (int gen_num = uoh_start_generation; gen_num < total_generation_count; gen_num++) + for (int i = uoh_start_generation; i < total_generation_count; i++) { - assert(uoh_a_bgc_marking[gen_num - uoh_start_generation] == 0); - assert(uoh_a_no_bgc[gen_num - uoh_start_generation] == 0); - current_gc_data_per_heap->gen_data[gen_num].size_before += uoh_a_bgc_planning[gen_num - uoh_start_generation]; + assert(uoh_a_bgc_marking[i - uoh_start_generation] == 0); + assert(uoh_a_no_bgc[i - uoh_start_generation] == 0); + current_gc_data_per_heap->gen_data[i].size_before += uoh_a_bgc_planning[i - uoh_start_generation]; } #ifdef DOUBLY_LINKED_FL @@ -50304,18 +50304,18 @@ void gc_heap::get_and_reset_uoh_alloc_info() // before we lose the values here. gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap(); // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before. - for (int gen_num = uoh_start_generation; gen_num < total_generation_count; gen_num++) + for (int i = uoh_start_generation; i < total_generation_count; i++) { - current_gc_data_per_heap->gen_data[gen_num].size_before += hp->uoh_a_bgc_marking[gen_num - uoh_start_generation]; + current_gc_data_per_heap->gen_data[i].size_before += hp->uoh_a_bgc_marking[i - uoh_start_generation]; - total_uoh_a_no_bgc += hp->uoh_a_no_bgc[gen_num - uoh_start_generation]; - hp->uoh_a_no_bgc[gen_num - uoh_start_generation] = 0; + total_uoh_a_no_bgc += hp->uoh_a_no_bgc[i - uoh_start_generation]; + hp->uoh_a_no_bgc[i - uoh_start_generation] = 0; - total_uoh_a_bgc_marking += hp->uoh_a_bgc_marking[gen_num - uoh_start_generation]; - hp->uoh_a_bgc_marking[gen_num - uoh_start_generation] = 0; + total_uoh_a_bgc_marking += hp->uoh_a_bgc_marking[i - uoh_start_generation]; + hp->uoh_a_bgc_marking[i - uoh_start_generation] = 0; - total_uoh_a_bgc_planning += hp->uoh_a_bgc_planning[gen_num - uoh_start_generation]; - hp->uoh_a_bgc_planning[gen_num - uoh_start_generation] = 0; + total_uoh_a_bgc_planning += hp->uoh_a_bgc_planning[i - uoh_start_generation]; + hp->uoh_a_bgc_planning[i - uoh_start_generation] = 0; } } dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd", From 5868149b053df95f03b1d747b06e8ca836ad66bd Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Mon, 29 Apr 2024 11:41:41 -0700 Subject: [PATCH 16/17] use correct uoh base --- src/coreclr/gc/gc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/gc/gc.h b/src/coreclr/gc/gc.h index df6dc8a42f6c19..f91fa280abecae 100644 --- a/src/coreclr/gc/gc.h +++ b/src/coreclr/gc/gc.h @@ -127,7 +127,7 @@ enum gc_generation_num total_generation_count = poh_generation + 1, // number of uoh generations - uoh_generation_count = total_generation_count - ephemeral_generation_count + uoh_generation_count = total_generation_count - loh_generation }; #ifdef GC_CONFIG_DRIVEN From 838cbc6cbfeed4df4d6d1f8de79317e679f2a6c8 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Wed, 1 May 2024 10:51:13 -0700 Subject: [PATCH 17/17] use better name --- src/coreclr/gc/gc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/gc/gc.h b/src/coreclr/gc/gc.h index f91fa280abecae..e80853489a2410 100644 --- a/src/coreclr/gc/gc.h +++ b/src/coreclr/gc/gc.h @@ -127,7 +127,7 @@ enum gc_generation_num total_generation_count = poh_generation + 1, // number of uoh generations - uoh_generation_count = total_generation_count - loh_generation + uoh_generation_count = total_generation_count - uoh_start_generation }; #ifdef GC_CONFIG_DRIVEN