From c4b37bb8ba02d3db862685ba24de3f37f0f4746b Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 21 Jan 2021 21:22:42 +0000 Subject: [PATCH 1/5] Introduce T_PAYLOAD to store object data on the eden heap Co-authored-by: peterzhu2118 --- gc.c | 274 ++++++++++++++++++++++++++++- include/ruby/internal/value_type.h | 7 + ractor_core.h | 4 + vm_eval.c | 3 + 4 files changed, 287 insertions(+), 1 deletion(-) diff --git a/gc.c b/gc.c index 30badf975f70d4..9a6c9d3d91766f 100644 --- a/gc.c +++ b/gc.c @@ -552,6 +552,28 @@ struct RMoved { #define RMOVED(obj) ((struct RMoved *)(obj)) +#if USE_RVARGC +/* Disable RINCGC because it's broken for now */ +#ifdef USE_RINCGC +# undef USE_RINCGC +# define USE_RINCGC 0 +#endif + +#define RPAYLOAD(val) (RBIMPL_CAST(struct RPayload *)(val)) + +#define NEXT_SLOT(x) ((VALUE)x + sizeof(RVALUE)) + +struct RPayloadHead { + VALUE flags; + unsigned short length; +}; + +struct RPayload { + struct RPayloadHead head; + char data[]; +}; +#endif /* USE_RVARGC */ + #if (SIZEOF_DOUBLE > SIZEOF_VALUE) && (defined(_MSC_VER) || defined(__CYGWIN__)) #pragma pack(push, 4) /* == SIZEOF_VALUE: magic for reducing sizeof(RVALUE): 24 -> 20 */ #endif @@ -563,6 +585,9 @@ typedef struct RVALUE { struct RVALUE *next; } free; struct RMoved moved; +#if USE_RVARGC + struct RPayload payload; +#endif struct RBasic basic; struct RObject object; struct RClass klass; @@ -802,6 +827,12 @@ typedef struct rb_objspace { } rincgc; #endif +#if USE_RVARGC + struct { + unsigned short requested_slots; + } rvargc; +#endif + st_table *id_to_obj_tbl; st_table *obj_to_id_tbl; @@ -847,6 +878,10 @@ struct heap_page { bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT]; bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT]; +#if USE_RVARGC + bits_t payload_bits[HEAP_PAGE_BITMAP_LIMIT]; +#endif + /* If set, the object is not movable */ bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT]; }; @@ -871,6 +906,9 @@ struct heap_page { #define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0]) #define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0]) #define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0]) +#if USE_RVARGC +#define GET_HEAP_PAYLOAD_BITS(x) (&GET_HEAP_PAGE(x)->payload_bits[0]) +#endif /* Aliases */ #define rb_objspace (*rb_objspace_of(GET_VM())) @@ -1218,10 +1256,16 @@ tick(void) #define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj)) #define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj)) #define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj)) +#if USE_RVARGC +#define RVALUE_PAYLOAD_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PAYLOAD_BITS(obj), (obj)) +#endif #define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj)) #define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj)) #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj)) +#if USE_RVARGC +#define RVALUE_PAGE_PAYLOAD(page, obj) MARKED_IN_BITMAP((page)->payload_bits, (obj)) +#endif #define RVALUE_OLD_AGE 3 #define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */ @@ -2079,6 +2123,41 @@ gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb } \ } while (0) +#if USE_RVARGC +static inline void +newobj_init_region_as_payload(rb_objspace_t *objspace, unsigned short slots, VALUE start, rb_ractor_t *cr) +{ + GC_ASSERT(slots > 1); + GC_ASSERT(BUILTIN_TYPE(start) == T_NONE); + + struct heap_page *page = cr->newobj_cache.using_page; + + for (int i = 0; i < slots; i++) { + VALUE p = start + (i * sizeof(RVALUE)); + GC_ASSERT(BUILTIN_TYPE(p) == T_NONE); + GC_ASSERT(GET_HEAP_PAGE(p) == page); + GC_ASSERT(!RVALUE_PAGE_PAYLOAD(page, p)); + MARK_IN_BITMAP(page->payload_bits, p); + } + + RPAYLOAD(start)->head.flags = T_PAYLOAD; + RPAYLOAD(start)->head.length = slots; + + // TODO: keep track of payloads separately + objspace->total_allocated_objects += slots; + +#if RGENGC_CHECK_MODE + for (int i = 0; i < slots; i++) { + VALUE p = start + i * sizeof(RVALUE); + GC_ASSERT(RVALUE_MARK_BITMAP(p) == 0); + GC_ASSERT(RVALUE_MARKING_BITMAP(p) == FALSE); + GC_ASSERT(RVALUE_UNCOLLECTIBLE_BITMAP(p) == FALSE); + GC_ASSERT(RVALUE_WB_UNPROTECTED_BITMAP(p) == FALSE); + } +#endif +} +#endif /* USE_RVARGC */ + static inline VALUE newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj) { @@ -2169,14 +2248,67 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, return obj; } +#if USE_RVARGC +static inline RVALUE * +find_freelist_gap(struct heap_page *page, RVALUE **freelist, unsigned short slots) +{ + if (!*freelist) return 0; + + RVALUE **end_ptr = freelist; + RVALUE *end = *freelist; + + do { + // If we're right at the beginning of the page, then break. + if (end - slots + 1 < page->start) break; + + RVALUE *curr = end->as.free.next; + unsigned short i; + for (i = 1; i < slots; i++) { + if (!curr) break; + + void *poisoned = asan_poisoned_object_p((VALUE)curr); + asan_unpoison_object((VALUE)curr, false); + // if any adjacent slots within range are occupied + // then we need to break out and try the next free region + if (BUILTIN_TYPE((VALUE)curr) != T_NONE) break; + // if the this freelist slot is not adjacent then bail + if (end - curr != i) break; + if (poisoned) asan_poison_object((VALUE)curr); + curr = curr->as.free.next; + } + + if (i == slots) { + RVALUE *start = end - slots + 1; + for (RVALUE *p = start; p <= end; p++) { + asan_unpoison_object((VALUE)p, false); + GC_ASSERT(BUILTIN_TYPE((VALUE)p) == T_NONE); + } + *end_ptr = start->as.free.next; + return start; + } else { + end_ptr = &end->as.free.next; + } + } while ((end = RANY(end)->as.free.next)); + + return NULL; +} +#endif + + static inline VALUE ractor_cached_freeobj(rb_objspace_t *objspace, rb_ractor_t *cr) { +#if USE_RVARGC + RVALUE *p = find_freelist_gap(cr->newobj_cache.using_page, &cr->newobj_cache.freelist, cr->newobj_cache.requested_payload_slots + 1); +#else RVALUE *p = cr->newobj_cache.freelist; +#endif if (p) { VALUE obj = (VALUE)p; +#if !USE_RVARGC cr->newobj_cache.freelist = p->as.free.next; +#endif asan_unpoison_object(obj, true); return obj; } @@ -2210,7 +2342,9 @@ static inline void ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr) { ASSERT_vm_locking(); +#if !USE_RVARGC GC_ASSERT(cr->newobj_cache.freelist == NULL); +#endif struct heap_page *page = heap_next_freepage(objspace, heap_eden); @@ -2234,6 +2368,10 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t * RB_VM_LOCK_ENTER_CR_LEV(cr, &lev); { +#if USE_RVARGC + objspace->rvargc.requested_slots = cr->newobj_cache.requested_payload_slots + 1; +#endif + if (UNLIKELY(during_gc || ruby_gc_stressful)) { if (during_gc) { dont_gc_on(); @@ -2254,6 +2392,11 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t * } GC_ASSERT(obj != 0); newobj_init(klass, flags, wb_protected, objspace, obj); +#if USE_RVARGC + if (cr->newobj_cache.requested_payload_slots > 0) { + newobj_init_region_as_payload(objspace, cr->newobj_cache.requested_payload_slots, NEXT_SLOT(obj), cr); + } +#endif gc_event_hook(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj); } RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev); @@ -2303,6 +2446,12 @@ newobj_of0(VALUE klass, VALUE flags, int wb_protected, rb_ractor_t *cr) (obj = ractor_cached_freeobj(objspace, cr)) != Qfalse)) { newobj_init(klass, flags, wb_protected, objspace, obj); + +#if USE_RVARGC + if (cr->newobj_cache.requested_payload_slots > 0) { + newobj_init_region_as_payload(objspace, cr->newobj_cache.requested_payload_slots, NEXT_SLOT(obj), cr); + } +#endif } else { RB_DEBUG_COUNTER_INC(obj_newobj_slowpath); @@ -2312,6 +2461,10 @@ newobj_of0(VALUE klass, VALUE flags, int wb_protected, rb_ractor_t *cr) newobj_slowpath_wb_unprotected(klass, flags, objspace, cr); } +#if USE_RVARGC + cr->newobj_cache.requested_payload_slots = 0; +#endif + return obj; } @@ -2360,6 +2513,19 @@ rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flag return newobj_of_cr(rb_ec_ractor_ptr(ec), klass, flags, 0, 0, 0, TRUE); } +#if USE_RVARGC +void * +rb_payload_start_zero(VALUE obj) +{ + struct RPayload *payload = RPAYLOAD(NEXT_SLOT(obj)); + GC_ASSERT(BUILTIN_TYPE((VALUE)payload) == T_PAYLOAD); + GC_ASSERT(RVALUE_PAYLOAD_BITMAP(payload)); + memset(payload->data, 0, payload->head.length * sizeof(RVALUE) - sizeof(struct RPayloadHead)); + + return payload->data; +} +#endif + /* for compatibility */ VALUE @@ -2998,6 +3164,9 @@ obj_free(rb_objspace_t *objspace, VALUE obj) RB_DEBUG_COUNTER_INC(obj_complex); break; case T_MOVED: +#if USE_RVARGC + case T_PAYLOAD: +#endif break; case T_ICLASS: /* Basically , T_ICLASS shares table with the module */ @@ -3173,6 +3342,12 @@ Init_heap(void) } #endif +#if USE_RVARGC + /* If we've compiled with the experimental RVARGC we have to disable compaction + * because the compaction algorithm only supports fixed width objects. */ + ruby_enable_autocompact = 0; +#endif + objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL); objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type); objspace->obj_to_id_tbl = st_init_numtable(); @@ -3225,9 +3400,22 @@ objspace_each_objects_without_setup(rb_objspace_t *objspace, each_obj_callback * pstart = page->start; pend = pstart + page->total_slots; +#if USE_RVARGC + RVALUE *p = pstart; + while (p < pend) { + if (!MARKED_IN_BITMAP(page->payload_bits, p)) { + if ((*callback)(p, p + 1, sizeof(RVALUE), data)) { + break; + } + } + + p++; + } +#else if ((*callback)(pstart, pend, sizeof(RVALUE), data)) { break; } +#endif } } @@ -3819,6 +4007,11 @@ rb_objspace_call_finalizer(rb_objspace_t *objspace) VALUE vp = (VALUE)p; void *poisoned = asan_poisoned_object_p(vp); asan_unpoison_object(vp, false); +#if USE_RVARGC + if (RVALUE_PAGE_PAYLOAD(heap_pages_sorted[i], p)) { + goto skip; + } +#endif switch (BUILTIN_TYPE(vp)) { case T_DATA: if (!DATA_PTR(p) || !RANY(p)->as.data.dfree) break; @@ -3845,6 +4038,9 @@ rb_objspace_call_finalizer(rb_objspace_t *objspace) default: break; } +#if USE_RVARGC + skip: +#endif if (poisoned) { GC_ASSERT(BUILTIN_TYPE(vp) == T_NONE); asan_poison_object(vp); @@ -4296,6 +4492,9 @@ obj_memsize_of(VALUE obj, int use_all_types) case T_ZOMBIE: case T_MOVED: +#if USE_RVARGC + case T_PAYLOAD: +#endif break; default: @@ -4422,6 +4621,12 @@ count_objects(int argc, VALUE *argv, VALUE os) VALUE vp = (VALUE)p; void *poisoned = asan_poisoned_object_p(vp); asan_unpoison_object(vp, false); + +#if USE_RVARGC + if (RVALUE_PAYLOAD_BITMAP(p)) + continue; +#endif + if (p->as.basic.flags) { counts[BUILTIN_TYPE(vp)]++; } @@ -4864,6 +5069,11 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_ RVALUE *p, *offset; bits_t *bits, bitset; +#if USE_RVARGC + bits_t *pbits; + pbits = sweep_page->payload_bits; +#endif + gc_report(2, objspace, "page_sweep: start.\n"); if (heap->compact_cursor) { @@ -4896,7 +5106,11 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_ } for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) { +#if USE_RVARGC + bitset = ~bits[i] & ~pbits[i]; +#else bitset = ~bits[i]; +#endif if (bitset) { p = offset + i * BITS_BITLENGTH; do { @@ -4976,6 +5190,29 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_ } } +#if USE_RVARGC + if (objspace->rvargc.requested_slots > 0) { + size_t consecutive_free_len = 0; + for (i = 0; i < sweep_page->total_slots; i++) { + // This page satisfies the requested slot size + if (consecutive_free_len == objspace->rvargc.requested_slots) { + objspace->rvargc.requested_slots = 0; + break; + } + + RVALUE *p = sweep_page->start + i; + void *poisoned = asan_poisoned_object_p((VALUE)p); + asan_unpoison_object((VALUE)p, false); + if (BUILTIN_TYPE((VALUE)p) == T_NONE) { + consecutive_free_len++; + } else { + consecutive_free_len = 0; + } + if (poisoned) asan_poison_object((VALUE)p); + } + } +#endif + if (heap->compact_cursor) { if (gc_fill_swept_page(objspace, heap, sweep_page, &freed_slots, &empty_slots)) { gc_compact_finish(objspace, heap); @@ -5128,9 +5365,9 @@ gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap) { struct heap_page *sweep_page = heap->sweeping_page; int unlink_limit = 3; +#if GC_ENABLE_INCREMENTAL_MARK int swept_slots = 0; -#if GC_ENABLE_INCREMENTAL_MARK int need_pool = will_be_incremental_marking(objspace) ? TRUE : FALSE; gc_report(2, objspace, "gc_sweep_step (need_pool: %d)\n", need_pool); @@ -5453,6 +5690,9 @@ push_mark_stack(mark_stack_t *stack, VALUE data) case T_NIL: case T_FIXNUM: case T_MOVED: +#if USE_RVARGC + case T_PAYLOAD: +#endif rb_bug("push_mark_stack() called for broken object"); break; @@ -5958,6 +6198,10 @@ gc_mark_maybe(rb_objspace_t *objspace, VALUE obj) (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj)); if (is_pointer_to_heap(objspace, (void *)obj)) { +#if USE_RVARGC + if (RVALUE_PAYLOAD_BITMAP(obj)) + return; +#endif void *ptr = __asan_region_is_poisoned((void *)obj, SIZEOF_VALUE); asan_unpoison_object(obj, false); @@ -5966,6 +6210,10 @@ gc_mark_maybe(rb_objspace_t *objspace, VALUE obj) case T_ZOMBIE: case T_NONE: break; +#if USE_RVARGC + case T_PAYLOAD: + rb_bug("Attempting to mark T_PAYLOAD\n"); +#endif default: gc_mark_and_pin(objspace, obj); break; @@ -6930,6 +7178,13 @@ verify_internal_consistency_i(void *page_start, void *page_end, size_t stride, v void *poisoned = asan_poisoned_object_p(obj); asan_unpoison_object(obj, false); +#if USE_RVARGC + if (RVALUE_PAYLOAD_BITMAP(obj)) { + data->live_object_count++; + continue; + } +#endif + if (is_live_object(objspace, obj)) { /* count objects */ data->live_object_count++; @@ -7393,6 +7648,14 @@ gc_marks_finish(rb_objspace_t *objspace) } } +#if USE_RVARGC + // If sweeping did not satisfy the requested slot size, allow a new page to be created + if (objspace->rvargc.requested_slots) { + objspace->rvargc.requested_slots = 0; + heap_set_increment(objspace, 1); + } +#endif + if (full_marking) { /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */ const double r = gc_params.oldobject_limit_factor; @@ -9986,7 +10249,13 @@ gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v) rb_raise(rb_eNotImpError, "Automatic compaction isn't available on this platform"); } #endif +#if USE_RVARGC + /* If we've compiled with the experimental RVARGC we have to disable compaction + * because the compaction algorithm only supports fixed width objects. */ + ruby_enable_autocompact = 0; +#else ruby_enable_autocompact = RTEST(v); +#endif return v; } @@ -12767,6 +13036,9 @@ Init_GC(void) #define OPT(o) if (o) rb_ary_push(opts, rb_fstring_lit(#o)) OPT(GC_DEBUG); OPT(USE_RGENGC); +#if USE_RVARGC + OPT(USE_RVARGC); +#endif OPT(RGENGC_DEBUG); OPT(RGENGC_CHECK_MODE); OPT(RGENGC_PROFILE); diff --git a/include/ruby/internal/value_type.h b/include/ruby/internal/value_type.h index 6f24f08910897e..0f3be4c8162bae 100644 --- a/include/ruby/internal/value_type.h +++ b/include/ruby/internal/value_type.h @@ -80,6 +80,9 @@ #define T_TRUE RUBY_T_TRUE #define T_UNDEF RUBY_T_UNDEF #define T_ZOMBIE RUBY_T_ZOMBIE +#if USE_RVARGC +#define T_PAYLOAD RUBY_T_PAYLOAD +#endif #define BUILTIN_TYPE RB_BUILTIN_TYPE #define DYNAMIC_SYM_P RB_DYNAMIC_SYM_P @@ -132,6 +135,10 @@ ruby_value_type { RUBY_T_SYMBOL = 0x14, /**< @see struct ::RSymbol */ RUBY_T_FIXNUM = 0x15, /**< Integers formerly known as Fixnums. */ RUBY_T_UNDEF = 0x16, /**< @see ::RUBY_Qundef */ +#if USE_RVARGC + RUBY_T_PAYLOAD = 0x17, /**< @see struct ::RPayload */ +#endif + RUBY_T_IMEMO = 0x1a, /**< @see struct ::RIMemo */ RUBY_T_NODE = 0x1b, /**< @see struct ::RNode */ diff --git a/ractor_core.h b/ractor_core.h index fb104cd22c78f2..46515a022cd28c 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -142,6 +142,10 @@ struct rb_ractor_struct { struct { struct RVALUE *freelist; struct heap_page *using_page; + +#if USE_RVARGC + unsigned short requested_payload_slots; +#endif } newobj_cache; // gc.c rb_objspace_reachable_objects_from diff --git a/vm_eval.c b/vm_eval.c index eae9464a1b3391..711f0c160ffb0b 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -729,6 +729,9 @@ rb_type_str(enum ruby_value_type type) case type_case(T_ICLASS); case type_case(T_ZOMBIE); case type_case(T_MOVED); +#if USE_RVARGC + case type_case(T_PAYLOAD); +#endif case T_MASK: break; } #undef type_case From 1b469cb2417f8e40b28e76c2da82923256f94b42 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 21 Jan 2021 20:05:39 +0000 Subject: [PATCH 2/5] Expose the variable width allocation API to other Ruby types Co-authored-by: peterzhu2118 --- gc.c | 25 +++++++++++++++++++++++++ internal/gc.h | 13 +++++++++++++ 2 files changed, 38 insertions(+) diff --git a/gc.c b/gc.c index 9a6c9d3d91766f..7b18e6446f271f 100644 --- a/gc.c +++ b/gc.c @@ -2514,6 +2514,31 @@ rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flag } #if USE_RVARGC +static inline VALUE +newobj_of_with_size(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, size_t payload_size) +{ + rb_ractor_t *cr = GET_RACTOR(); + unsigned short slots = (unsigned short)roomof(payload_size + sizeof(struct RPayloadHead), sizeof(RVALUE)); + GC_ASSERT(cr->newobj_cache.requested_payload_slots == 0); + GC_ASSERT(slots > 0); + cr->newobj_cache.requested_payload_slots = slots; + return newobj_of_cr(cr, klass, flags, v1, v2, v3, wb_protected); +} + +VALUE +rb_wb_unprotected_newobj_of_with_size(VALUE klass, VALUE flags, size_t payload_size) +{ + GC_ASSERT((flags & FL_WB_PROTECTED) == 0); + return newobj_of_with_size(klass, flags, 0, 0, 0, FALSE, payload_size); +} + +VALUE +rb_wb_protected_newobj_of_with_size(VALUE klass, VALUE flags, size_t payload_size) +{ + GC_ASSERT((flags & FL_WB_PROTECTED) == 0); + return newobj_of_with_size(klass, flags, 0, 0, 0, TRUE, payload_size); +} + void * rb_payload_start_zero(VALUE obj) { diff --git a/internal/gc.h b/internal/gc.h index a602f0c9b34495..80a65030445502 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -39,6 +39,14 @@ struct rb_objspace; /* in vm_core.h */ #define NEWOBJ_OF(var, T, c, f) RB_NEWOBJ_OF((var), T, (c), (f)) #define RB_OBJ_GC_FLAGS_MAX 6 /* used in ext/objspace */ +#if USE_RVARGC +#define RB_NEWOBJ_OF_SIZE(var, T, c, f, s) \ + T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ + rb_wb_protected_newobj_of_with_size((c), (f) & ~FL_WB_PROTECTED, s) : \ + rb_wb_unprotected_newobj_of_with_size((c), (f), s)) +#define NEWOBJ_OF_SIZE RB_NEWOBJ_OF_SIZE +#endif + #ifndef USE_UNALIGNED_MEMBER_ACCESS # define UNALIGNED_MEMBER_ACCESS(expr) (expr) #elif ! USE_UNALIGNED_MEMBER_ACCESS @@ -92,6 +100,11 @@ const char *rb_objspace_data_type_name(VALUE obj); VALUE rb_wb_protected_newobj_of(VALUE, VALUE); VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE); VALUE rb_ec_wb_protected_newobj_of(struct rb_execution_context_struct *ec, VALUE klass, VALUE flags); +#if USE_RVARGC +VALUE rb_wb_unprotected_newobj_of_with_size(VALUE klass, VALUE flags, size_t size); +VALUE rb_wb_protected_newobj_of_with_size(VALUE klass, VALUE flags, size_t size); +void *rb_payload_data_start_ptr(VALUE obj); +#endif size_t rb_obj_memsize_of(VALUE); void rb_gc_verify_internal_consistency(void); size_t rb_obj_gc_flags(VALUE, ID[], size_t); From f4d78885f8788499a00c8a72ba18179e2f2ea463 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 21 Jan 2021 20:09:24 +0000 Subject: [PATCH 3/5] Implement variable width allocation for T_CLASS/rb_classext_t Co-authored-by: peterzhu2118 --- class.c | 5 +++++ gc.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- internal/gc.h | 2 +- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/class.c b/class.c index f97956c1d3fb65..343ef870862b01 100644 --- a/class.c +++ b/class.c @@ -173,8 +173,13 @@ rb_class_detach_module_subclasses(VALUE klass) static VALUE class_alloc(VALUE flags, VALUE klass) { +#if USE_RVARGC + NEWOBJ_OF_SIZE(obj, struct RClass, klass, (flags & T_MASK) | FL_PROMOTED1 /* start from age == 2 */ | (RGENGC_WB_PROTECTED_CLASS ? FL_WB_PROTECTED : 0), sizeof(rb_classext_t)); + obj->ptr = rb_payload_start_zero((VALUE)obj); +#else NEWOBJ_OF(obj, struct RClass, klass, (flags & T_MASK) | FL_PROMOTED1 /* start from age == 2 */ | (RGENGC_WB_PROTECTED_CLASS ? FL_WB_PROTECTED : 0)); obj->ptr = ZALLOC(rb_classext_t); +#endif /* ZALLOC RCLASS_IV_TBL(obj) = 0; RCLASS_CONST_TBL(obj) = 0; diff --git a/gc.c b/gc.c index 7b18e6446f271f..37ca239ec2eb0f 100644 --- a/gc.c +++ b/gc.c @@ -1616,6 +1616,14 @@ RVALUE_WHITE_P(VALUE obj) return RVALUE_MARKED(obj) == FALSE; } +#if USE_RVARGC +int +rb_is_payload_object(VALUE obj) +{ + return !!MARKED_IN_BITMAP(GET_HEAP_PAYLOAD_BITS(obj), obj); +} +#endif + /* --------------------------- ObjectSpace ----------------------------- */ @@ -2962,6 +2970,30 @@ obj_free_object_id(rb_objspace_t *objspace, VALUE obj) } } +#if USE_RVARGC +static void +gc_payload_free(rb_objspace_t *objspace, void *ptr) +{ + VALUE pstart = (VALUE)ptr - sizeof(struct RPayloadHead); + GC_ASSERT(pstart % sizeof(RVALUE) == 0); + GC_ASSERT(BUILTIN_TYPE(pstart) == T_PAYLOAD); + + int size = RPAYLOAD(pstart)->head.length; + + struct heap_page *page = GET_HEAP_PAGE(pstart); + + for (int i = 0; i < size; i++) { + VALUE slot = pstart + i * sizeof(RVALUE); + + GC_ASSERT(GET_HEAP_PAGE(slot) == page); + heap_page_add_freeobj(objspace, page, slot); + + GC_ASSERT(RVALUE_PAYLOAD_BITMAP(slot)); + CLEAR_IN_BITMAP(page->payload_bits, slot); + } +} +#endif + static int obj_free(rb_objspace_t *objspace, VALUE obj) { @@ -3038,8 +3070,12 @@ obj_free(rb_objspace_t *objspace, VALUE obj) } rb_class_remove_from_module_subclasses(obj); rb_class_remove_from_super_subclasses(obj); - if (RCLASS_EXT(obj)) + if (RANY(obj)->as.klass.ptr) +#if USE_RVARGC + gc_payload_free(objspace, RANY(obj)->as.klass.ptr); +#else xfree(RCLASS_EXT(obj)); +#endif RCLASS_EXT(obj) = NULL; (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE); @@ -3209,7 +3245,11 @@ obj_free(rb_objspace_t *objspace, VALUE obj) cc_table_free(objspace, obj, FALSE); rb_class_remove_from_module_subclasses(obj); rb_class_remove_from_super_subclasses(obj); +#if USE_RVARGC + gc_payload_free(objspace, RCLASS_EXT(obj)); +#else xfree(RCLASS_EXT(obj)); +#endif RCLASS_EXT(obj) = NULL; RB_DEBUG_COUNTER_INC(obj_iclass_ptr); @@ -7382,8 +7422,16 @@ gc_verify_internal_consistency_(rb_objspace_t *objspace) /* check relations */ +#if USE_RVARGC + objspace->rvargc.dump_payload = TRUE; +#endif + objspace_each_objects_without_setup(objspace, verify_internal_consistency_i, &data); +#if USE_RVARGC + objspace->rvargc.dump_payload = FALSE; +#endif + if (data.err_count != 0) { #if RGENGC_CHECK_MODE >= 5 objspace->rgengc.error_count = data.err_count; diff --git a/internal/gc.h b/internal/gc.h index 80a65030445502..9904fb240465cd 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -103,7 +103,7 @@ VALUE rb_ec_wb_protected_newobj_of(struct rb_execution_context_struct *ec, VALUE #if USE_RVARGC VALUE rb_wb_unprotected_newobj_of_with_size(VALUE klass, VALUE flags, size_t size); VALUE rb_wb_protected_newobj_of_with_size(VALUE klass, VALUE flags, size_t size); -void *rb_payload_data_start_ptr(VALUE obj); +void *rb_payload_start_zero(VALUE obj); #endif size_t rb_obj_memsize_of(VALUE); void rb_gc_verify_internal_consistency(void); From 355406707f5fad3d29d5bb41b881f5a70a7860ef Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 21 Jan 2021 20:44:50 +0000 Subject: [PATCH 4/5] Export T_PAYLOAD slots as part of ObjectSpace.dump_all Co-authored-by: peterzhu2118 --- ext/objspace/objspace_dump.c | 10 ++++++++++ gc.c | 16 ++++++++++++++-- gc.h | 8 ++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/ext/objspace/objspace_dump.c b/ext/objspace/objspace_dump.c index c3619a96566536..5becb9b48a2bc7 100644 --- a/ext/objspace/objspace_dump.c +++ b/ext/objspace/objspace_dump.c @@ -357,6 +357,12 @@ dump_object(VALUE obj, struct dump_config *dc) dump_append_ref(dc, obj); dump_append(dc, ", \"type\":\""); +#if USE_RVARGC + if (rb_is_payload_object(obj)) { + dump_append(dc, "PAYLOAD\" }\n"); + return; + } +#endif dump_append(dc, obj_type(obj)); dump_append(dc, "\""); @@ -636,7 +642,11 @@ objspace_dump_all(VALUE os, VALUE output, VALUE full, VALUE since) } /* dump all objects */ +#if USE_RVARGC + rb_objspace_each_objects_with_payload(heap_i, &dc); +#else rb_objspace_each_objects(heap_i, &dc); +#endif return dump_result(&dc); } diff --git a/gc.c b/gc.c index 37ca239ec2eb0f..e550caea2f49b6 100644 --- a/gc.c +++ b/gc.c @@ -830,6 +830,7 @@ typedef struct rb_objspace { #if USE_RVARGC struct { unsigned short requested_slots; + bool dump_payload; } rvargc; #endif @@ -1620,7 +1621,7 @@ RVALUE_WHITE_P(VALUE obj) int rb_is_payload_object(VALUE obj) { - return !!MARKED_IN_BITMAP(GET_HEAP_PAYLOAD_BITS(obj), obj); + return !!RVALUE_PAYLOAD_BITMAP(obj); } #endif @@ -3467,8 +3468,9 @@ objspace_each_objects_without_setup(rb_objspace_t *objspace, each_obj_callback * #if USE_RVARGC RVALUE *p = pstart; + while (p < pend) { - if (!MARKED_IN_BITMAP(page->payload_bits, p)) { + if (!MARKED_IN_BITMAP(page->payload_bits, p) || objspace->rvargc.dump_payload) { if ((*callback)(p, p + 1, sizeof(RVALUE), data)) { break; } @@ -3543,6 +3545,16 @@ rb_objspace_each_objects(each_obj_callback *callback, void *data) objspace_each_objects(&rb_objspace, callback, data); } +#if USE_RVARGC +void +rb_objspace_each_objects_with_payload(each_obj_callback *callback, void *data) +{ + rb_objspace.rvargc.dump_payload = TRUE; + objspace_each_objects(&rb_objspace, callback, data); + rb_objspace.rvargc.dump_payload = FALSE; +} +#endif + static void objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data) { diff --git a/gc.h b/gc.h index 5d113cafceb137..0d6aceb8d6f18f 100644 --- a/gc.h +++ b/gc.h @@ -131,6 +131,14 @@ void rb_objspace_each_objects( int (*callback)(void *start, void *end, size_t stride, void *data), void *data); +#if USE_RVARGC +void rb_objspace_each_objects_with_payload( + int (*callback)(void *start, void *end, size_t stride, void *data), + void *data); + +int rb_is_payload_object(VALUE obj); +#endif + void rb_objspace_each_objects_without_setup( int (*callback)(void *, void *, size_t, void *), void *data); From 59552d1b711a9cc2980e908541180f522ce9c6e4 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 21 Jan 2021 19:58:24 +0000 Subject: [PATCH 5/5] Check compilation with USE_RVARGC on Github Actions Co-authored-by: peterzhu2118 --- .github/workflows/compilers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 719917a23046fb..47d18db351c5d9 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -123,6 +123,7 @@ jobs: # - { key: cppflags, name: USE_GC_MALLOC_OBJ_INFO_DETAILS, value: '-DUSE_GC_MALLOC_OBJ_INFO_DETAILS' } - { key: cppflags, name: USE_LAZY_LOAD, value: '-DUSE_LAZY_LOAD' } # - { key: cppflags, name: USE_RINCGC=0, value: '-DUSE_RINCGC=0' } + - { key: cppflags, name: USE_RVARGC=1, value: '-DUSE_RVARGC=1' } # - { key: cppflags, name: USE_SYMBOL_GC=0, value: '-DUSE_SYMBOL_GC=0' } # - { key: cppflags, name: USE_THREAD_CACHE=0, value: '-DUSE_THREAD_CACHE=0' } # - { key: cppflags, name: USE_TRANSIENT_HEAP=0, value: '-DUSE_TRANSIENT_HEAP=0' }