From ea4df9a40d133edac8f3a3255d92ed84c4187ed6 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Fri, 7 Sep 2018 15:41:35 -0400 Subject: [PATCH 01/11] support nested structs and arrays of nested structs --- ext/fiddle/pointer.c | 22 +++++++++ lib/fiddle/cparser.rb | 17 ++++++- lib/fiddle/struct.rb | 75 +++++++++++++++++++++++----- test/fiddle/test_cparser.rb | 21 ++++++++ test/fiddle/test_import.rb | 97 +++++++++++++++++++++++++++++++++++++ test/fiddle/test_pointer.rb | 14 ++++++ 6 files changed, 234 insertions(+), 12 deletions(-) diff --git a/ext/fiddle/pointer.c b/ext/fiddle/pointer.c index 2fb21f83..7483f26b 100644 --- a/ext/fiddle/pointer.c +++ b/ext/fiddle/pointer.c @@ -631,6 +631,27 @@ rb_fiddle_ptr_size_get(VALUE self) return LONG2NUM(RPTR_DATA(self)->size); } +/* + * call-seq: memcpy + * + * Copies the contents of the given pointer into this one. Will copy up to the + * allocated size of the given pointer or the allocated size of this one, + * whichever is smaller (to prevent overflow). + * + * Returns the number of bytes actually copied. + */ +static VALUE +rb_fiddle_ptr_memcpy(VALUE self, VALUE other) +{ + long self_size = NUM2LONG(rb_funcall(self, rb_intern("size"), 0)); + long other_size = NUM2LONG(rb_funcall(other, rb_intern("size"), 0)); + void *self_ptr = NUM2PTR(rb_fiddle_ptr_to_i(self)); + void *other_ptr = NUM2PTR(rb_fiddle_ptr_to_i(other)); + long size = self_size > other_size ? other_size : self_size; + memcpy(self_ptr, other_ptr, size); + return LONG2NUM(size); +} + /* * call-seq: * Fiddle::Pointer[val] => cptr @@ -713,6 +734,7 @@ Init_fiddle_pointer(void) rb_define_method(rb_cPointer, "[]=", rb_fiddle_ptr_aset, -1); rb_define_method(rb_cPointer, "size", rb_fiddle_ptr_size_get, 0); rb_define_method(rb_cPointer, "size=", rb_fiddle_ptr_size_set, 1); + rb_define_method(rb_cPointer, "memcpy", rb_fiddle_ptr_memcpy, 1); /* Document-const: NULL * diff --git a/lib/fiddle/cparser.rb b/lib/fiddle/cparser.rb index cd0a64fe..78d80a27 100644 --- a/lib/fiddle/cparser.rb +++ b/lib/fiddle/cparser.rb @@ -35,12 +35,27 @@ module CParser def parse_struct_signature(signature, tymap=nil) if signature.is_a?(String) signature = split_arguments(signature, /[,;]/) + elsif signature.kind_of?(Hash) + signature = [signature] end mems = [] tys = [] signature.each{|msig| - msig = compact(msig) + msig = compact(msig) if msig.is_a?(String) case msig + when Hash + msig.each do |structure_name, struct_signature| + structure_count = nil + if structure_name.to_s =~ /^([\w\*\s]+)\[(\d+)\]$/ + structure_count = $2.to_i + structure_name = $1 + end + structure_parsed = parse_struct_signature(struct_signature, tymap) + structure_types = structure_parsed[0] + structure_members = structure_parsed[1] + mems.push([structure_name.to_s, structure_members]) + tys.push([structure_types, structure_count]) + end when /^[\w\*\s]+[\*\s](\w+)$/ mems.push($1) tys.push(parse_ctype(msig, tymap)) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index 7c0dedb3..9d31797b 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -20,6 +20,13 @@ def CUnion.entity_class end end + # Wrapper for arrays of structs within a struct + class NestedStructArray < Array + def []=(index, value) + self[index].to_ptr.memcpy(value.to_ptr) + end + end + # Used to construct C classes (CUnion, CStruct, etc) # # Fiddle::Importer#struct and Fiddle::Importer#union wrap this functionality in an @@ -57,6 +64,10 @@ def create(klass, types, members) define_method(:to_ptr){ @entity } define_method(:to_i){ @entity.to_i } members.each{|name| + if name.kind_of?(Array) # name is a nested struct + next if method_defined?(name[0]) + name = name[0] + end define_method(name){ @entity[name] } define_method(name + "="){|val| @entity[name] = val } } @@ -102,9 +113,15 @@ def CStructEntity.size(types) max_align = types.map { |type, count = 1| last_offset = offset - align = PackInfo::ALIGN_MAP[type] - offset = PackInfo.align(last_offset, align) + - (PackInfo::SIZE_MAP[type] * count) + if type.kind_of?(Array) # type is a nested array representing a nested struct + align = CStructEntity.size(type) + offset = PackInfo.align(last_offset, align) + + (align * (count || 1)) + else + align = PackInfo::ALIGN_MAP[type] + offset = PackInfo.align(last_offset, align) + + (PackInfo::SIZE_MAP[type] * count) + end align }.max @@ -118,13 +135,30 @@ def CStructEntity.size(types) # # See also Fiddle::Pointer.new def initialize(addr, types, func = nil) + @addr = addr set_ctypes(types) super(addr, @size, func) end # Set the names of the +members+ in this C struct def assign_names(members) - @members = members + @members = members.map { |member| member.kind_of?(Array) ? member[0] : member } + + @nested_structs = {} + @ctypes.each_with_index do |ty, idx| + if ty.kind_of?(Array) && ty[0].kind_of?(Array) + member = members[idx] + member = member[0] if member.kind_of?(Array) + entity_class = CStructBuilder.create(CStruct, ty[0], members[idx][1]) + @nested_structs[member] ||= if ty[1] + NestedStructArray.new(ty[1].times.map do |i| + entity_class.new(@addr + @offset[idx] + i * CStructEntity.size(ty[0])) + end) + else + entity_class.new(@addr + @offset[idx]) + end + end + end end # Calculates the offsets and sizes for the given +types+ in the struct. @@ -135,12 +169,17 @@ def set_ctypes(types) max_align = types.map { |type, count = 1| orig_offset = offset - align = ALIGN_MAP[type] - offset = PackInfo.align(orig_offset, align) - - @offset << offset - - offset += (SIZE_MAP[type] * count) + if type.kind_of?(Array) # type is a nested array representing a nested struct + align = CStructEntity.size(type) + offset = PackInfo.align(orig_offset, align) + @offset << offset + offset += (align * (count || 1)) + else + align = ALIGN_MAP[type] + offset = PackInfo.align(orig_offset, align) + @offset << offset + offset += (SIZE_MAP[type] * (count || 1)) + end align }.max @@ -156,7 +195,11 @@ def [](name) end ty = @ctypes[idx] if( ty.is_a?(Array) ) - r = super(@offset[idx], SIZE_MAP[ty[0]] * ty[1]) + if ty.first.kind_of?(Array) + return @nested_structs[name] + else + r = super(@offset[idx], SIZE_MAP[ty[0]] * ty[1]) + end else r = super(@offset[idx], SIZE_MAP[ty.abs]) end @@ -188,6 +231,16 @@ def []=(name, val) if( idx.nil? ) raise(ArgumentError, "no such member: #{name}") end + if @nested_structs[name] + if @nested_structs[name].kind_of?(Array) + val.size.times do |i| + @nested_structs[name][i].to_ptr.memcpy(val[i].to_ptr) + end + else + @nested_structs[name].to_ptr.memcpy(val.to_ptr) + end + return + end ty = @ctypes[idx] packer = Packer.new([ty]) val = wrap_arg(val, ty, []) diff --git a/test/fiddle/test_cparser.rb b/test/fiddle/test_cparser.rb index 5d9ac3c8..8ddd06c2 100644 --- a/test/fiddle/test_cparser.rb +++ b/test/fiddle/test_cparser.rb @@ -76,6 +76,27 @@ def test_struct_array assert_equal [[[TYPE_CHAR,80],[TYPE_INT,5]], ['buffer','x']], parse_struct_signature(['char buffer[80]', 'int[5] x']) end + def test_struct_nested_struct + assert_equal [[TYPE_INT, [[TYPE_INT, TYPE_CHAR], nil]], ['x', ['inner', ['i', 'c']]]], parse_struct_signature(['int x', {inner: ['int i', 'char c']}]) + end + + def test_struct_double_nested_struct + assert_equal [[TYPE_INT, [[TYPE_INT, [[TYPE_INT, TYPE_CHAR], nil]], nil]], ['x', ['outer', ['y', ['inner', ['i', 'c']]]]]], + parse_struct_signature(['int x', {outer: ['int y', { inner: ['int i', 'char c'] }]}]) + end + + def test_struct_nested_struct_array + assert_equal [[TYPE_INT, [[TYPE_INT, TYPE_CHAR], 2]], ['x', ['inner', ['i', 'c']]]], parse_struct_signature(['int x', {'inner[2]' => ['int i', 'char c']}]) + end + + def test_struct_double_nested_struct_inner_array + assert_equal [[[[TYPE_INT, [[TYPE_INT, TYPE_CHAR], 2]], nil]], [['outer', ['x', ['inner', ['i', 'c']]]]]], parse_struct_signature(outer: ['int x', { 'inner[2]' => ['int i', 'char c'] }]) + end + + def test_struct_double_nested_struct_outer_array + assert_equal [[TYPE_INT, [[[[TYPE_INT, TYPE_CHAR], nil]], 2]], ['x', ['outer', [['inner', ['i', 'c']]]]]], parse_struct_signature(['int x', {'outer[2]' => { inner: ['int i', 'char c'] }}]) + end + def test_struct_array_str assert_equal [[[TYPE_CHAR,80],[TYPE_INT,5]], ['buffer','x']], parse_struct_signature('char buffer[80], int[5] x') end diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index 99294ea1..a3ea6c2c 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -36,6 +36,16 @@ module LIBC "char c", "unsigned char buff[7]", ] + NestedStruct = struct [ + { + "vertices[2]" => { + position: [ "float x", "float y", "float z" ], + texcoord: [ "float u", "float v" ] + }, + object: [ "int id" ] + }, + "int id" + ] CallCallback = bind("void call_callback(void*, void*)"){ | ptr1, ptr2| f = Function.new(ptr1.to_i, [TYPE_VOIDP], TYPE_VOID) @@ -65,6 +75,7 @@ def test_sizeof() assert_equal(LIBC::MyStruct.size(), LIBC.sizeof(LIBC::MyStruct)) assert_equal(LIBC::MyStruct.size(), LIBC.sizeof(LIBC::MyStruct.malloc())) assert_equal(SIZEOF_LONG_LONG, LIBC.sizeof("long long")) if defined?(SIZEOF_LONG_LONG) + assert_equal(LIBC::NestedStruct.size(), LIBC.sizeof(LIBC::NestedStruct)) end Fiddle.constants.grep(/\ATYPE_(?!VOID\z)(.*)/) do @@ -107,6 +118,92 @@ def test_value() assert_equal([0,1,2], ary.value) end + def test_nested_struct_members() + s = LIBC::NestedStruct.malloc + s.vertices[0].position.x = 1 + s.vertices[0].position.y = 2 + s.vertices[0].position.z = 3 + s.vertices[0].texcoord.u = 4 + s.vertices[0].texcoord.v = 5 + s.vertices[1].position.x = 6 + s.vertices[1].position.y = 7 + s.vertices[1].position.z = 8 + s.vertices[1].texcoord.u = 9 + s.vertices[1].texcoord.v = 10 + s.object.id = 100 + s.id = 101 + assert_equal(1, s.vertices[0].position.x) + assert_equal(2, s.vertices[0].position.y) + assert_equal(3, s.vertices[0].position.z) + assert_equal(4, s.vertices[0].texcoord.u) + assert_equal(5, s.vertices[0].texcoord.v) + assert_equal(6, s.vertices[1].position.x) + assert_equal(7, s.vertices[1].position.y) + assert_equal(8, s.vertices[1].position.z) + assert_equal(9, s.vertices[1].texcoord.u) + assert_equal(10, s.vertices[1].texcoord.v) + assert_equal(100, s.object.id) + assert_equal(101, s.id) + end + + def test_nested_struct_replace_array_element() + s = LIBC::NestedStruct.malloc + s.vertices[0].position.x = 5 + + vertex_struct = Fiddle::Importer.struct [{ + position: [ "float x", "float y", "float z" ], + texcoord: [ "float u", "float v" ] + }] + vertex = vertex_struct.malloc + vertex.position.x = 100 + s.vertices[0] = vertex + + # make sure element was copied by value, but things like memory address + # should not be changed + assert_equal(100, s.vertices[0].position.x) + refute_equal(vertex.object_id, s.vertices[0].object_id) + refute_equal(vertex.to_ptr, s.vertices[0].to_ptr) + end + + def test_nested_struct_replace_entire_array() + s = LIBC::NestedStruct.malloc + vertex_struct = Fiddle::Importer.struct [{ + position: [ "float x", "float y", "float z" ], + texcoord: [ "float u", "float v" ] + }] + + different_struct_same_size = Fiddle::Importer.struct [{ + a: [ 'float i', 'float j', 'float k' ], + b: [ 'float l', 'float m' ] + }] + + same = [vertex_struct.malloc, vertex_struct.malloc] + same[0].position.x = 1; same[1].position.x = 6 + same[0].position.y = 2; same[1].position.y = 7 + same[0].position.z = 3; same[1].position.z = 8 + same[0].texcoord.u = 4; same[1].texcoord.u = 9 + same[0].texcoord.v = 5; same[1].texcoord.v = 10 + s.vertices = same + assert_equal(1, s.vertices[0].position.x); assert_equal(6, s.vertices[1].position.x) + assert_equal(2, s.vertices[0].position.y); assert_equal(7, s.vertices[1].position.y) + assert_equal(3, s.vertices[0].position.z); assert_equal(8, s.vertices[1].position.z) + assert_equal(4, s.vertices[0].texcoord.u); assert_equal(9, s.vertices[1].texcoord.u) + assert_equal(5, s.vertices[0].texcoord.v); assert_equal(10, s.vertices[1].texcoord.v) + + different = [different_struct_same_size.malloc, different_struct_same_size.malloc] + different[0].a.i = 11; different[1].a.i = 16 + different[0].a.j = 12; different[1].a.j = 17 + different[0].a.k = 13; different[1].a.k = 18 + different[0].b.l = 14; different[1].b.l = 19 + different[0].b.m = 15; different[1].b.m = 20 + s.vertices = different + assert_equal(11, s.vertices[0].position.x); assert_equal(16, s.vertices[1].position.x) + assert_equal(12, s.vertices[0].position.y); assert_equal(17, s.vertices[1].position.y) + assert_equal(13, s.vertices[0].position.z); assert_equal(18, s.vertices[1].position.z) + assert_equal(14, s.vertices[0].texcoord.u); assert_equal(19, s.vertices[1].texcoord.u) + assert_equal(15, s.vertices[0].texcoord.v); assert_equal(20, s.vertices[1].texcoord.v) + end + def test_struct() s = LIBC::MyStruct.malloc() s.num = [0,1,2,3,4] diff --git a/test/fiddle/test_pointer.rb b/test/fiddle/test_pointer.rb index b1122aa9..1a4f51b2 100644 --- a/test/fiddle/test_pointer.rb +++ b/test/fiddle/test_pointer.rb @@ -32,6 +32,20 @@ def test_malloc_free_func assert_equal free.to_i, ptr.free.to_i end + def test_memcpy + ptr = Pointer[Marshal.load(Marshal.dump("hello world"))] + smaller = Pointer[Marshal.load(Marshal.dump("1234567890"))] + same = Pointer[Marshal.load(Marshal.dump("12345678901"))] + larger = Pointer[Marshal.load(Marshal.dump("123456789012"))] + + assert_equal(ptr.size - 1, smaller.memcpy(ptr)) + assert_equal(ptr.size , same.memcpy(ptr)) + assert_equal(ptr.size , larger.memcpy(ptr)) + assert_equal("hello worl", smaller.to_s) + assert_equal("hello world", same.to_s) + assert_equal("hello world2", larger.to_s) + end + def test_to_str str = Marshal.load(Marshal.dump("hello world")) ptr = Pointer[str] From 0d5124f63bfbf19b81132efa8cc692a781b5c757 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Fri, 7 Sep 2018 16:07:26 -0400 Subject: [PATCH 02/11] allow an already defined struct to be reused in the definition of another --- lib/fiddle/cparser.rb | 2 ++ lib/fiddle/struct.rb | 2 ++ test/fiddle/test_import.rb | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/lib/fiddle/cparser.rb b/lib/fiddle/cparser.rb index 78d80a27..e12c938c 100644 --- a/lib/fiddle/cparser.rb +++ b/lib/fiddle/cparser.rb @@ -37,6 +37,8 @@ def parse_struct_signature(signature, tymap=nil) signature = split_arguments(signature, /[,;]/) elsif signature.kind_of?(Hash) signature = [signature] + elsif signature.respond_to?(:types) && signature.respond_to?(:members) + return signature.types, signature.members end mems = [] tys = [] diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index 9d31797b..7a796ab5 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -63,6 +63,8 @@ def create(klass, types, members) } define_method(:to_ptr){ @entity } define_method(:to_i){ @entity.to_i } + define_singleton_method(:types) { types } + define_singleton_method(:members) { members } members.each{|name| if name.kind_of?(Array) # name is a nested struct next if method_defined?(name[0]) diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index a3ea6c2c..96c99aa8 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -118,6 +118,14 @@ def test_value() assert_equal([0,1,2], ary.value) end + def test_nested_struct_reusing_other_structs() + position_struct = Fiddle::Importer.struct([ 'float x', 'float y', 'float z' ]) + texcoord_struct = Fiddle::Importer.struct([ 'float u', 'float v' ]) + vertex_struct = Fiddle::Importer.struct(position: position_struct, texcoord: texcoord_struct) + mesh_struct = Fiddle::Importer.struct([{"vertices[2]" => vertex_struct, object: [ "int id" ]}, "int id"]) + assert_equal LIBC::NestedStruct.size, mesh_struct.size + end + def test_nested_struct_members() s = LIBC::NestedStruct.malloc s.vertices[0].position.x = 1 From 3bb4bc43042b0ea9a155077ff3931504cfc871ff Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Mon, 10 Sep 2018 14:04:08 -0400 Subject: [PATCH 03/11] support for structs nested inside of unions --- lib/fiddle/struct.rb | 6 ++++- test/fiddle/test_import.rb | 45 ++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index 7a796ab5..cc2d75e2 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -283,7 +283,11 @@ def CUnionEntity.malloc(types, func=nil) # Fiddle::TYPE_VOIDP ]) #=> 8 def CUnionEntity.size(types) types.map { |type, count = 1| - PackInfo::SIZE_MAP[type] * count + if type.kind_of?(Array) # type is a nested array representing a nested struct + CStructEntity.size(type) * (count || 1) + else + PackInfo::SIZE_MAP[type] * count + end }.max end diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index 96c99aa8..d3c45818 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -36,7 +36,7 @@ module LIBC "char c", "unsigned char buff[7]", ] - NestedStruct = struct [ + StructNestedStruct = struct [ { "vertices[2]" => { position: [ "float x", "float y", "float z" ], @@ -46,6 +46,19 @@ module LIBC }, "int id" ] + UnionNestedStruct = union [ + { + keyboard: [ + 'unsigned int state', + 'char key' + ], + mouse: [ + 'unsigned int button', + 'unsigned short x', + 'unsigned short y' + ] + } + ] CallCallback = bind("void call_callback(void*, void*)"){ | ptr1, ptr2| f = Function.new(ptr1.to_i, [TYPE_VOIDP], TYPE_VOID) @@ -75,7 +88,7 @@ def test_sizeof() assert_equal(LIBC::MyStruct.size(), LIBC.sizeof(LIBC::MyStruct)) assert_equal(LIBC::MyStruct.size(), LIBC.sizeof(LIBC::MyStruct.malloc())) assert_equal(SIZEOF_LONG_LONG, LIBC.sizeof("long long")) if defined?(SIZEOF_LONG_LONG) - assert_equal(LIBC::NestedStruct.size(), LIBC.sizeof(LIBC::NestedStruct)) + assert_equal(LIBC::StructNestedStruct.size(), LIBC.sizeof(LIBC::StructNestedStruct)) end Fiddle.constants.grep(/\ATYPE_(?!VOID\z)(.*)/) do @@ -123,11 +136,17 @@ def test_nested_struct_reusing_other_structs() texcoord_struct = Fiddle::Importer.struct([ 'float u', 'float v' ]) vertex_struct = Fiddle::Importer.struct(position: position_struct, texcoord: texcoord_struct) mesh_struct = Fiddle::Importer.struct([{"vertices[2]" => vertex_struct, object: [ "int id" ]}, "int id"]) - assert_equal LIBC::NestedStruct.size, mesh_struct.size + assert_equal LIBC::StructNestedStruct.size, mesh_struct.size + + + keyboard_event_struct = Fiddle::Importer.struct([ 'unsigned int state', 'char key' ]) + mouse_event_struct = Fiddle::Importer.struct([ 'unsigned int button', 'unsigned short x', 'unsigned short y' ]) + event_union = Fiddle::Importer.union([{ keboard: keyboard_event_struct, mouse: mouse_event_struct}]) + assert_equal LIBC::UnionNestedStruct.size, event_union.size end - def test_nested_struct_members() - s = LIBC::NestedStruct.malloc + def test_struct_nested_struct_members() + s = LIBC::StructNestedStruct.malloc s.vertices[0].position.x = 1 s.vertices[0].position.y = 2 s.vertices[0].position.z = 3 @@ -154,8 +173,16 @@ def test_nested_struct_members() assert_equal(101, s.id) end - def test_nested_struct_replace_array_element() - s = LIBC::NestedStruct.malloc + def test_union_nested_struct_members() + s = LIBC::UnionNestedStruct.malloc + s.keyboard.state = 100 + s.keyboard.key = 101 + assert_equal(100, s.mouse.button) + refute_equal( 0, s.mouse.x) + end + + def test_struct_nested_struct_replace_array_element() + s = LIBC::StructNestedStruct.malloc s.vertices[0].position.x = 5 vertex_struct = Fiddle::Importer.struct [{ @@ -173,8 +200,8 @@ def test_nested_struct_replace_array_element() refute_equal(vertex.to_ptr, s.vertices[0].to_ptr) end - def test_nested_struct_replace_entire_array() - s = LIBC::NestedStruct.malloc + def test_struct_nested_struct_replace_entire_array() + s = LIBC::StructNestedStruct.malloc vertex_struct = Fiddle::Importer.struct [{ position: [ "float x", "float y", "float z" ], texcoord: [ "float u", "float v" ] From 5c99d72f6346ddc064945b09f88fd8e6db47005b Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Mon, 10 Sep 2018 23:49:28 -0400 Subject: [PATCH 04/11] Initialize memory to 0 when calling Fiddle.malloc(). --- ext/fiddle/fiddle.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/fiddle/fiddle.c b/ext/fiddle/fiddle.c index 9f3d1537..bb6b1070 100644 --- a/ext/fiddle/fiddle.c +++ b/ext/fiddle/fiddle.c @@ -47,8 +47,9 @@ static VALUE rb_fiddle_malloc(VALUE self, VALUE size) { void *ptr; - - ptr = (void*)ruby_xmalloc(NUM2SIZET(size)); + size_t sizet = NUM2SIZET(size); + ptr = (void*)ruby_xmalloc(sizet); + memset(ptr, 0, sizet); return PTR2NUM(ptr); } From f01a26d1ffb7d199c6695c786023caf10c4146c1 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Mon, 29 Oct 2018 20:52:22 -0400 Subject: [PATCH 05/11] Allow access to a struct's underlying memory with `struct[offset, length]`. --- lib/fiddle/struct.rb | 8 ++++++-- test/fiddle/test_import.rb | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index cc2d75e2..2153e16d 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -190,7 +190,9 @@ def set_ctypes(types) end # Fetch struct member +name+ - def [](name) + def [](*args) + return super(*args) if args.size > 1 + name = args[0] idx = @members.index(name) if( idx.nil? ) raise(ArgumentError, "no such member: #{name}") @@ -228,7 +230,9 @@ def [](name) end # Set struct member +name+, to value +val+ - def []=(name, val) + def []=(*args) + return super(*args) if args.size > 2 + name, val = *args idx = @members.index(name) if( idx.nil? ) raise(ArgumentError, "no such member: #{name}") diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index d3c45818..ae10b096 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -131,6 +131,14 @@ def test_value() assert_equal([0,1,2], ary.value) end + def test_struct_array_subscript_multiarg() + struct = Fiddle::Importer.struct([ 'int x' ]).malloc + assert_equal("\x00".b * Fiddle::SIZEOF_INT, struct.to_ptr[0, Fiddle::SIZEOF_INT]) + + struct.to_ptr[0, Fiddle::SIZEOF_INT] = "\x01".b * Fiddle::SIZEOF_INT + assert_equal 16843009, struct.x + end + def test_nested_struct_reusing_other_structs() position_struct = Fiddle::Importer.struct([ 'float x', 'float y', 'float z' ]) texcoord_struct = Fiddle::Importer.struct([ 'float u', 'float v' ]) From 4851d07a1aead6340079128a420485e3143b197d Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Mon, 29 Oct 2018 21:44:01 -0400 Subject: [PATCH 06/11] Make accessing a struct's underlying memory more convenient. --- lib/fiddle/struct.rb | 27 +++++++++++++++++++++++++-- test/fiddle/test_import.rb | 10 ++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index 2153e16d..ae5c44c4 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -61,6 +61,8 @@ def create(klass, types, members) @entity = klass.entity_class.new(addr, types) @entity.assign_names(members) } + define_method(:[]) { |*args| @entity.send(:[], *args) } + define_method(:[]=) { |*args| @entity.send(:[]=, *args) } define_method(:to_ptr){ @entity } define_method(:to_i){ @entity.to_i } define_singleton_method(:types) { types } @@ -189,7 +191,18 @@ def set_ctypes(types) @size = PackInfo.align(offset, max_align) end - # Fetch struct member +name+ + # Fetch struct member +name+ if only one argument is specified. If two + # arguments are specified, the first is an offset and the second is a + # length and this method returns the string of +length+ bytes beginning at + # +offset+. + # + # Examples: + # + # my_struct = struct(['int id']).malloc + # my_struct.id = 1 + # my_struct['id'] # => 1 + # my_struct[0, 4] # => "\x01\x00\x00\x00".b + # def [](*args) return super(*args) if args.size > 1 name = args[0] @@ -229,7 +242,17 @@ def [](*args) end end - # Set struct member +name+, to value +val+ + # Set struct member +name+, to value +val+. If more arguments are + # specified, writes the string of bytes to the memory at the given + # +offset+ and +length+. + # + # Examples: + # + # my_struct = struct(['int id']).malloc + # my_struct['id'] = 1 + # my_struct[0, 4] = "\x01\x00\x00\x00".b + # my_struct.id # => 1 + # def []=(*args) return super(*args) if args.size > 2 name, val = *args diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index ae10b096..8a5971ae 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -77,6 +77,16 @@ def test_ensure_call_dlload assert_match(/call dlload before/, err.message) end + def test_struct_memory_access + my_struct = Fiddle::Importer.struct(['int id']).malloc + my_struct['id'] = 1 + my_struct[0, Fiddle::SIZEOF_INT] = "\x01".b * Fiddle::SIZEOF_INT + refute_equal 0, my_struct.id + + my_struct.id = 0 + assert_equal "\x00".b * Fiddle::SIZEOF_INT, my_struct[0, Fiddle::SIZEOF_INT] + end + def test_malloc() s1 = LIBC::Timeval.malloc() s2 = LIBC::Timeval.malloc() From 37112f0a1acb60c7e764916368c66c86d0bc1f21 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Mon, 29 Oct 2018 21:44:18 -0400 Subject: [PATCH 07/11] Add documentation to README for nested structs. --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/README.md b/README.md index 29c21e55..5576c8e4 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,57 @@ floor = Fiddle::Function.new( puts floor.call(3.14159) #=> 3.0 ``` +### Nested Structs + +You can use hashes to create nested structs, where the hash keys are member names and the values are the nested structs: + +```ruby +StudentCollegeDetail = struct [ + 'int college_id', + 'char college_name[50]' +] + +StudentDetail = struct [ + 'int id', + 'char name[20]', + { clg_data: StudentCollegeDetail } +] +``` + +You can also specify an anonymous nested struct, like so: + +```ruby +StudentDetail = struct [ + 'int id', + 'char name[20]', + { + clg_data: struct([ + 'int college_id', + 'char college_name[50]' + ]) + } +] +``` + +The position of a hash (and the order of the keys in the hash, in the case of a hash with multiple values), dictate the offsets of the nested struct in memory. For example, the following are both syntactically valid but will lay out the data in the nested structs differently in memory: + +```ruby +# order of members in memory: position, id, dimensions +Rect = struct [ { position: struct(['float x', 'float y']) }, + 'int id', + { dimensions: struct(['float w', 'float h']) } + ] + +# order of members in memory: id, position, dimensions +Rect = struct [ 'int id', + { + position: struct(['float x', 'float y']), + dimensions: struct(['float w', 'float h']) + } + ] +``` + + ## Development After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. From d0ce3419868d13e495b4873b29189a122af79c97 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Mon, 29 Oct 2018 21:46:20 -0400 Subject: [PATCH 08/11] clarify documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5576c8e4..3026c26c 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ StudentDetail = struct [ ] ``` -The position of a hash (and the order of the keys in the hash, in the case of a hash with multiple values), dictate the offsets of the nested struct in memory. For example, the following are both syntactically valid but will lay out the data in the nested structs differently in memory: +The position of a hash (and the order of the keys in the hash, in the case of a hash with multiple entries), dictate the offsets of the nested struct in memory. The following examples are both syntactically valid but will lay out the structs differently in memory: ```ruby # order of members in memory: position, id, dimensions From e3036a4c84d208849c7e709e96d9d27c1ad46ee1 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Thu, 1 Nov 2018 00:23:51 -0400 Subject: [PATCH 09/11] fix alignment and size when members appear after a nested struct --- lib/fiddle/struct.rb | 23 +++++++++++++++++++---- test/fiddle/test_import.rb | 10 ++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index ae5c44c4..d5d6b2b8 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -96,6 +96,19 @@ class CStructEntity < Fiddle::Pointer include PackInfo include ValueUtil + def CStructEntity.alignment(types) + max = 1 + types.each do |type, count = 1| + if type.kind_of?(Array) # nested struct + n = CStructEntity.alignment(type) + else + n = ALIGN_MAP[type] + end + max = n if n > max + end + max + end + # Allocates a C struct with the +types+ provided. # # When the instance is garbage collected, the C function +func+ is called. @@ -118,9 +131,10 @@ def CStructEntity.size(types) last_offset = offset if type.kind_of?(Array) # type is a nested array representing a nested struct - align = CStructEntity.size(type) + align = CStructEntity.alignment(type) + total_size = CStructEntity.size(type) offset = PackInfo.align(last_offset, align) + - (align * (count || 1)) + (total_size * (count || 1)) else align = PackInfo::ALIGN_MAP[type] offset = PackInfo.align(last_offset, align) + @@ -174,10 +188,11 @@ def set_ctypes(types) max_align = types.map { |type, count = 1| orig_offset = offset if type.kind_of?(Array) # type is a nested array representing a nested struct - align = CStructEntity.size(type) + align = CStructEntity.alignment(type) + total_size = CStructEntity.size(type) offset = PackInfo.align(orig_offset, align) @offset << offset - offset += (align * (count || 1)) + offset += (total_size * (count || 1)) else align = ALIGN_MAP[type] offset = PackInfo.align(orig_offset, align) diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index 8a5971ae..b8969181 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -163,6 +163,16 @@ def test_nested_struct_reusing_other_structs() assert_equal LIBC::UnionNestedStruct.size, event_union.size end + def test_nested_struct_alignment_is_not_its_size() + inner = Fiddle::Importer.struct(['int x', 'int y', 'int z', 'int w']) + outer = Fiddle::Importer.struct(['char a', { 'nested' => inner }, 'char b']) + instance = outer.malloc + offset = instance.to_ptr.instance_variable_get(:"@offset") + assert_equal Fiddle::SIZEOF_INT * 5, offset.last + assert_equal Fiddle::SIZEOF_INT * 6, outer.size + assert_equal instance.to_ptr.size, outer.size + end + def test_struct_nested_struct_members() s = LIBC::StructNestedStruct.malloc s.vertices[0].position.x = 1 From c4db0daa07a45210bbef8023f9bb627668011f1f Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Thu, 1 Nov 2018 00:24:45 -0400 Subject: [PATCH 10/11] fix assignment to elements in a member which is an array --- lib/fiddle/struct.rb | 29 ++++++++++++++++++++++++++++- test/fiddle/test_import.rb | 10 ++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index d5d6b2b8..8912c5b0 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -27,6 +27,33 @@ def []=(index, value) end end + # Wrapper for arrays within a struct + class StructArray < Array + include ValueUtil + + def initialize(ptr, type, initial_values) + @ptr = ptr + @type = type + @align = PackInfo::ALIGN_MAP[type] + @size = Fiddle::PackInfo::SIZE_MAP[type] + @pack_format = Fiddle::PackInfo::PACK_MAP[type] + super(initial_values.collect { |v| unsigned_value(v, type) }) + end + + def to_ptr + @ptr + end + + def []=(index, value) + if index < 0 || index >= size + raise RangeError, 'index %d outside of array bounds 0...%d' % [index, size] + end + + to_ptr[index * @align, @size] = [value].pack(@pack_format) + super(index, value) + end + end + # Used to construct C classes (CUnion, CStruct, etc) # # Fiddle::Importer#struct and Fiddle::Importer#union wrap this functionality in an @@ -251,7 +278,7 @@ def [](*args) if( ty.is_a?(Integer) && (ty < 0) ) return unsigned_value(val, ty) elsif( ty.is_a?(Array) && (ty[0] < 0) ) - return val.collect{|v| unsigned_value(v,ty[0])} + return StructArray.new(self + @offset[idx], ty[0], val) else return val end diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index b8969181..00b771d0 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -209,6 +209,16 @@ def test_union_nested_struct_members() refute_equal( 0, s.mouse.x) end + def test_struct_array_assignment() + instance = Fiddle::Importer.struct(["unsigned int stages[1]"]).malloc + instance.stages[0] = 1024 + assert_equal 1024, instance.stages[0] + assert_equal [1024].pack(Fiddle::PackInfo::PACK_MAP[-Fiddle::TYPE_INT]), + instance.to_ptr[0, Fiddle::SIZEOF_INT] + assert_raise(RangeError) { instance.stages[-1] = 5 } + assert_raise(RangeError) { instance.stages[2] = 5 } + end + def test_struct_nested_struct_replace_array_element() s = LIBC::StructNestedStruct.malloc s.vertices[0].position.x = 5 From 94dbbaf07b1fb400aebbf1deca9cee302fe80e30 Mon Sep 17 00:00:00 2001 From: Colin MacKenzie IV Date: Thu, 15 Nov 2018 14:30:18 -0500 Subject: [PATCH 11/11] Fix incorrect sizing of structs containing unions and pointers. --- lib/fiddle/cparser.rb | 7 +++++-- lib/fiddle/struct.rb | 42 +++++++++++++++++++++++++++++--------- test/fiddle/test_import.rb | 29 ++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 12 deletions(-) diff --git a/lib/fiddle/cparser.rb b/lib/fiddle/cparser.rb index e12c938c..43f2373d 100644 --- a/lib/fiddle/cparser.rb +++ b/lib/fiddle/cparser.rb @@ -38,7 +38,7 @@ def parse_struct_signature(signature, tymap=nil) elsif signature.kind_of?(Hash) signature = [signature] elsif signature.respond_to?(:types) && signature.respond_to?(:members) - return signature.types, signature.members + return signature.types, signature.members, signature.entity_class end mems = [] tys = [] @@ -55,8 +55,11 @@ def parse_struct_signature(signature, tymap=nil) structure_parsed = parse_struct_signature(struct_signature, tymap) structure_types = structure_parsed[0] structure_members = structure_parsed[1] + structure_klass = structure_parsed[2] + ty = [structure_types, structure_count] + ty << structure_klass if structure_klass mems.push([structure_name.to_s, structure_members]) - tys.push([structure_types, structure_count]) + tys.push(ty) end when /^[\w\*\s]+[\*\s](\w+)$/ mems.push($1) diff --git a/lib/fiddle/struct.rb b/lib/fiddle/struct.rb index 8912c5b0..4b53be1d 100644 --- a/lib/fiddle/struct.rb +++ b/lib/fiddle/struct.rb @@ -92,8 +92,10 @@ def create(klass, types, members) define_method(:[]=) { |*args| @entity.send(:[]=, *args) } define_method(:to_ptr){ @entity } define_method(:to_i){ @entity.to_i } + define_method(:offset_of) { |*args| @entity.offset_of(*args) } define_singleton_method(:types) { types } define_singleton_method(:members) { members } + define_singleton_method(:offset_of) { |mbr| klass.entity_class.compute_offset(types, members, mbr) } members.each{|name| if name.kind_of?(Array) # name is a nested struct next if method_defined?(name[0]) @@ -136,6 +138,15 @@ def CStructEntity.alignment(types) max end + def CStructEntity.compute_offset(types, members, mbr) + members.each_with_index do |m, idx| + if (m.kind_of?(Array) ? m[0] : m) == mbr.to_s + return idx == 0 ? 0 : CStructEntity.size(types[0...idx]) + end + end + raise(ArgumentError, "no such member: #{mbr}") + end + # Allocates a C struct with the +types+ provided. # # When the instance is garbage collected, the C function +func+ is called. @@ -154,12 +165,12 @@ def CStructEntity.malloc(types, func = nil) def CStructEntity.size(types) offset = 0 - max_align = types.map { |type, count = 1| + max_align = types.map { |type, count = 1, klass = CStructEntity| last_offset = offset if type.kind_of?(Array) # type is a nested array representing a nested struct - align = CStructEntity.alignment(type) - total_size = CStructEntity.size(type) + align = klass.alignment(type) + total_size = klass.size(type) offset = PackInfo.align(last_offset, align) + (total_size * (count || 1)) else @@ -197,7 +208,7 @@ def assign_names(members) entity_class = CStructBuilder.create(CStruct, ty[0], members[idx][1]) @nested_structs[member] ||= if ty[1] NestedStructArray.new(ty[1].times.map do |i| - entity_class.new(@addr + @offset[idx] + i * CStructEntity.size(ty[0])) + entity_class.new(@addr + @offset[idx] + i * (ty[2] || CStructEntity).size(ty[0])) end) else entity_class.new(@addr + @offset[idx]) @@ -212,11 +223,11 @@ def set_ctypes(types) @offset = [] offset = 0 - max_align = types.map { |type, count = 1| + max_align = types.map { |type, count = 1, klass = CStructEntity| orig_offset = offset if type.kind_of?(Array) # type is a nested array representing a nested struct - align = CStructEntity.alignment(type) - total_size = CStructEntity.size(type) + align = klass.alignment(type) + total_size = klass.size(type) offset = PackInfo.align(orig_offset, align) @offset << offset offset += (total_size * (count || 1)) @@ -233,6 +244,11 @@ def set_ctypes(types) @size = PackInfo.align(offset, max_align) end + def offset_of(mbr) + idx = @members.index(mbr.to_s) || raise(ArgumentError, "no such member: #{mbr}") + @offset[idx] + end + # Fetch struct member +name+ if only one argument is specified. If two # arguments are specified, the first is an offset and the second is a # length and this method returns the string of +length+ bytes beginning at @@ -268,10 +284,11 @@ def [](*args) when Array case ty[0] when TYPE_VOIDP - val = val.collect{|v| Pointer.new(v)} + val = val.collect{|v| v = Pointer.new(v); v.size = SIZEOF_VOIDP; v } end when TYPE_VOIDP val = Pointer.new(val[0]) + val.size = SIZEOF_VOIDP else val = val[0] end @@ -335,6 +352,11 @@ def to_s() # :nodoc: class CUnionEntity < CStructEntity include PackInfo + def CUnionEntity.compute_offset(types, members, mbr) + # all members begin at offset 0 + 0 + end + # Allocates a C union the +types+ provided. # # When the instance is garbage collected, the C function +func+ is called. @@ -351,9 +373,9 @@ def CUnionEntity.malloc(types, func=nil) # Fiddle::TYPE_CHAR, # Fiddle::TYPE_VOIDP ]) #=> 8 def CUnionEntity.size(types) - types.map { |type, count = 1| + types.map { |type, count = 1, klass = CStructEntity| if type.kind_of?(Array) # type is a nested array representing a nested struct - CStructEntity.size(type) * (count || 1) + klass.size(type) * (count || 1) else PackInfo::SIZE_MAP[type] * count end diff --git a/test/fiddle/test_import.rb b/test/fiddle/test_import.rb index 00b771d0..468a7e5f 100644 --- a/test/fiddle/test_import.rb +++ b/test/fiddle/test_import.rb @@ -209,6 +209,35 @@ def test_union_nested_struct_members() refute_equal( 0, s.mouse.x) end + def test_size_of_struct_accessor_returning_void_pointer() + s = Fiddle::Importer.struct(['void *x', 'void *y[2]']).malloc + assert_equal Fiddle::SIZEOF_VOIDP, s['x'].size + assert_equal Fiddle::SIZEOF_VOIDP, s['y'][0].size + end + + def test_struct_size_and_offset_of_nested_unions() + a = Fiddle::Importer.union ['float f[4]', 'int i[2]'] + b = Fiddle::Importer.struct ['float x', 'int y'] + c = Fiddle::Importer.struct [ a: a, b: b ] + + assert_equal 0, a.offset_of(:f) + assert_equal 0, a.offset_of(:i) + assert_equal 0, b.offset_of(:x) + assert_equal Fiddle::SIZEOF_FLOAT, b.offset_of(:y) + assert_equal 0, c.offset_of(:a) + assert_equal 4 * Fiddle::SIZEOF_FLOAT, c.offset_of(:b) + + assert_equal a.offset_of(:f), a.malloc.offset_of(:f) + assert_equal a.offset_of(:i), a.malloc.offset_of(:i) + assert_equal b.offset_of(:x), b.malloc.offset_of(:x) + assert_equal b.offset_of(:y), b.malloc.offset_of(:y) + assert_equal c.offset_of(:a), c.malloc.offset_of(:a) + assert_equal c.offset_of(:b), c.malloc.offset_of(:b) + + assert_equal Fiddle::SIZEOF_FLOAT * 4 + Fiddle::SIZEOF_INT * 2, + c.size + end + def test_struct_array_assignment() instance = Fiddle::Importer.struct(["unsigned int stages[1]"]).malloc instance.stages[0] = 1024