From f94136cd2844c6f13b1e37cc0578f13345da0462 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 6 Feb 2020 18:50:00 +0300 Subject: [PATCH 1/3] IGNITE-14057 Support big-endian systems Fix primitives decoding on big-endian architectures --- pyignite/api/binary.py | 2 +- pyignite/datatypes/internal.py | 4 ++-- pyignite/datatypes/primitive.py | 20 ++++++++++++++++---- pyignite/datatypes/primitive_objects.py | 7 ++++++- pyignite/queries/response.py | 2 +- pytest.ini | 2 ++ tests/test_binary.py | 2 +- 7 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 pytest.ini diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py index 1d63b49..722001a 100644 --- a/pyignite/api/binary.py +++ b/pyignite/api/binary.py @@ -86,7 +86,7 @@ def get_binary_type( if result.status != 0: return result result.value = { - 'type_exists': response.type_exists + 'type_exists': Bool.to_python(response.type_exists) } if hasattr(response, 'body'): result.value.update(body_struct.to_python(response.body)) diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py index 9fd5d64..9989015 100644 --- a/pyignite/datatypes/internal.py +++ b/pyignite/datatypes/internal.py @@ -204,7 +204,7 @@ class Struct: def parse( self, client: 'Client' - ) -> Tuple[ctypes.BigEndianStructure, bytes]: + ) -> Tuple[ctypes.LittleEndianStructure, bytes]: buffer = b'' fields = [] values = {} @@ -219,7 +219,7 @@ def parse( values[name] = buffer_fragment data_class = type( - 'Struct', + 'StructLE', (ctypes.LittleEndianStructure,), { '_pack_': 1, diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py index 23d070d..66b05d8 100644 --- a/pyignite/datatypes/primitive.py +++ b/pyignite/datatypes/primitive.py @@ -14,6 +14,7 @@ # limitations under the License. import ctypes +import sys from pyignite.constants import * from .base import IgniteDataType @@ -48,13 +49,20 @@ class Primitive(IgniteDataType): def parse(cls, client: 'Client'): return cls.c_type, client.recv(ctypes.sizeof(cls.c_type)) - @staticmethod - def to_python(ctype_object, *args, **kwargs): + @classmethod + def to_python(cls, ctype_object, *args, **kwargs): return ctype_object @classmethod def from_python(cls, value): - return bytes(cls.c_type(value)) + return Primitive.fix_endianness(bytes(cls.c_type(value))) + + @staticmethod + def fix_endianness(buf): + if len(buf) > 1 and sys.byteorder != PROTOCOL_BYTE_ORDER: + buf = buf[::-1] + + return buf class Byte(Primitive): @@ -122,4 +130,8 @@ def from_python(cls, value): class Bool(Primitive): _type_name = NAME_BOOLEAN _type_id = TYPE_BOOLEAN - c_type = ctypes.c_bool + c_type = ctypes.c_byte # Use c_byte because c_bool throws endianness conversion error on BE systems. + + @classmethod + def to_python(cls, ctype_object, *args, **kwargs): + return ctype_object != 0 diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py index 0bd0ec6..033ac9e 100644 --- a/pyignite/datatypes/primitive_objects.py +++ b/pyignite/datatypes/primitive_objects.py @@ -207,7 +207,7 @@ def from_python(cls, value): class BoolObject(DataObject): _type_name = NAME_BOOLEAN _type_id = TYPE_BOOLEAN - c_type = ctypes.c_bool + c_type = ctypes.c_byte # Use c_byte because c_bool throws endianness conversion error on BE systems. type_code = TC_BOOL pythonic = bool default = False @@ -215,3 +215,8 @@ class BoolObject(DataObject): @staticmethod def hashcode(value: bool, *args, **kwargs) -> int: return 1231 if value else 1237 + + @classmethod + def to_python(cls, ctype_object, *args, **kwargs): + return ctype_object.value != 0 + diff --git a/pyignite/queries/response.py b/pyignite/queries/response.py index 6003959..05a519a 100644 --- a/pyignite/queries/response.py +++ b/pyignite/queries/response.py @@ -179,7 +179,7 @@ def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): ) fields += body_class._fields_ + [ ('data', data_class), - ('more', ctypes.c_bool), + ('more', ctypes.c_byte), ] def _create_parse_result(self, conn: Connection, header_class, fields: list, buffer: bytearray): diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..ac6441b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +log_cli = True \ No newline at end of file diff --git a/tests/test_binary.py b/tests/test_binary.py index 1c051f0..5190a6a 100644 --- a/tests/test_binary.py +++ b/tests/test_binary.py @@ -225,7 +225,7 @@ class OuterType( def test_add_schema_to_binary_object(client): - migrate_cache = client.create_cache('migrate_binary') + migrate_cache = client.get_or_create_cache('migrate_binary') class MyBinaryType( metaclass=GenericObjectMeta, From 45eb796721f482e7b9fd52955b35e3de9936057b Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Wed, 27 Jan 2021 19:33:41 +0300 Subject: [PATCH 2/3] IGNITE-14057: Optimize primitive packing --- pyignite/datatypes/primitive.py | 40 ++++++++++++++++++++++++--------- pytest.ini | 2 -- 2 files changed, 29 insertions(+), 13 deletions(-) delete mode 100644 pytest.ini diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py index 66b05d8..d549fda 100644 --- a/pyignite/datatypes/primitive.py +++ b/pyignite/datatypes/primitive.py @@ -14,6 +14,7 @@ # limitations under the License. import ctypes +import struct import sys from pyignite.constants import * @@ -53,53 +54,66 @@ def parse(cls, client: 'Client'): def to_python(cls, ctype_object, *args, **kwargs): return ctype_object - @classmethod - def from_python(cls, value): - return Primitive.fix_endianness(bytes(cls.c_type(value))) - - @staticmethod - def fix_endianness(buf): - if len(buf) > 1 and sys.byteorder != PROTOCOL_BYTE_ORDER: - buf = buf[::-1] - - return buf - class Byte(Primitive): _type_name = NAME_BYTE _type_id = TYPE_BYTE c_type = ctypes.c_byte + @classmethod + def from_python(cls, value): + return struct.pack(" Date: Wed, 27 Jan 2021 19:42:30 +0300 Subject: [PATCH 3/3] IGNITE-14057: Fix review finding --- pyignite/datatypes/internal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py index 9989015..9f23ec6 100644 --- a/pyignite/datatypes/internal.py +++ b/pyignite/datatypes/internal.py @@ -219,7 +219,7 @@ def parse( values[name] = buffer_fragment data_class = type( - 'StructLE', + 'Struct', (ctypes.LittleEndianStructure,), { '_pack_': 1,