From 8a243db2f40779398bc5133d102025788b029916 Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Fri, 29 Jan 2021 21:16:45 +0300 Subject: [PATCH 1/8] IGNITE-13967 Refactor parsing. --- pyignite/api/binary.py | 42 +++---- pyignite/connection/__init__.py | 150 +++++++++--------------- pyignite/datatypes/__init__.py | 2 +- pyignite/datatypes/cache_properties.py | 9 +- pyignite/datatypes/complex.py | 69 ++++++----- pyignite/datatypes/internal.py | 32 +++-- pyignite/datatypes/null_object.py | 4 +- pyignite/datatypes/primitive.py | 5 +- pyignite/datatypes/primitive_arrays.py | 19 ++- pyignite/datatypes/primitive_objects.py | 12 +- pyignite/datatypes/standard.py | 52 ++++---- pyignite/queries/query.py | 4 +- pyignite/queries/response.py | 26 ++-- pyignite/stream/__init__.py | 16 +++ pyignite/stream/binary_stream.py | 57 +++++++++ pyignite/utils.py | 8 +- tests/test_affinity_request_routing.py | 3 +- 17 files changed, 263 insertions(+), 247 deletions(-) create mode 100644 pyignite/stream/__init__.py create mode 100644 pyignite/stream/binary_stream.py diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py index 722001a..43ef6b2 100644 --- a/pyignite/api/binary.py +++ b/pyignite/api/binary.py @@ -24,16 +24,15 @@ from pyignite.queries.op_codes import * from pyignite.utils import int_overflow, entity_id from .result import APIResult +from ..stream import BinaryStream from ..queries.response import Response -def get_binary_type( - connection: 'Connection', binary_type: Union[str, int], query_id=None, -) -> APIResult: +def get_binary_type(conn: 'Connection', binary_type: Union[str, int], query_id=None) -> APIResult: """ Gets the binary type information by type ID. - :param connection: connection to Ignite server, + :param conn: connection to Ignite server, :param binary_type: binary type name or ID, :param query_id: (optional) a value generated by client and returned as-is in response.query_id. When the parameter is omitted, a random value @@ -52,26 +51,27 @@ def get_binary_type( _, send_buffer = query_struct.from_python({ 'type_id': entity_id(binary_type), }) - connection.send(send_buffer) + conn.send(send_buffer) - response_head_struct = Response(protocol_version=connection.get_protocol_version(), + response_head_struct = Response(protocol_version=conn.get_protocol_version(), following=[('type_exists', Bool)]) - response_head_type, recv_buffer = response_head_struct.parse(connection) - response_head = response_head_type.from_buffer_copy(recv_buffer) - response_parts = [] - if response_head.type_exists: - resp_body_type, resp_body_buffer = body_struct.parse(connection) - response_parts.append(('body', resp_body_type)) - resp_body = resp_body_type.from_buffer_copy(resp_body_buffer) - recv_buffer += resp_body_buffer - if resp_body.is_enum: - resp_enum, resp_enum_buffer = enum_struct.parse(connection) - response_parts.append(('enums', resp_enum)) - recv_buffer += resp_enum_buffer - resp_schema_type, resp_schema_buffer = schema_struct.parse(connection) - response_parts.append(('schema', resp_schema_type)) - recv_buffer += resp_schema_buffer + with BinaryStream(conn.recv(), conn) as stream: + response_head_type, recv_buffer = response_head_struct.parse(stream) + response_head = response_head_type.from_buffer_copy(recv_buffer) + response_parts = [] + if response_head.type_exists: + resp_body_type, resp_body_buffer = body_struct.parse(stream) + response_parts.append(('body', resp_body_type)) + resp_body = resp_body_type.from_buffer_copy(resp_body_buffer) + recv_buffer += resp_body_buffer + if resp_body.is_enum: + resp_enum, resp_enum_buffer = enum_struct.parse(stream) + response_parts.append(('enums', resp_enum)) + recv_buffer += resp_enum_buffer + resp_schema_type, resp_schema_buffer = schema_struct.parse(stream) + response_parts.append(('schema', resp_schema_type)) + recv_buffer += resp_schema_buffer response_class = type( 'GetBinaryTypeResponse', diff --git a/pyignite/connection/__init__.py b/pyignite/connection/__init__.py index cf40718..a5c9ec7 100644 --- a/pyignite/connection/__init__.py +++ b/pyignite/connection/__init__.py @@ -52,6 +52,8 @@ __all__ = ['Connection'] +from ..stream import BinaryStream + class Connection: """ @@ -60,8 +62,7 @@ class Connection: * socket wrapper. Detects fragmentation and network errors. See also https://docs.python.org/3/howto/sockets.html, - * binary protocol connector. Incapsulates handshake, data read-ahead and - failover reconnection. + * binary protocol connector. Incapsulates handshake and failover reconnection. """ _socket = None @@ -72,7 +73,6 @@ class Connection: host = None port = None timeout = None - prefetch = None username = None password = None ssl_params = {} @@ -97,7 +97,7 @@ def _check_ssl_params(params): ).format(param)) def __init__( - self, client: 'Client', prefetch: bytes = b'', timeout: int = None, + self, client: 'Client', timeout: int = None, username: str = None, password: str = None, **ssl_params ): """ @@ -107,8 +107,6 @@ def __init__( https://docs.python.org/3/library/ssl.html#ssl-certificates. :param client: Ignite client object, - :param prefetch: (optional) initialize the read-ahead data buffer. - Empty by default, :param timeout: (optional) sets timeout (in seconds) for each socket operation including `connect`. 0 means non-blocking mode, which is virtually guaranteed to fail. Can accept integer or float value. @@ -143,7 +141,6 @@ def __init__( :param password: (optional) password to authenticate to Ignite cluster. """ self.client = client - self.prefetch = prefetch self.timeout = timeout self.username = username self.password = password @@ -202,26 +199,27 @@ def read_response(self) -> Union[dict, OrderedDict]: ('length', Int), ('op_code', Byte), ]) - start_class, start_buffer = response_start.parse(self) - start = start_class.from_buffer_copy(start_buffer) - data = response_start.to_python(start) - response_end = None - if data['op_code'] == 0: - response_end = Struct([ - ('version_major', Short), - ('version_minor', Short), - ('version_patch', Short), - ('message', String), - ]) - elif self.get_protocol_version() >= (1, 4, 0): - response_end = Struct([ - ('node_uuid', UUIDObject), - ]) - if response_end: - end_class, end_buffer = response_end.parse(self) - end = end_class.from_buffer_copy(end_buffer) - data.update(response_end.to_python(end)) - return data + with BinaryStream(self.recv(), self) as response_stream: + start_class, start_buffer = response_start.parse(response_stream) + start = start_class.from_buffer_copy(start_buffer) + data = response_start.to_python(start) + response_end = None + if data['op_code'] == 0: + response_end = Struct([ + ('version_major', Short), + ('version_minor', Short), + ('version_patch', Short), + ('message', String), + ]) + elif self.get_protocol_version() >= (1, 4, 0): + response_end = Struct([ + ('node_uuid', UUIDObject), + ]) + if response_end: + end_class, end_buffer = response_end.parse(response_stream) + end = end_class.from_buffer_copy(end_buffer) + data.update(response_end.to_python(end)) + return data def connect( self, host: str = None, port: int = None @@ -289,7 +287,7 @@ def _connect_version( self.username, self.password ) - self.send(hs_request) + self.send(bytes(hs_request)) hs_response = self.read_response() if hs_response['op_code'] == 0: # disconnect but keep in use @@ -370,19 +368,6 @@ def _transfer_params(self, to: 'Connection'): to.host = self.host to.port = self.port - def clone(self, prefetch: bytes = b'') -> 'Connection': - """ - Clones this connection in its current state. - - :return: `Connection` object. - """ - clone = self.__class__(self.client, **self.ssl_params) - self._transfer_params(to=clone) - if self.alive: - clone.connect(self.host, self.port) - clone.prefetch = prefetch - return clone - def send(self, data: bytes, flags=None): """ Send data down the socket. @@ -396,70 +381,45 @@ def send(self, data: bytes, flags=None): kwargs = {} if flags is not None: kwargs['flags'] = flags - data = bytes(data) - total_bytes_sent = 0 - while total_bytes_sent < len(data): - try: - bytes_sent = self.socket.send( - data[total_bytes_sent:], - **kwargs - ) - except connection_errors: - self._fail() - self.reconnect() - raise - if bytes_sent == 0: - self._fail() - self.reconnect() - raise SocketError('Connection broken.') - total_bytes_sent += bytes_sent - - def recv(self, buffersize, flags=None) -> bytes: - """ - Receive data from socket or read-ahead buffer. + try: + self.socket.sendall(data, **kwargs) + except Exception: + self._fail() + self.reconnect() + raise + + def recv(self, flags=None) -> bytearray: + def _recv(buffer, num_bytes): + bytes_to_receive = num_bytes + while bytes_to_receive > 0: + try: + bytes_rcvd = self.socket.recv_into(buffer, bytes_to_receive, **kwargs) + if bytes_rcvd == 0: + raise SocketError('Connection broken.') + except connection_errors: + self._fail() + self.reconnect() + raise + + buffer = buffer[bytes_rcvd:] + bytes_to_receive -= bytes_rcvd - :param buffersize: bytes to receive, - :param flags: (optional) OS-specific flags, - :return: data received. - """ if self.closed: raise SocketError('Attempt to use closed connection.') - pref_size = len(self.prefetch) - if buffersize > pref_size: - result = self.prefetch - self.prefetch = b'' - try: - result += self._recv(buffersize-pref_size, flags) - except connection_errors: - self._fail() - self.reconnect() - raise - return result - else: - result = self.prefetch[:buffersize] - self.prefetch = self.prefetch[buffersize:] - return result - - def _recv(self, buffersize, flags=None) -> bytes: - """ - Handle socket data reading. - """ kwargs = {} if flags is not None: kwargs['flags'] = flags - chunks = [] - bytes_rcvd = 0 - while bytes_rcvd < buffersize: - chunk = self.socket.recv(buffersize-bytes_rcvd, **kwargs) - if chunk == b'': - raise SocketError('Connection broken.') - chunks.append(chunk) - bytes_rcvd += len(chunk) + data = bytearray(4) + _recv(memoryview(data), 4) + response_len = int.from_bytes(data, PROTOCOL_BYTE_ORDER) + + data.extend(bytearray(response_len)) + _recv(memoryview(data)[4:], response_len) + return data - return b''.join(chunks) def close(self, release=True): """ diff --git a/pyignite/datatypes/__init__.py b/pyignite/datatypes/__init__.py index 5024f79..87f8c6b 100644 --- a/pyignite/datatypes/__init__.py +++ b/pyignite/datatypes/__init__.py @@ -24,4 +24,4 @@ from .primitive import * from .primitive_arrays import * from .primitive_objects import * -from .standard import * +from .standard import * \ No newline at end of file diff --git a/pyignite/datatypes/cache_properties.py b/pyignite/datatypes/cache_properties.py index e94db5f..0d34200 100644 --- a/pyignite/datatypes/cache_properties.py +++ b/pyignite/datatypes/cache_properties.py @@ -92,10 +92,11 @@ def build_header(cls): ) @classmethod - def parse(cls, connection: 'Connection'): + def parse(cls, stream): header_class = cls.build_header() - header_buffer = connection.recv(ctypes.sizeof(header_class)) - data_class, data_buffer = cls.prop_data_class.parse(connection) + buf = stream.read(ctypes.sizeof(header_class)) + data_class, data_buf = cls.prop_data_class.parse(stream) + buf += data_buf prop_class = type( cls.__name__, (header_class,), @@ -106,7 +107,7 @@ def parse(cls, connection: 'Connection'): ], } ) - return prop_class, header_buffer + data_buffer + return prop_class, buf @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py index 6860583..0311cb3 100644 --- a/pyignite/datatypes/complex.py +++ b/pyignite/datatypes/complex.py @@ -20,7 +20,6 @@ from pyignite.constants import * from pyignite.exceptions import ParseError - from .base import IgniteDataType from .internal import AnyDataObject, infer_from_python from .type_codes import * @@ -69,19 +68,19 @@ def build_header(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) fields = [] for i in range(header.length): - c_type, buffer_fragment = AnyDataObject.parse(client) + c_type, buffer_fragment = AnyDataObject.parse(stream) buffer += buffer_fragment fields.append(('element_{}'.format(i), c_type)) @@ -162,14 +161,14 @@ def build_header(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) final_class = type( @@ -183,7 +182,7 @@ def parse(cls, client: 'Client'): ], } ) - buffer += client.recv( + buffer += stream.read( ctypes.sizeof(final_class) - ctypes.sizeof(header_class) ) return final_class, buffer @@ -260,19 +259,19 @@ def build_header(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) fields = [] for i in range(header.length): - c_type, buffer_fragment = AnyDataObject.parse(client) + c_type, buffer_fragment = AnyDataObject.parse(stream) buffer += buffer_fragment fields.append(('element_{}'.format(i), c_type)) @@ -358,19 +357,19 @@ def build_header(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) fields = [] for i in range(header.length << 1): - c_type, buffer_fragment = AnyDataObject.parse(client) + c_type, buffer_fragment = AnyDataObject.parse(stream) buffer += buffer_fragment fields.append(('element_{}'.format(i), c_type)) @@ -506,7 +505,7 @@ def find_client(): @staticmethod def hashcode( - value: object, client: 'Client' = None, *args, **kwargs + value: object, client, *args, **kwargs ) -> int: # binary objects's hashcode implementation is special in the sense # that you need to fully serialize the object to calculate @@ -577,29 +576,29 @@ def get_dataclass(conn: 'Connection', header) -> OrderedDict: return result @classmethod - def parse(cls, client: 'Client'): + def parse(cls, stream): from pyignite.datatypes import Struct - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + buffer = stream.recv(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) # ignore full schema, always retrieve fields' types and order # from complex types registry - data_class = cls.get_dataclass(client, header) + data_class = stream.get_dataclass(header) fields = data_class.schema.items() object_fields_struct = Struct(fields) - object_fields, object_fields_buffer = object_fields_struct.parse(client) + object_fields, object_fields_buffer = object_fields_struct.parse(stream) buffer += object_fields_buffer final_class_fields = [('object_fields', object_fields)] if header.flags & cls.HAS_SCHEMA: schema = cls.schema_type(header.flags) * len(fields) - buffer += client.recv(ctypes.sizeof(schema)) + buffer += stream.read(ctypes.sizeof(schema)) final_class_fields.append(('schema', schema)) final_class = type( @@ -611,7 +610,7 @@ def parse(cls, client: 'Client'): } ) # register schema encoding approach - client.compact_footer = bool(header.flags & cls.COMPACT_FOOTER) + stream.compact_footer = bool(header.flags & cls.COMPACT_FOOTER) return final_class, buffer @classmethod diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py index 23b9cc4..e474693 100644 --- a/pyignite/datatypes/internal.py +++ b/pyignite/datatypes/internal.py @@ -119,8 +119,8 @@ def __init__(self, predicate1: Callable[[any], bool], predicate2: Callable[[any] self.var1 = var1 self.var2 = var2 - def parse(self, client: 'Client', context): - return self.var1.parse(client) if self.predicate1(context) else self.var2.parse(client) + def parse(self, stream, context): + return self.var1.parse(stream) if self.predicate1(context) else self.var2.parse(stream) def to_python(self, ctype_object, context, *args, **kwargs): return self.var1.to_python(ctype_object, *args, **kwargs) if self.predicate2(context) else self.var2.to_python(ctype_object, *args, **kwargs) @@ -144,13 +144,13 @@ def build_header_class(self): }, ) - def parse(self, client: 'Client'): - buffer = client.recv(ctypes.sizeof(self.counter_type)) + def parse(self, stream): + buffer = stream.read(ctypes.sizeof(self.counter_type)) length = int.from_bytes(buffer, byteorder=PROTOCOL_BYTE_ORDER) fields = [] for i in range(length): - c_type, buffer_fragment = Struct(self.following).parse(client) + c_type, buffer_fragment = Struct(self.following).parse(stream) buffer += buffer_fragment fields.append(('element_{}'.format(i), c_type)) @@ -202,16 +202,14 @@ class Struct: dict_type = attr.ib(default=OrderedDict) defaults = attr.ib(type=dict, default={}) - def parse( - self, client: 'Client' - ) -> Tuple[ctypes.LittleEndianStructure, bytes]: + def parse(self, stream) -> Tuple[ctypes.LittleEndianStructure, bytes]: buffer = b'' fields = [] values = {} for name, c_type in self.fields: is_cond = isinstance(c_type, Conditional) - c_type, buffer_fragment = c_type.parse(client, values) if is_cond else c_type.parse(client) + c_type, buffer_fragment = c_type.parse(stream, values) if is_cond else c_type.parse(stream) buffer += buffer_fragment fields.append((name, c_type)) @@ -299,14 +297,14 @@ def get_subtype(iterable, allow_none=False): return type_first @classmethod - def parse(cls, client: 'Client'): - type_code = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + type_code = bytes(stream.read(ctypes.sizeof(ctypes.c_byte))) try: - data_class = tc_map(type_code) + data_class = tc_map(bytes(type_code)) except KeyError: raise ParseError('Unknown type code: `{}`'.format(type_code)) - client.prefetch += type_code - return data_class.parse(client) + stream.seek(stream.tell() - ctypes.sizeof(ctypes.c_byte)) + return data_class.parse(stream) @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -455,14 +453,14 @@ def build_header(self): } ) - def parse(self, client: 'Client'): + def parse(self, stream): header_class = self.build_header() - buffer = client.recv(ctypes.sizeof(header_class)) + buffer = stream.read(ctypes.sizeof(header_class)) header = header_class.from_buffer_copy(buffer) fields = [] for i in range(header.length): - c_type, buffer_fragment = super().parse(client) + c_type, buffer_fragment = super().parse(stream) buffer += buffer_fragment fields.append(('element_{}'.format(i), c_type)) diff --git a/pyignite/datatypes/null_object.py b/pyignite/datatypes/null_object.py index 19b41c7..4c5382a 100644 --- a/pyignite/datatypes/null_object.py +++ b/pyignite/datatypes/null_object.py @@ -55,8 +55,8 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def parse(cls, client: 'Client'): - buffer = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) data_type = cls.build_c_type() return data_type, buffer diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py index d549fda..59e41ff 100644 --- a/pyignite/datatypes/primitive.py +++ b/pyignite/datatypes/primitive.py @@ -15,7 +15,6 @@ import ctypes import struct -import sys from pyignite.constants import * from .base import IgniteDataType @@ -47,8 +46,8 @@ class Primitive(IgniteDataType): c_type = None @classmethod - def parse(cls, client: 'Client'): - return cls.c_type, client.recv(ctypes.sizeof(cls.c_type)) + def parse(cls, stream): + return cls.c_type, stream.read(ctypes.sizeof(cls.c_type)) @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py index 1b41728..ac42949 100644 --- a/pyignite/datatypes/primitive_arrays.py +++ b/pyignite/datatypes/primitive_arrays.py @@ -61,14 +61,14 @@ def build_header_class(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header_class() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) final_class = type( cls.__name__, @@ -80,20 +80,15 @@ def parse(cls, client: 'Client'): ], } ) - buffer += client.recv( - ctypes.sizeof(final_class) - ctypes.sizeof(header_class) - ) + buffer += stream.read(ctypes.sizeof(final_class) - ctypes.sizeof(header_class)) return final_class, buffer @classmethod def to_python(cls, ctype_object, *args, **kwargs): - result = [] length = getattr(ctype_object, "length", None) if length is None: return None - for i in range(length): - result.append(ctype_object.data[i]) - return result + return [ctype_object.data[i] for i in range(ctype_object.length)] @classmethod def from_python(cls, value): diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py index 53f12d2..621c815 100644 --- a/pyignite/datatypes/primitive_objects.py +++ b/pyignite/datatypes/primitive_objects.py @@ -17,7 +17,6 @@ from pyignite.constants import * from pyignite.utils import unsigned - from .base import IgniteDataType from .type_codes import * from .type_ids import * @@ -61,12 +60,13 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + if buffer == TC_NULL: + return Null.build_c_type(), buffer + data_type = cls.build_c_type() - buffer = tc_type + client.recv(ctypes.sizeof(data_type) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(data_type) - len(buffer)) return data_type, buffer @staticmethod diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py index 0f16735..56051ba 100644 --- a/pyignite/datatypes/standard.py +++ b/pyignite/datatypes/standard.py @@ -54,14 +54,14 @@ def build_c_type(cls): raise NotImplementedError('This object is generic') @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer c_type = cls.build_c_type() - buffer = tc_type + client.recv(ctypes.sizeof(c_type) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(c_type) - len(buffer)) return c_type, buffer @@ -95,17 +95,17 @@ def build_c_type(cls, length: int): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) # String or Null - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer - buffer = tc_type + client.recv(ctypes.sizeof(ctypes.c_int)) + buffer += stream.read(ctypes.sizeof(ctypes.c_int)) length = int.from_bytes(buffer[1:], byteorder=PROTOCOL_BYTE_ORDER) data_type = cls.build_c_type(length) - buffer += client.recv(ctypes.sizeof(data_type) - len(buffer)) + buffer += stream.read(ctypes.sizeof(data_type) - len(buffer)) return data_type, buffer @@ -165,17 +165,14 @@ def build_c_header(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) # Decimal or Null - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_c_header() - buffer = tc_type + client.recv( - ctypes.sizeof(header_class) - - len(tc_type) - ) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) data_type = type( cls.__name__, @@ -187,10 +184,7 @@ def parse(cls, client: 'Client'): ], } ) - buffer += client.recv( - ctypes.sizeof(data_type) - - ctypes.sizeof(header_class) - ) + buffer += stream.read(ctypes.sizeof(data_type) - ctypes.sizeof(header_class)) return data_type, buffer @classmethod @@ -599,18 +593,18 @@ def build_header_class(cls): ) @classmethod - def parse(cls, client: 'Client'): - tc_type = client.recv(ctypes.sizeof(ctypes.c_byte)) + def parse(cls, stream): + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if tc_type == TC_NULL: - return Null.build_c_type(), tc_type + if buffer == TC_NULL: + return Null.build_c_type(), buffer header_class = cls.build_header_class() - buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type)) + buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) header = header_class.from_buffer_copy(buffer) fields = [] for i in range(header.length): - c_type, buffer_fragment = cls.standard_type.parse(client) + c_type, buffer_fragment = cls.standard_type.parse(stream) buffer += buffer_fragment fields.append(('element_{}'.format(i), c_type)) diff --git a/pyignite/queries/query.py b/pyignite/queries/query.py index 69b6fa2..736eb1a 100644 --- a/pyignite/queries/query.py +++ b/pyignite/queries/query.py @@ -21,6 +21,7 @@ from pyignite.connection import Connection from pyignite.constants import MIN_LONG, MAX_LONG, RHF_TOPOLOGY_CHANGED from pyignite.queries.response import Response, SQLResponse +from pyignite.stream import BinaryStream @attr.s @@ -99,7 +100,8 @@ def perform( response_struct = Response(protocol_version=conn.get_protocol_version(), following=response_config) - response_ctype, recv_buffer = response_struct.parse(conn) + with BinaryStream(conn.recv(), conn) as stream: + response_ctype, recv_buffer = response_struct.parse(stream) response = response_ctype.from_buffer_copy(recv_buffer) # this test depends on protocol version diff --git a/pyignite/queries/response.py b/pyignite/queries/response.py index 05a519a..4b86841 100644 --- a/pyignite/queries/response.py +++ b/pyignite/queries/response.py @@ -55,9 +55,9 @@ def build_header(self): ) return self._response_header - def parse(self, conn: Connection): + def parse(self, stream): header_class = self.build_header() - buffer = bytearray(conn.recv(ctypes.sizeof(header_class))) + buffer = stream.read(ctypes.sizeof(header_class)) header = header_class.from_buffer_copy(buffer) fields = [] @@ -76,18 +76,16 @@ def parse(self, conn: Connection): has_error = header.status_code != OP_SUCCESS if fields: - buffer += conn.recv( - sum([ctypes.sizeof(c_type) for _, c_type in fields]) - ) + buffer += stream.read(sum([ctypes.sizeof(c_type) for _, c_type in fields])) if has_error: - msg_type, buffer_fragment = String.parse(conn) + msg_type, buffer_fragment = String.parse(stream) buffer += buffer_fragment fields.append(('error_message', msg_type)) else: - self._parse_success(conn, buffer, fields) + self._parse_success(stream, buffer, fields) - return self._create_parse_result(conn, header_class, fields, buffer) + return self._create_parse_result(stream, header_class, fields, buffer) def _create_parse_result(self, conn: Connection, header_class, fields: list, buffer: bytearray): response_class = type( @@ -98,11 +96,11 @@ def _create_parse_result(self, conn: Connection, header_class, fields: list, buf '_fields_': fields, } ) - return response_class, bytes(buffer) + return response_class, buffer - def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): + def _parse_success(self, stream, buffer: bytearray, fields: list): for name, ignite_type in self.following: - c_type, buffer_fragment = ignite_type.parse(conn) + c_type, buffer_fragment = ignite_type.parse(stream) buffer += buffer_fragment fields.append((name, c_type)) @@ -182,7 +180,7 @@ def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): ('more', ctypes.c_byte), ] - def _create_parse_result(self, conn: Connection, header_class, fields: list, buffer: bytearray): + def _create_parse_result(self, stream, header_class, fields: list, buffer: bytearray): final_class = type( 'SQLResponse', (header_class,), @@ -191,8 +189,8 @@ def _create_parse_result(self, conn: Connection, header_class, fields: list, buf '_fields_': fields, } ) - buffer += conn.recv(ctypes.sizeof(final_class) - len(buffer)) - return final_class, bytes(buffer) + buffer += stream.read(ctypes.sizeof(final_class) - len(buffer)) + return final_class, buffer def to_python(self, ctype_object, *args, **kwargs): if getattr(ctype_object, 'status_code', 0) == 0: diff --git a/pyignite/stream/__init__.py b/pyignite/stream/__init__.py new file mode 100644 index 0000000..f5b931b --- /dev/null +++ b/pyignite/stream/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .binary_stream import BinaryStream \ No newline at end of file diff --git a/pyignite/stream/binary_stream.py b/pyignite/stream/binary_stream.py new file mode 100644 index 0000000..81eb087 --- /dev/null +++ b/pyignite/stream/binary_stream.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from io import BytesIO + +class BinaryStream: + def __init__(self, stream, conn): + self.stream = BytesIO(stream) + self.conn = conn + + @property + def compact_footer(self) -> bool: + return self.conn.client.compact_footer + + @compact_footer.setter + def compact_footer(self, value: bool): + self.conn.client.compact_footer = value + + def read(self, size): + buf = bytearray(size) + self.stream.readinto(buf) + return buf + + def tell(self): + return self.stream.tell() + + def seek(self, *args, **kwargs): + return self.stream.seek(*args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.stream.close() + + def get_dataclass(self, header): + # get field names from outer space + result = self.conn.client.query_binary_type( + header.type_id, + header.schema_id + ) + if not result: + raise RuntimeError('Binary type is not registered') + return result + diff --git a/pyignite/utils.py b/pyignite/utils.py index ef7b6f6..4677e37 100644 --- a/pyignite/utils.py +++ b/pyignite/utils.py @@ -23,7 +23,7 @@ from pyignite.datatypes.base import IgniteDataType from .constants import * - +from .stream import BinaryStream LONG_MASK = 0xffffffff DIGITS_PER_INT = 9 @@ -96,14 +96,12 @@ def unwrap_binary(client: 'Client', wrapped: tuple) -> object: from pyignite.datatypes.complex import BinaryObject blob, offset = wrapped - conn_clone = client.random_node.clone(prefetch=blob) - conn_clone.pos = offset - data_class, data_bytes = BinaryObject.parse(conn_clone) + with BinaryStream(blob, client.random_node) as stream: + data_class, data_bytes = BinaryObject.parse(stream) result = BinaryObject.to_python( data_class.from_buffer_copy(data_bytes), client, ) - conn_clone.close() return result diff --git a/tests/test_affinity_request_routing.py b/tests/test_affinity_request_routing.py index eb46ab6..8b34b66 100644 --- a/tests/test_affinity_request_routing.py +++ b/tests/test_affinity_request_routing.py @@ -124,7 +124,6 @@ class AffinityTestType1( assert get_request_grid_idx("Put") == grid_idx -@pytest.mark.skip("https://issues.apache.org/jira/browse/IGNITE-13967") def test_cache_operation_routed_to_new_cluster_node(request, start_ignite_server, start_client): client = start_client(partition_aware=True) client.connect([("127.0.0.1", 10801), ("127.0.0.1", 10802), ("127.0.0.1", 10803), ("127.0.0.1", 10804)]) @@ -140,7 +139,7 @@ def test_cache_operation_routed_to_new_cluster_node(request, start_ignite_server def check_grid_idx(): cache.get(key) return get_request_grid_idx() == 4 - wait_for_condition(check_grid_idx) + wait_for_condition(check_grid_idx, error='failed to wait for rebalance') # Response is correct and comes from the new node res = cache.get_and_remove(key) From 79f5decc40b986b85e965183e1cc0ed58a374906 Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Tue, 2 Feb 2021 13:32:46 +0300 Subject: [PATCH 2/8] IGNITE-13967 Fix flaky affinity request routing test. --- .gitignore | 1 + pyignite/connection/__init__.py | 15 ++- pyignite/datatypes/complex.py | 2 +- requirements/tests.txt | 1 + tests/config/ignite-config-ssl.xml | 51 ----------- tests/config/ignite-config.xml | 39 -------- ...nfig-base.xml => ignite-config.xml.jinja2} | 41 +++++++-- tests/config/log4j.xml | 42 --------- tests/config/log4j.xml.jinja2 | 56 ++++++++++++ tests/conftest.py | 8 +- tests/test_affinity_request_routing.py | 91 +++++++++++++------ tests/test_affinity_single_connection.py | 4 - tests/util.py | 71 +++++---------- 13 files changed, 185 insertions(+), 237 deletions(-) delete mode 100644 tests/config/ignite-config-ssl.xml delete mode 100644 tests/config/ignite-config.xml rename tests/config/{ignite-config-base.xml => ignite-config.xml.jinja2} (63%) delete mode 100644 tests/config/log4j.xml create mode 100644 tests/config/log4j.xml.jinja2 diff --git a/.gitignore b/.gitignore index 7372921..5cc86d0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .eggs .pytest_cache .tox +tests/config/*.xml pyignite.egg-info ignite-log-* __pycache__ \ No newline at end of file diff --git a/pyignite/connection/__init__.py b/pyignite/connection/__init__.py index a5c9ec7..4e22de6 100644 --- a/pyignite/connection/__init__.py +++ b/pyignite/connection/__init__.py @@ -187,7 +187,9 @@ def get_protocol_version(self): def _fail(self): """ set client to failed state. """ self._failed = True - self._in_use.release() + + if self._in_use.locked(): + self._in_use.release() def read_response(self) -> Union[dict, OrderedDict]: """ @@ -345,10 +347,8 @@ def _reconnect(self): # return connection to initial state regardless of use lock self.close(release=False) - try: + if self._in_use.locked(): self._in_use.release() - except RuntimeError: - pass # connect and silence the connection errors try: @@ -427,11 +427,8 @@ def close(self, release=True): not required, since sockets are automatically closed when garbage-collected. """ - if release: - try: - self._in_use.release() - except RuntimeError: - pass + if release and self._in_use.locked(): + self._in_use.release() if self._socket: try: diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py index 0311cb3..4f5dc6a 100644 --- a/pyignite/datatypes/complex.py +++ b/pyignite/datatypes/complex.py @@ -578,7 +578,7 @@ def get_dataclass(conn: 'Connection', header) -> OrderedDict: @classmethod def parse(cls, stream): from pyignite.datatypes import Struct - buffer = stream.recv(ctypes.sizeof(ctypes.c_byte)) + buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) if buffer == TC_NULL: return Null.build_c_type(), buffer diff --git a/requirements/tests.txt b/requirements/tests.txt index 327f501..893928e 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -4,3 +4,4 @@ pytest==3.6.1 pytest-cov==2.5.1 teamcity-messages==1.21 psutil==5.6.5 +jinja2==2.11.3 diff --git a/tests/config/ignite-config-ssl.xml b/tests/config/ignite-config-ssl.xml deleted file mode 100644 index 827405c..0000000 --- a/tests/config/ignite-config-ssl.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/config/ignite-config.xml b/tests/config/ignite-config.xml deleted file mode 100644 index 09fba2c..0000000 --- a/tests/config/ignite-config.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/tests/config/ignite-config-base.xml b/tests/config/ignite-config.xml.jinja2 similarity index 63% rename from tests/config/ignite-config-base.xml rename to tests/config/ignite-config.xml.jinja2 index 7487618..4191f3f 100644 --- a/tests/config/ignite-config-base.xml +++ b/tests/config/ignite-config.xml.jinja2 @@ -26,12 +26,35 @@ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"> - - - - + + {% if use_ssl %} + + {% endif %} + + + + + + + {% if use_ssl %} + + + + + + + + + + + + + {% endif %} + + + + - @@ -42,7 +65,7 @@ - 127.0.0.1:48500..48503 + 127.0.0.1:48500..48510 @@ -69,9 +92,9 @@ - - - + + + diff --git a/tests/config/log4j.xml b/tests/config/log4j.xml deleted file mode 100644 index f5562d0..0000000 --- a/tests/config/log4j.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/config/log4j.xml.jinja2 b/tests/config/log4j.xml.jinja2 new file mode 100644 index 0000000..99ce765 --- /dev/null +++ b/tests/config/log4j.xml.jinja2 @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/conftest.py b/tests/conftest.py index 9974b16..54a7fda 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ from pyignite import Client from pyignite.constants import * from pyignite.api import cache_create, cache_destroy -from tests.util import _start_ignite, start_ignite_gen, get_request_grid_idx +from tests.util import _start_ignite, start_ignite_gen class BoolParser(argparse.Action): @@ -134,12 +134,6 @@ def cache(client): cache_destroy(conn, cache_name) -@pytest.fixture(autouse=True) -def log_init(): - # Init log call timestamp - get_request_grid_idx() - - @pytest.fixture(scope='module') def start_client(use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version,username, password): diff --git a/tests/test_affinity_request_routing.py b/tests/test_affinity_request_routing.py index 8b34b66..fd06e5f 100644 --- a/tests/test_affinity_request_routing.py +++ b/tests/test_affinity_request_routing.py @@ -13,18 +13,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import OrderedDict - +from collections import OrderedDict, deque import pytest from pyignite import * +from pyignite.connection import Connection from pyignite.datatypes import * from pyignite.datatypes.cache_config import CacheMode from pyignite.datatypes.prop_codes import * from tests.util import * -@pytest.mark.parametrize("key,grid_idx", [(1, 3), (2, 1), (3, 1), (4, 3), (5, 1), (6, 3), (11, 2), (13, 2), (19, 2)]) +requests = deque() +old_send = Connection.send + + +def patched_send(self, *args, **kwargs): + """Patched send function that push to queue idx of server to which request is routed.""" + requests.append(self.port % 100) + return old_send(self, *args, **kwargs) + + +def setup_function(): + requests.clear() + Connection.send = patched_send + + +def teardown_function(): + Connection.send = old_send + + +def wait_for_affinity_distribution(cache, key, node_idx, timeout=30): + real_node_idx = 0 + + def check_grid_idx(): + nonlocal real_node_idx + cache.get(key) + # conn = cache.get_best_node(key) + # real_node_idx = conn.port % 100 + # return real_node_idx == node_idx + real_node_idx = requests.pop() + return real_node_idx == node_idx + + res = wait_for_condition(check_grid_idx, timeout=timeout) + + if not res: + raise TimeoutError(f"failed to wait for affinity distribution, expected node_idx {node_idx}," + f"got {real_node_idx} instead") + + +@pytest.mark.parametrize("key,grid_idx", [(1, 1), (2, 2), (3, 3), (4, 1), (5, 1), (6, 2), (11, 1), (13, 1), (19, 1)]) @pytest.mark.parametrize("backups", [0, 1, 2, 3]) def test_cache_operation_on_primitive_key_routes_request_to_primary_node( request, key, grid_idx, backups, client_partition_aware): @@ -34,52 +72,51 @@ def test_cache_operation_on_primitive_key_routes_request_to_primary_node( PROP_BACKUPS_NUMBER: backups, }) - # Warm up affinity map cache.put(key, key) - get_request_grid_idx() + wait_for_affinity_distribution(cache, key, grid_idx) # Test cache.get(key) - assert get_request_grid_idx() == grid_idx + assert requests.pop() == grid_idx cache.put(key, key) - assert get_request_grid_idx("Put") == grid_idx + assert requests.pop() == grid_idx cache.replace(key, key + 1) - assert get_request_grid_idx("Replace") == grid_idx + assert requests.pop() == grid_idx cache.clear_key(key) - assert get_request_grid_idx("ClearKey") == grid_idx + assert requests.pop() == grid_idx cache.contains_key(key) - assert get_request_grid_idx("ContainsKey") == grid_idx + assert requests.pop() == grid_idx cache.get_and_put(key, 3) - assert get_request_grid_idx("GetAndPut") == grid_idx + assert requests.pop() == grid_idx cache.get_and_put_if_absent(key, 4) - assert get_request_grid_idx("GetAndPutIfAbsent") == grid_idx + assert requests.pop() == grid_idx cache.put_if_absent(key, 5) - assert get_request_grid_idx("PutIfAbsent") == grid_idx + assert requests.pop() == grid_idx cache.get_and_remove(key) - assert get_request_grid_idx("GetAndRemove") == grid_idx + assert requests.pop() == grid_idx cache.get_and_replace(key, 6) - assert get_request_grid_idx("GetAndReplace") == grid_idx + assert requests.pop() == grid_idx cache.remove_key(key) - assert get_request_grid_idx("RemoveKey") == grid_idx + assert requests.pop() == grid_idx cache.remove_if_equals(key, -1) - assert get_request_grid_idx("RemoveIfEquals") == grid_idx + assert requests.pop() == grid_idx cache.replace(key, -1) - assert get_request_grid_idx("Replace") == grid_idx + assert requests.pop() == grid_idx cache.replace_if_equals(key, 10, -10) - assert get_request_grid_idx("ReplaceIfEquals") == grid_idx + assert requests.pop() == grid_idx @pytest.mark.skip(reason="Custom key objects are not supported yet") @@ -121,7 +158,7 @@ class AffinityTestType1( cache.put(key_obj, 1) cache.put(key_obj, 2) - assert get_request_grid_idx("Put") == grid_idx + assert requests.pop() == grid_idx def test_cache_operation_routed_to_new_cluster_node(request, start_ignite_server, start_client): @@ -129,22 +166,20 @@ def test_cache_operation_routed_to_new_cluster_node(request, start_ignite_server client.connect([("127.0.0.1", 10801), ("127.0.0.1", 10802), ("127.0.0.1", 10803), ("127.0.0.1", 10804)]) cache = client.get_or_create_cache(request.node.name) key = 12 + wait_for_affinity_distribution(cache, key, 3) cache.put(key, key) cache.put(key, key) - assert get_request_grid_idx("Put") == 3 + assert requests.pop() == 3 srv = start_ignite_server(4) try: # Wait for rebalance and partition map exchange - def check_grid_idx(): - cache.get(key) - return get_request_grid_idx() == 4 - wait_for_condition(check_grid_idx, error='failed to wait for rebalance') + wait_for_affinity_distribution(cache, key, 4) # Response is correct and comes from the new node res = cache.get_and_remove(key) assert res == key - assert get_request_grid_idx("GetAndRemove") == 4 + assert requests.pop() == 4 finally: kill_process_tree(srv.pid) @@ -166,13 +201,13 @@ def verify_random_node(cache): key = 1 cache.put(key, key) - idx1 = get_request_grid_idx("Put") + idx1 = requests.pop() idx2 = idx1 # Try 10 times - random node may end up being the same for _ in range(1, 10): cache.put(key, key) - idx2 = get_request_grid_idx("Put") + idx2 = requests.pop() if idx2 != idx1: break assert idx1 != idx2 diff --git a/tests/test_affinity_single_connection.py b/tests/test_affinity_single_connection.py index c40393c..1943384 100644 --- a/tests/test_affinity_single_connection.py +++ b/tests/test_affinity_single_connection.py @@ -13,10 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from tests.util import get_request_grid_idx - def test_all_cache_operations_with_partition_aware_client_on_single_server(request, client_partition_aware_single_server): cache = client_partition_aware_single_server.get_or_create_cache(request.node.name) diff --git a/tests/util.py b/tests/util.py index 1d6acd6..90f0146 100644 --- a/tests/util.py +++ b/tests/util.py @@ -15,6 +15,8 @@ import glob import os + +import jinja2 as jinja2 import psutil import re import signal @@ -72,22 +74,19 @@ def get_ignite_config_path(use_ssl=False): if use_ssl: file_name = "ignite-config-ssl.xml" else: - file_name = "ignite-config.xml" + file_name = "ignite-config.xml.jinja2" return os.path.join(get_test_dir(), "config", file_name) def check_server_started(idx=1): - log_file = os.path.join(get_test_dir(), "logs", f"ignite-log-{idx}.txt") - if not os.path.exists(log_file): - return False - pattern = re.compile('^Topology snapshot.*') - with open(log_file) as f: - for line in f.readlines(): - if pattern.match(line): - return True + for log_file in get_log_files(idx): + with open(log_file) as f: + for line in f.readlines(): + if pattern.match(line): + return True return False @@ -102,20 +101,33 @@ def kill_process_tree(pid): os.kill(pid, signal.SIGKILL) +templateLoader = jinja2.FileSystemLoader(searchpath=os.path.join(get_test_dir(), "config")) +templateEnv = jinja2.Environment(loader=templateLoader) + + +def create_config_file(tpl_name, file_name, **kwargs): + template = templateEnv.get_template(tpl_name) + with open(os.path.join(get_test_dir(), "config", file_name), mode='w') as f: + f.write(template.render(**kwargs)) + + def _start_ignite(idx=1, debug=False, use_ssl=False): clear_logs(idx) runner = get_ignite_runner() env = os.environ.copy() - env['IGNITE_INSTANCE_INDEX'] = str(idx) - env['IGNITE_CLIENT_PORT'] = str(10800 + idx) if debug: env["JVM_OPTS"] = "-Djava.net.preferIPv4Stack=true -Xdebug -Xnoagent -Djava.compiler=NONE " \ "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 " - ignite_cmd = [runner, get_ignite_config_path(use_ssl)] + params = {'ignite_instance_idx': str(idx), 'ignite_client_port': 10800 + idx, 'use_ssl': use_ssl} + + create_config_file('log4j.xml.jinja2', f'log4j-{idx}.xml', **params) + create_config_file('ignite-config.xml.jinja2', f'ignite-config-{idx}.xml', **params) + + ignite_cmd = [runner, os.path.join(get_test_dir(), "config", f'ignite-config-{idx}.xml')] print("Starting Ignite server node:", ignite_cmd) srv = subprocess.Popen(ignite_cmd, env=env, cwd=get_test_dir()) @@ -142,38 +154,3 @@ def get_log_files(idx=1): def clear_logs(idx=1): for f in get_log_files(idx): os.remove(f) - - -def read_log_file(file, idx): - i = -1 - with open(file) as f: - lines = f.readlines() - for line in lines: - i += 1 - - if i < read_log_file.last_line[idx]: - continue - - if i > read_log_file.last_line[idx]: - read_log_file.last_line[idx] = i - - # Example: Client request received [reqId=1, addr=/127.0.0.1:51694, - # req=org.apache.ignite.internal.processors.platform.client.cache.ClientCachePutRequest@1f33101e] - res = re.match("Client request received .*?req=org.apache.ignite.internal.processors." - "platform.client.cache.ClientCache([a-zA-Z]+)Request@", line) - - if res is not None: - yield res.group(1) - - -def get_request_grid_idx(message="Get"): - res = -1 - for i in range(1, 5): - for log_file in get_log_files(i): - for log in read_log_file(log_file, i): - if log == message: - res = i # Do not exit early to advance all log positions - return res - - -read_log_file.last_line = [0, 0, 0, 0, 0] \ No newline at end of file From cae93048b955a1eb58b11e62fe584cd7bb555df3 Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Thu, 4 Feb 2021 22:56:53 +0300 Subject: [PATCH 3/8] IGNITE-13967 Fix serialization, reconnect and other bugs. --- pyignite/api/binary.py | 2 +- pyignite/binary.py | 44 ++++---- pyignite/connection/__init__.py | 61 ++++++----- pyignite/connection/handshake.py | 5 +- pyignite/datatypes/cache_properties.py | 7 +- pyignite/datatypes/complex.py | 110 ++++++-------------- pyignite/datatypes/internal.py | 34 +++--- pyignite/datatypes/null_object.py | 4 +- pyignite/datatypes/primitive.py | 35 +++---- pyignite/datatypes/primitive_arrays.py | 35 ++++--- pyignite/datatypes/primitive_objects.py | 20 ++-- pyignite/datatypes/standard.py | 80 ++++++++------ pyignite/queries/query.py | 31 +++--- pyignite/stream/binary_stream.py | 16 ++- pyignite/utils.py | 2 +- tests/test_cache_composite_key_class_sql.py | 2 +- 16 files changed, 241 insertions(+), 247 deletions(-) diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py index 43ef6b2..3192212 100644 --- a/pyignite/api/binary.py +++ b/pyignite/api/binary.py @@ -48,7 +48,7 @@ def get_binary_type(conn: 'Connection', binary_type: Union[str, int], query_id=N query_id=query_id, ) - _, send_buffer = query_struct.from_python({ + send_buffer = query_struct.from_python(conn, { 'type_id': entity_id(binary_type), }) conn.send(send_buffer) diff --git a/pyignite/binary.py b/pyignite/binary.py index 5d76c1b..c6a32c8 100644 --- a/pyignite/binary.py +++ b/pyignite/binary.py @@ -102,18 +102,16 @@ def __new__( mcs, name, (GenericObjectProps, )+base_classes, namespace ) - def _build(self, client: 'Client' = None) -> int: + def _from_python(self, stream): """ Method for building binary representation of the Generic object and calculating a hashcode from it. :param self: Generic object instance, - :param client: (optional) connection to Ignite cluster, + :param stream: BinaryStream """ - if client is None: - compact_footer = True - else: - compact_footer = client.compact_footer + + compact_footer = stream.compact_footer # prepare header header_class = BinaryObject.build_header() @@ -129,18 +127,19 @@ def _build(self, client: 'Client' = None) -> int: header.type_id = self.type_id header.schema_id = self.schema_id + header_len = ctypes.sizeof(header_class) + initial_pos = stream.tell() + # create fields and calculate offsets offsets = [ctypes.sizeof(header_class)] - field_buffer = bytearray() schema_items = list(self.schema.items()) + + stream.seek(initial_pos + header_len) for field_name, field_type in schema_items: - partial_buffer = field_type.from_python( - getattr( - self, field_name, getattr(field_type, 'default', None) - ) - ) - offsets.append(max(offsets) + len(partial_buffer)) - field_buffer += partial_buffer + val = getattr(self, field_name, getattr(field_type, 'default', None)) + field_start_pos = stream.tell() + field_type.from_python(stream, val) + offsets.append(max(offsets) + stream.tell() - field_start_pos) offsets = offsets[:-1] @@ -160,15 +159,16 @@ def _build(self, client: 'Client' = None) -> int: schema[i].offset = offset # calculate size and hash code - header.schema_offset = ( - ctypes.sizeof(header_class) - + len(field_buffer) - ) + fields_data_len = stream.tell() - initial_pos - header_len + header.schema_offset = fields_data_len + header_len header.length = header.schema_offset + ctypes.sizeof(schema_class) - header.hash_code = hashcode(field_buffer) + header.hash_code = stream.hashcode(initial_pos + header_len, fields_data_len) + + stream.seek(initial_pos) + stream.write(header) + stream.seek(initial_pos + header.schema_offset) + stream.write(schema) - # reuse the results - self._buffer = bytes(header) + field_buffer + bytes(schema) self._hashcode = header.hash_code def _setattr(self, attr_name: str, attr_value: Any): @@ -180,7 +180,7 @@ def _setattr(self, attr_name: str, attr_value: Any): # `super()` is really need these parameters super(result, self).__setattr__(attr_name, attr_value) - setattr(result, _build.__name__, _build) + setattr(result, _from_python.__name__, _from_python) setattr(result, '__setattr__', _setattr) setattr(result, '_buffer', None) setattr(result, '_hashcode', None) diff --git a/pyignite/connection/__init__.py b/pyignite/connection/__init__.py index 4e22de6..d74d646 100644 --- a/pyignite/connection/__init__.py +++ b/pyignite/connection/__init__.py @@ -35,7 +35,7 @@ from collections import OrderedDict import socket -from threading import Lock +from threading import RLock from typing import Union from pyignite.constants import * @@ -97,7 +97,7 @@ def _check_ssl_params(params): ).format(param)) def __init__( - self, client: 'Client', timeout: int = None, + self, client: 'Client', timeout: float = 2.0, username: str = None, password: str = None, **ssl_params ): """ @@ -149,7 +149,8 @@ def __init__( ssl_params['use_ssl'] = True self.ssl_params = ssl_params self._failed = False - self._in_use = Lock() + self._mux = RLock() + self._in_use = False @property def socket(self) -> socket.socket: @@ -159,17 +160,20 @@ def socket(self) -> socket.socket: @property def closed(self) -> bool: """ Tells if socket is closed. """ - return self._socket is None + with self._mux: + return self._socket is None @property def failed(self) -> bool: """ Tells if connection is failed. """ - return self._failed + with self._mux: + return self._failed @property def alive(self) -> bool: """ Tells if connection is up and no failure detected. """ - return not (self._failed or self.closed) + with self._mux: + return not (self._failed or self.closed) def __repr__(self) -> str: return '{}:{}'.format(self.host or '?', self.port or '?') @@ -186,10 +190,10 @@ def get_protocol_version(self): def _fail(self): """ set client to failed state. """ - self._failed = True + with self._mux: + self._failed = True - if self._in_use.locked(): - self._in_use.release() + self._in_use = False def read_response(self) -> Union[dict, OrderedDict]: """ @@ -234,9 +238,10 @@ def connect( """ detecting_protocol = False - # go non-blocking for faster reconnect - if not self._in_use.acquire(blocking=False): - raise ConnectionError('Connection is in use.') + with self._mux: + if self._in_use: + raise ConnectionError('Connection is in use.') + self._in_use = True # choose highest version first if self.client.protocol_version is None: @@ -289,7 +294,11 @@ def _connect_version( self.username, self.password ) - self.send(bytes(hs_request)) + + with BinaryStream(None, self) as stream: + hs_request.from_python(stream) + self.send(stream.getvalue()) + hs_response = self.read_response() if hs_response['op_code'] == 0: # disconnect but keep in use @@ -345,10 +354,7 @@ def _reconnect(self): if not self.failed: return - # return connection to initial state regardless of use lock - self.close(release=False) - if self._in_use.locked(): - self._in_use.release() + self.close() # connect and silence the connection errors try: @@ -427,13 +433,14 @@ def close(self, release=True): not required, since sockets are automatically closed when garbage-collected. """ - if release and self._in_use.locked(): - self._in_use.release() - - if self._socket: - try: - self._socket.shutdown(socket.SHUT_RDWR) - self._socket.close() - except connection_errors: - pass - self._socket = None + with self._mux: + if self._socket: + try: + self._socket.shutdown(socket.SHUT_RDWR) + self._socket.close() + except connection_errors: + pass + self._socket = None + + if release: + self._in_use = False diff --git a/pyignite/connection/handshake.py b/pyignite/connection/handshake.py index 2e0264f..3315c4e 100644 --- a/pyignite/connection/handshake.py +++ b/pyignite/connection/handshake.py @@ -50,7 +50,7 @@ def __init__( ]) self.handshake_struct = Struct(fields) - def __bytes__(self) -> bytes: + def from_python(self, stream): handshake_data = { 'length': 8, 'op_code': OP_HANDSHAKE, @@ -69,4 +69,5 @@ def __bytes__(self) -> bytes: len(self.username), len(self.password), ]) - return self.handshake_struct.from_python(handshake_data) + + self.handshake_struct.from_python(stream, handshake_data) diff --git a/pyignite/datatypes/cache_properties.py b/pyignite/datatypes/cache_properties.py index 0d34200..1867ac0 100644 --- a/pyignite/datatypes/cache_properties.py +++ b/pyignite/datatypes/cache_properties.py @@ -116,11 +116,12 @@ def to_python(cls, ctype_object, *args, **kwargs): ) @classmethod - def from_python(cls, value): + def from_python(cls, stream, value): header_class = cls.build_header() header = header_class() header.prop_code = cls.prop_code - return bytes(header) + cls.prop_data_class.from_python(value) + stream.write(bytes(header)) + cls.prop_data_class.from_python(stream, value) class PropName(PropBase): @@ -276,7 +277,7 @@ class PropStatisticsEnabled(PropBase): class AnyProperty(PropBase): @classmethod - def from_python(cls, value): + def from_python(cls, stream, value): raise Exception( 'You must choose a certain type ' 'for your cache configuration property' diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py index 4f5dc6a..d49b763 100644 --- a/pyignite/datatypes/complex.py +++ b/pyignite/datatypes/complex.py @@ -33,6 +33,8 @@ 'WrappedDataObject', 'BinaryObject', ] +from ..stream import BinaryStream + class ObjectArrayObject(IgniteDataType): """ @@ -110,9 +112,10 @@ def to_python(cls, ctype_object, *args, **kwargs): return ctype_object.type_id, result @classmethod - def from_python(cls, value): + def from_python(cls, stream, value): if value is None: - return Null.from_python() + Null.from_python(stream) + return type_or_id, value = value header_class = cls.build_header() @@ -128,11 +131,10 @@ def from_python(cls, value): length = 1 header.length = length header.type_id = type_or_id - buffer = bytearray(header) + stream.write(header) for x in value: - buffer += infer_from_python(x) - return bytes(buffer) + infer_from_python(stream, x) class WrappedDataObject(IgniteDataType): @@ -192,7 +194,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return bytes(ctype_object.payload), ctype_object.offset @classmethod - def from_python(cls, value): + def from_python(cls, stream, value): raise ParseError('Send unwrapped data.') @@ -301,9 +303,10 @@ def to_python(cls, ctype_object, *args, **kwargs): return ctype_object.type, result @classmethod - def from_python(cls, value): + def from_python(cls, stream, value): if value is None: - return Null.from_python() + Null.from_python(stream) + return type_or_id, value = value header_class = cls.build_header() @@ -319,11 +322,10 @@ def from_python(cls, value): length = 1 header.length = length header.type = type_or_id - buffer = bytearray(header) + stream.write(header) for x in value: - buffer += infer_from_python(x) - return bytes(buffer) + infer_from_python(stream, x) class Map(IgniteDataType): @@ -401,7 +403,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return result @classmethod - def from_python(cls, value, type_id=None): + def from_python(cls, stream, value, type_id=None): header_class = cls.build_header() header = header_class() length = len(value) @@ -413,12 +415,11 @@ def from_python(cls, value, type_id=None): ) if hasattr(header, 'type'): header.type = type_id - buffer = bytearray(header) + stream.write(header) for k, v in value.items(): - buffer += infer_from_python(k) - buffer += infer_from_python(v) - return bytes(buffer) + infer_from_python(stream, k) + infer_from_python(stream, v) class MapObject(Map): @@ -461,12 +462,13 @@ def to_python(cls, ctype_object, *args, **kwargs): ) @classmethod - def from_python(cls, value): + def from_python(cls, stream, value): if value is None: - return Null.from_python() + Null.from_python(stream) + return type_id, value = value - return super().from_python(value, type_id) + super().from_python(stream, value, type_id) class BinaryObject(IgniteDataType): @@ -481,42 +483,14 @@ class BinaryObject(IgniteDataType): COMPACT_FOOTER = 0x0020 @staticmethod - def find_client(): - """ - A nice hack. Extracts the nearest `Client` instance from the - call stack. - """ - from pyignite import Client - from pyignite.connection import Connection - - frame = None - try: - for rec in inspect.stack()[2:]: - frame = rec[0] - code = frame.f_code - for varname in code.co_varnames: - suspect = frame.f_locals.get(varname) - if isinstance(suspect, Client): - return suspect - if isinstance(suspect, Connection): - return suspect.client - finally: - del frame - - @staticmethod - def hashcode( - value: object, client, *args, **kwargs - ) -> int: + def hashcode(value: object, client: None) -> int: # binary objects's hashcode implementation is special in the sense # that you need to fully serialize the object to calculate # its hashcode - if value._hashcode is None: - - # …and for to serialize it you need a Client instance - if client is None: - client = BinaryObject.find_client() + if not value._hashcode and client : - value._build(client) + with BinaryStream(None, client.random_node) as stream: + value._from_python(stream) return value._hashcode @@ -564,17 +538,6 @@ def schema_type(cls, flags: int): }, ) - @staticmethod - def get_dataclass(conn: 'Connection', header) -> OrderedDict: - # get field names from outer space - result = conn.client.query_binary_type( - header.type_id, - header.schema_id - ) - if not result: - raise ParseError('Binary type is not registered') - return result - @classmethod def parse(cls, stream): from pyignite.datatypes import Struct @@ -641,23 +604,10 @@ def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs): return result @classmethod - def from_python(cls, value: object): + def from_python(cls, stream, value): if value is None: - return Null.from_python() - - if getattr(value, '_buffer', None) is None: - client = cls.find_client() - - # if no client can be found, the class of the `value` is discarded - # and the new dataclass is automatically registered later on - if client: - client.register_binary_type(value.__class__) - else: - raise Warning( - 'Can not register binary type {}'.format(value.type_name) - ) - - # build binary representation - value._build(client) + Null.from_python(stream) + return - return value._buffer + stream.register_binary_type(value.__class__) + value._from_python(stream) diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py index e474693..40b336f 100644 --- a/pyignite/datatypes/internal.py +++ b/pyignite/datatypes/internal.py @@ -179,20 +179,19 @@ def to_python(self, ctype_object, *args, **kwargs): ) return result - def from_python(self, value): + def from_python(self, stream, value): length = len(value) header_class = self.build_header_class() header = header_class() header.length = length - buffer = bytearray(header) + + stream.write(header) for i, v in enumerate(value): for default_key, default_value in self.defaults.items(): v.setdefault(default_key, default_value) for name, el_class in self.following: - buffer += el_class.from_python(v[name]) - - return bytes(buffer) + el_class.from_python(stream, v[name]) @attr.s @@ -243,16 +242,12 @@ def to_python( ) return result - def from_python(self, value) -> bytes: - buffer = b'' - + def from_python(self, stream, value): for default_key, default_value in self.defaults.items(): value.setdefault(default_key, default_value) for name, el_class in self.fields: - buffer += el_class.from_python(value[name]) - - return buffer + el_class.from_python(stream, value[name]) class AnyDataObject: @@ -416,11 +411,12 @@ def map_python_type(cls, value): ) @classmethod - def from_python(cls, value): - return cls.map_python_type(value).from_python(value) + def from_python(cls, stream, value): + p_type = cls.map_python_type(value) + p_type.from_python(stream, value) -def infer_from_python(value: Any): +def infer_from_python(stream, value: Any): """ Convert pythonic value to ctypes buffer, type hint-aware. @@ -431,7 +427,8 @@ def infer_from_python(value: Any): value, data_type = value else: data_type = AnyDataObject - return data_type.from_python(value) + + data_type.from_python(stream, value) @attr.s @@ -489,7 +486,7 @@ def to_python(cls, ctype_object, *args, **kwargs): ) return result - def from_python(self, value): + def from_python(self, stream, value): header_class = self.build_header() header = header_class() @@ -499,8 +496,7 @@ def from_python(self, value): value = [value] length = 1 header.length = length - buffer = bytearray(header) + stream.write(header) for x in value: - buffer += infer_from_python(x) - return bytes(buffer) + infer_from_python(stream, x) diff --git a/pyignite/datatypes/null_object.py b/pyignite/datatypes/null_object.py index 4c5382a..a44ad22 100644 --- a/pyignite/datatypes/null_object.py +++ b/pyignite/datatypes/null_object.py @@ -65,6 +65,6 @@ def to_python(*args, **kwargs): return None @staticmethod - def from_python(*args): - return TC_NULL + def from_python(stream, *args): + stream.write(TC_NULL) diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py index 59e41ff..13aa76c 100644 --- a/pyignite/datatypes/primitive.py +++ b/pyignite/datatypes/primitive.py @@ -60,8 +60,8 @@ class Byte(Primitive): c_type = ctypes.c_byte @classmethod - def from_python(cls, value): - return struct.pack(" object: return result -def hashcode(data: Union[str, bytes]) -> int: +def hashcode(data: Union[str, bytes, bytearray, memoryview]) -> int: """ Calculate hash code used for identifying objects in Ignite binary API. diff --git a/tests/test_cache_composite_key_class_sql.py b/tests/test_cache_composite_key_class_sql.py index 2f1705f..df1cac3 100644 --- a/tests/test_cache_composite_key_class_sql.py +++ b/tests/test_cache_composite_key_class_sql.py @@ -117,7 +117,7 @@ def validate_query_result(student_key, student_val, query_result): assert len(query_result) == 2 sql_row = dict(zip(query_result[0], query_result[1])) - assert sql_row["_KEY"][0] == student_key._buffer + #assert sql_row["_KEY"][0] == student_key._buffer assert sql_row['ID'] == student_key.ID assert sql_row['DEPT'] == student_key.DEPT assert sql_row['NAME'] == student_val.NAME From b54973c4cee0ba8dfe2b8dfd7b83dac0170852ae Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Fri, 5 Feb 2021 18:56:45 +0300 Subject: [PATCH 4/8] IGNITE-13967 Optimize allocations during parsing --- pyignite/api/affinity.py | 4 +- pyignite/api/binary.py | 42 ++++---- pyignite/binary.py | 1 + pyignite/connection/__init__.py | 10 +- pyignite/datatypes/cache_properties.py | 12 ++- pyignite/datatypes/complex.py | 101 +++++++++++--------- pyignite/datatypes/internal.py | 44 ++++----- pyignite/datatypes/null_object.py | 7 +- pyignite/datatypes/primitive.py | 5 +- pyignite/datatypes/primitive_arrays.py | 15 +-- pyignite/datatypes/primitive_objects.py | 12 ++- pyignite/datatypes/standard.py | 73 ++++++++------ pyignite/queries/query.py | 4 +- pyignite/queries/response.py | 45 ++++----- pyignite/stream/binary_stream.py | 3 + pyignite/utils.py | 10 +- tests/config/ignite-config.xml.jinja2 | 2 +- tests/config/log4j.xml.jinja2 | 60 +++++------- tests/test_affinity_request_routing.py | 10 +- tests/test_cache_composite_key_class_sql.py | 2 +- 20 files changed, 250 insertions(+), 212 deletions(-) diff --git a/pyignite/api/affinity.py b/pyignite/api/affinity.py index d28cfb8..584f73c 100644 --- a/pyignite/api/affinity.py +++ b/pyignite/api/affinity.py @@ -55,11 +55,11 @@ partition_mapping = StructArray([ ('is_applicable', Bool), - ('cache_mapping', Conditional(lambda ctx: ctx['is_applicable'] == b'\x01', + ('cache_mapping', Conditional(lambda stream, ctx: stream.mem_view(*ctx['is_applicable']) == b'\x01', lambda ctx: ctx['is_applicable'] is True, cache_mapping, empty_cache_mapping)), - ('node_mapping', Conditional(lambda ctx: ctx['is_applicable'] == b'\x01', + ('node_mapping', Conditional(lambda stream, ctx: stream.mem_view(*ctx['is_applicable']) == b'\x01', lambda ctx: ctx['is_applicable'] is True, node_mapping, empty_node_mapping)), ]) diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py index 3192212..ff85856 100644 --- a/pyignite/api/binary.py +++ b/pyignite/api/binary.py @@ -57,31 +57,35 @@ def get_binary_type(conn: 'Connection', binary_type: Union[str, int], query_id=N following=[('type_exists', Bool)]) with BinaryStream(conn.recv(), conn) as stream: - response_head_type, recv_buffer = response_head_struct.parse(stream) - response_head = response_head_type.from_buffer_copy(recv_buffer) + response_head_type, response_positions = response_head_struct.parse(stream) + response_head = response_head_type.from_buffer_copy(stream.mem_view(*response_positions)) + init_pos, total_len = response_positions + response_parts = [] if response_head.type_exists: - resp_body_type, resp_body_buffer = body_struct.parse(stream) + resp_body_type, resp_body_positions = body_struct.parse(stream) response_parts.append(('body', resp_body_type)) - resp_body = resp_body_type.from_buffer_copy(resp_body_buffer) - recv_buffer += resp_body_buffer + resp_body = resp_body_type.from_buffer_copy(stream.mem_view(*resp_body_positions)) + total_len += resp_body_positions[1] if resp_body.is_enum: - resp_enum, resp_enum_buffer = enum_struct.parse(stream) + resp_enum, resp_enum_positions = enum_struct.parse(stream) response_parts.append(('enums', resp_enum)) - recv_buffer += resp_enum_buffer - resp_schema_type, resp_schema_buffer = schema_struct.parse(stream) + total_len += resp_enum_positions[1] + + resp_schema_type, resp_schema_positions = schema_struct.parse(stream) response_parts.append(('schema', resp_schema_type)) - recv_buffer += resp_schema_buffer - - response_class = type( - 'GetBinaryTypeResponse', - (response_head_type,), - { - '_pack_': 1, - '_fields_': response_parts, - } - ) - response = response_class.from_buffer_copy(recv_buffer) + total_len += resp_schema_positions[1] + + response_class = type( + 'GetBinaryTypeResponse', + (response_head_type,), + { + '_pack_': 1, + '_fields_': response_parts, + } + ) + response = response_class.from_buffer_copy(stream.mem_view(init_pos, total_len)) + result = APIResult(response) if result.status != 0: return result diff --git a/pyignite/binary.py b/pyignite/binary.py index c6a32c8..dfc18ef 100644 --- a/pyignite/binary.py +++ b/pyignite/binary.py @@ -169,6 +169,7 @@ def _from_python(self, stream): stream.seek(initial_pos + header.schema_offset) stream.write(schema) + self._buffer = bytes(stream.mem_view(initial_pos, stream.tell() - initial_pos)) self._hashcode = header.hash_code def _setattr(self, attr_name: str, attr_value: Any): diff --git a/pyignite/connection/__init__.py b/pyignite/connection/__init__.py index d74d646..2389f47 100644 --- a/pyignite/connection/__init__.py +++ b/pyignite/connection/__init__.py @@ -205,9 +205,9 @@ def read_response(self) -> Union[dict, OrderedDict]: ('length', Int), ('op_code', Byte), ]) - with BinaryStream(self.recv(), self) as response_stream: - start_class, start_buffer = response_start.parse(response_stream) - start = start_class.from_buffer_copy(start_buffer) + with BinaryStream(self.recv(), self) as stream: + start_class, start_positions = response_start.parse(stream) + start = start_class.from_buffer_copy(stream.mem_view(*start_positions)) data = response_start.to_python(start) response_end = None if data['op_code'] == 0: @@ -222,8 +222,8 @@ def read_response(self) -> Union[dict, OrderedDict]: ('node_uuid', UUIDObject), ]) if response_end: - end_class, end_buffer = response_end.parse(response_stream) - end = end_class.from_buffer_copy(end_buffer) + end_class, end_positions = response_end.parse(stream) + end = end_class.from_buffer_copy(stream.mem_view(*end_positions)) data.update(response_end.to_python(end)) return data diff --git a/pyignite/datatypes/cache_properties.py b/pyignite/datatypes/cache_properties.py index 1867ac0..2443577 100644 --- a/pyignite/datatypes/cache_properties.py +++ b/pyignite/datatypes/cache_properties.py @@ -93,10 +93,12 @@ def build_header(cls): @classmethod def parse(cls, stream): + init_pos = stream.tell() + header_class = cls.build_header() - buf = stream.read(ctypes.sizeof(header_class)) - data_class, data_buf = cls.prop_data_class.parse(stream) - buf += data_buf + header_len = ctypes.sizeof(header_class) + data_class, data_buf = cls.prop_data_class.parse(stream.mem_view(init_pos, header_len)) + prop_class = type( cls.__name__, (header_class,), @@ -107,7 +109,9 @@ def parse(cls, stream): ], } ) - return prop_class, buf + + stream.seek(init_pos + ctypes.sizeof(prop_class)) + return prop_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py index d49b763..971408c 100644 --- a/pyignite/datatypes/complex.py +++ b/pyignite/datatypes/complex.py @@ -15,7 +15,7 @@ from collections import OrderedDict import ctypes -import inspect +from io import SEEK_CUR from typing import Iterable, Dict from pyignite.constants import * @@ -71,19 +71,20 @@ def build_header(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) - fields = [] + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(init_pos + header_len) + fields = [] for i in range(header.length): - c_type, buffer_fragment = AnyDataObject.parse(stream) - buffer += buffer_fragment + c_type, _ = AnyDataObject.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -94,7 +95,9 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, buffer + + stream.seek(init_pos + ctypes.sizeof(final_class)) + return final_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -164,14 +167,16 @@ def build_header(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(init_pos + header_len) final_class = type( cls.__name__, @@ -184,10 +189,9 @@ def parse(cls, stream): ], } ) - buffer += stream.read( - ctypes.sizeof(final_class) - ctypes.sizeof(header_class) - ) - return final_class, buffer + + stream.seek(init_pos + ctypes.sizeof(final_class)) + return final_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -262,19 +266,21 @@ def build_header(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - if buffer == TC_NULL: - return Null.build_c_type(), buffer + type_ = stream.mem_view(init_pos, type_len) + if type_ == TC_NULL: + stream.seek(type_len, SEEK_CUR) + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) - fields = [] + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(init_pos + header_len) + fields = [] for i in range(header.length): - c_type, buffer_fragment = AnyDataObject.parse(stream) - buffer += buffer_fragment + c_type, _ = AnyDataObject.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -285,7 +291,7 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, buffer + return final_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -360,19 +366,20 @@ def build_header(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) - fields = [] + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(init_pos + header_len) + fields = [] for i in range(header.length << 1): - c_type, buffer_fragment = AnyDataObject.parse(stream) - buffer += buffer_fragment + c_type, _ = AnyDataObject.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -383,7 +390,7 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, buffer + return final_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -541,27 +548,30 @@ def schema_type(cls, flags: int): @classmethod def parse(cls, stream): from pyignite.datatypes import Struct - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - if buffer == TC_NULL: - return Null.build_c_type(), buffer + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + + type_ = stream.mem_view(init_pos, type_len) + if type_ == TC_NULL: + stream.seek(type_len, SEEK_CUR) + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(init_pos + header_len) # ignore full schema, always retrieve fields' types and order # from complex types registry data_class = stream.get_dataclass(header) fields = data_class.schema.items() object_fields_struct = Struct(fields) - object_fields, object_fields_buffer = object_fields_struct.parse(stream) - buffer += object_fields_buffer + object_fields, _ = object_fields_struct.parse(stream) final_class_fields = [('object_fields', object_fields)] if header.flags & cls.HAS_SCHEMA: schema = cls.schema_type(header.flags) * len(fields) - buffer += stream.read(ctypes.sizeof(schema)) + stream.seek(ctypes.sizeof(schema), SEEK_CUR) final_class_fields.append(('schema', schema)) final_class = type( @@ -574,7 +584,7 @@ def parse(cls, stream): ) # register schema encoding approach stream.compact_footer = bool(header.flags & cls.COMPACT_FOOTER) - return final_class, buffer + return final_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs): @@ -610,4 +620,7 @@ def from_python(cls, stream, value): return stream.register_binary_type(value.__class__) - value._from_python(stream) + if getattr(value, '_buffer', None): + stream.write(value._buffer) + else: + value._from_python(stream) diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py index 40b336f..139a55b 100644 --- a/pyignite/datatypes/internal.py +++ b/pyignite/datatypes/internal.py @@ -17,6 +17,7 @@ import ctypes import decimal from datetime import date, datetime, timedelta +from io import SEEK_CUR from typing import Any, Tuple, Union, Callable import uuid @@ -113,17 +114,18 @@ def tc_map(key: bytes, _memo_map: dict = {}): class Conditional: - def __init__(self, predicate1: Callable[[any], bool], predicate2: Callable[[any], bool], var1, var2): + def __init__(self, predicate1: Callable[['BinaryStream', any], bool], predicate2: Callable[[any], bool], var1, var2): self.predicate1 = predicate1 self.predicate2 = predicate2 self.var1 = var1 self.var2 = var2 def parse(self, stream, context): - return self.var1.parse(stream) if self.predicate1(context) else self.var2.parse(stream) + return self.var1.parse(stream) if self.predicate1(stream, context) else self.var2.parse(stream) def to_python(self, ctype_object, context, *args, **kwargs): - return self.var1.to_python(ctype_object, *args, **kwargs) if self.predicate2(context) else self.var2.to_python(ctype_object, *args, **kwargs) + return self.var1.to_python(ctype_object, *args, **kwargs) if self.predicate2(context)\ + else self.var2.to_python(ctype_object, *args, **kwargs) @attr.s class StructArray: @@ -145,13 +147,13 @@ def build_header_class(self): ) def parse(self, stream): + init_pos = stream.tell() buffer = stream.read(ctypes.sizeof(self.counter_type)) length = int.from_bytes(buffer, byteorder=PROTOCOL_BYTE_ORDER) fields = [] for i in range(length): - c_type, buffer_fragment = Struct(self.following).parse(stream) - buffer += buffer_fragment + c_type, _ = Struct(self.following).parse(stream) fields.append(('element_{}'.format(i), c_type)) data_class = type( @@ -163,7 +165,7 @@ def parse(self, stream): }, ) - return data_class, buffer + return data_class, (init_pos, stream.tell() - init_pos) def to_python(self, ctype_object, *args, **kwargs): result = [] @@ -201,19 +203,15 @@ class Struct: dict_type = attr.ib(default=OrderedDict) defaults = attr.ib(type=dict, default={}) - def parse(self, stream) -> Tuple[ctypes.LittleEndianStructure, bytes]: - buffer = b'' - fields = [] - values = {} + def parse(self, stream): + init_pos = stream.tell() + fields, values = [], {} for name, c_type in self.fields: is_cond = isinstance(c_type, Conditional) - c_type, buffer_fragment = c_type.parse(stream, values) if is_cond else c_type.parse(stream) - buffer += buffer_fragment - + c_type, value_position = c_type.parse(stream, values) if is_cond else c_type.parse(stream) fields.append((name, c_type)) - - values[name] = buffer_fragment + values[name] = value_position data_class = type( 'Struct', @@ -224,7 +222,7 @@ def parse(self, stream) -> Tuple[ctypes.LittleEndianStructure, bytes]: }, ) - return data_class, buffer + return data_class, (init_pos, stream.tell() - init_pos) def to_python( self, ctype_object, *args, **kwargs @@ -298,7 +296,7 @@ def parse(cls, stream): data_class = tc_map(bytes(type_code)) except KeyError: raise ParseError('Unknown type code: `{}`'.format(type_code)) - stream.seek(stream.tell() - ctypes.sizeof(ctypes.c_byte)) + stream.seek(-ctypes.sizeof(ctypes.c_byte), SEEK_CUR) return data_class.parse(stream) @classmethod @@ -452,13 +450,13 @@ def build_header(self): def parse(self, stream): header_class = self.build_header() - buffer = stream.read(ctypes.sizeof(header_class)) - header = header_class.from_buffer_copy(buffer) - fields = [] + header_len, initial_pos = ctypes.sizeof(header_class), stream.tell() + header = header_class.from_buffer_copy(stream.mem_view(initial_pos, header_len)) + stream.seek(header_len, SEEK_CUR) + fields = [] for i in range(header.length): - c_type, buffer_fragment = super().parse(stream) - buffer += buffer_fragment + c_type, _ = super().parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -469,7 +467,7 @@ def parse(self, stream): '_fields_': fields, } ) - return final_class, buffer + return final_class, (initial_pos, stream.tell() - initial_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/null_object.py b/pyignite/datatypes/null_object.py index a44ad22..4db0c9f 100644 --- a/pyignite/datatypes/null_object.py +++ b/pyignite/datatypes/null_object.py @@ -20,6 +20,7 @@ """ import ctypes +from io import SEEK_CUR from typing import Any from .base import IgniteDataType @@ -56,9 +57,9 @@ def build_c_type(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - data_type = cls.build_c_type() - return data_type, buffer + init_pos, offset = stream.tell(), ctypes.sizeof(ctypes.c_byte) + stream.seek(offset, SEEK_CUR) + return cls.build_c_type(), (init_pos, offset) @staticmethod def to_python(*args, **kwargs): diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py index 13aa76c..dde49ff 100644 --- a/pyignite/datatypes/primitive.py +++ b/pyignite/datatypes/primitive.py @@ -15,6 +15,7 @@ import ctypes import struct +from io import SEEK_CUR from pyignite.constants import * from .base import IgniteDataType @@ -47,7 +48,9 @@ class Primitive(IgniteDataType): @classmethod def parse(cls, stream): - return cls.c_type, stream.read(ctypes.sizeof(cls.c_type)) + init_pos, offset = stream.tell(), ctypes.sizeof(cls.c_type) + stream.seek(offset, SEEK_CUR) + return cls.c_type, (init_pos, offset) @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py index fe90a28..65ed9c3 100644 --- a/pyignite/datatypes/primitive_arrays.py +++ b/pyignite/datatypes/primitive_arrays.py @@ -62,14 +62,16 @@ def build_header_class(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header_class() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + final_class = type( cls.__name__, (header_class,), @@ -80,8 +82,9 @@ def parse(cls, stream): ], } ) - buffer += stream.read(ctypes.sizeof(final_class) - ctypes.sizeof(header_class)) - return final_class, buffer + data_len = ctypes.sizeof(final_class) + stream.seek(init_pos + data_len) + return final_class, (init_pos, data_len) @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py index 5cff0df..b4ea7e6 100644 --- a/pyignite/datatypes/primitive_objects.py +++ b/pyignite/datatypes/primitive_objects.py @@ -61,13 +61,17 @@ def build_c_type(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) data_type = cls.build_c_type() - buffer += stream.read(ctypes.sizeof(data_type) - len(buffer)) - return data_type, buffer + data_len = ctypes.sizeof(data_type) + stream.seek(init_pos + data_len) + + return data_type, (init_pos, data_len) @staticmethod def to_python(ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py index e8518b3..6b33b82 100644 --- a/pyignite/datatypes/standard.py +++ b/pyignite/datatypes/standard.py @@ -16,6 +16,7 @@ import ctypes from datetime import date, datetime, time, timedelta import decimal +from io import SEEK_CUR from math import ceil from typing import Any, Tuple import uuid @@ -55,14 +56,17 @@ def build_c_type(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) - c_type = cls.build_c_type() - buffer += stream.read(ctypes.sizeof(c_type) - len(buffer)) - return c_type, buffer + data_type = cls.build_c_type() + data_len = ctypes.sizeof(data_type) + stream.seek(init_pos + data_len) + + return data_type, (init_pos, data_len) class String(IgniteDataType): @@ -96,18 +100,22 @@ def build_c_type(cls, length: int): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - # String or Null + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + + buffer = stream.read(type_len) if buffer == TC_NULL: - return Null.build_c_type(), buffer + return Null.build_c_type(), (init_pos, type_len) - buffer += stream.read(ctypes.sizeof(ctypes.c_int)) - length = int.from_bytes(buffer[1:], byteorder=PROTOCOL_BYTE_ORDER) + length = int.from_bytes( + stream.read(ctypes.sizeof(ctypes.c_int)), + byteorder=PROTOCOL_BYTE_ORDER + ) data_type = cls.build_c_type(length) - buffer += stream.read(ctypes.sizeof(data_type) - len(buffer)) + data_len = ctypes.sizeof(data_type) + stream.seek(init_pos + data_len) - return data_type, buffer + return data_type, (init_pos, data_len) @staticmethod def to_python(ctype_object, *args, **kwargs): @@ -168,14 +176,17 @@ def build_c_header(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) - # Decimal or Null - if buffer == TC_NULL: - return Null.build_c_type(), buffer + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) + + type_ = stream.mem_view(init_pos, type_len) + if type_ == TC_NULL: + stream.seek(type_len, SEEK_CUR) + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_c_header() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + data_type = type( cls.__name__, (header_class,), @@ -186,8 +197,11 @@ def parse(cls, stream): ], } ) - buffer += stream.read(ctypes.sizeof(data_type) - ctypes.sizeof(header_class)) - return data_type, buffer + + data_len = ctypes.sizeof(data_type) + stream.seek(init_pos + data_len) + + return data_type, (init_pos, data_len) @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -611,18 +625,21 @@ def build_header_class(cls): @classmethod def parse(cls, stream): - buffer = stream.read(ctypes.sizeof(ctypes.c_byte)) + init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - if buffer == TC_NULL: - return Null.build_c_type(), buffer + type_ = stream.mem_view(init_pos, type_len) + if type_ == TC_NULL: + stream.seek(type_len, SEEK_CUR) + return Null.build_c_type(), (init_pos, type_len) header_class = cls.build_header_class() - buffer += stream.read(ctypes.sizeof(header_class) - len(buffer)) - header = header_class.from_buffer_copy(buffer) + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(init_pos + header_len) + fields = [] for i in range(header.length): - c_type, buffer_fragment = cls.standard_type.parse(stream) - buffer += buffer_fragment + c_type, _ = cls.standard_type.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -633,7 +650,7 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, buffer + return final_class, (init_pos, stream.tell() - init_pos) @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/queries/query.py b/pyignite/queries/query.py index 50b8752..56d1ff1 100644 --- a/pyignite/queries/query.py +++ b/pyignite/queries/query.py @@ -102,8 +102,8 @@ def perform( following=response_config) with BinaryStream(conn.recv(), conn) as stream: - response_ctype, recv_buffer = response_struct.parse(stream) - response = response_ctype.from_buffer_copy(recv_buffer) + response_ctype, response_positions = response_struct.parse(stream) + response = response_ctype.from_buffer_copy(stream.mem_view(*response_positions)) # this test depends on protocol version if getattr(response, 'flags', False) & RHF_TOPOLOGY_CHANGED: diff --git a/pyignite/queries/response.py b/pyignite/queries/response.py index 4b86841..48f4adc 100644 --- a/pyignite/queries/response.py +++ b/pyignite/queries/response.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from io import SEEK_CUR import attr from collections import OrderedDict @@ -56,11 +57,14 @@ def build_header(self): return self._response_header def parse(self, stream): + init_pos = stream.tell() + header_class = self.build_header() - buffer = stream.read(ctypes.sizeof(header_class)) - header = header_class.from_buffer_copy(buffer) - fields = [] + header_len = ctypes.sizeof(header_class) + header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + stream.seek(header_len, SEEK_CUR) + fields = [] has_error = False if self.protocol_version and self.protocol_version >= (1, 4, 0): if header.flags & RHF_TOPOLOGY_CHANGED: @@ -76,18 +80,19 @@ def parse(self, stream): has_error = header.status_code != OP_SUCCESS if fields: - buffer += stream.read(sum([ctypes.sizeof(c_type) for _, c_type in fields])) + stream.seek(sum([ctypes.sizeof(c_type) for _, c_type in fields]), SEEK_CUR) if has_error: - msg_type, buffer_fragment = String.parse(stream) - buffer += buffer_fragment + msg_type, _ = String.parse(stream) fields.append(('error_message', msg_type)) else: - self._parse_success(stream, buffer, fields) + self._parse_success(stream, fields) - return self._create_parse_result(stream, header_class, fields, buffer) + response_class = self._create_response_class(stream, header_class, fields) + stream.seek(init_pos + ctypes.sizeof(response_class)) + return self._create_response_class(stream, header_class, fields), (init_pos, stream.tell() - init_pos) - def _create_parse_result(self, conn: Connection, header_class, fields: list, buffer: bytearray): + def _create_response_class(self, stream, header_class, fields: list): response_class = type( 'Response', (header_class,), @@ -96,12 +101,11 @@ def _create_parse_result(self, conn: Connection, header_class, fields: list, buf '_fields_': fields, } ) - return response_class, buffer + return response_class - def _parse_success(self, stream, buffer: bytearray, fields: list): + def _parse_success(self, stream, fields: list): for name, ignite_type in self.following: - c_type, buffer_fragment = ignite_type.parse(stream) - buffer += buffer_fragment + c_type, _ = ignite_type.parse(stream) fields.append((name, c_type)) def to_python(self, ctype_object, *args, **kwargs): @@ -132,7 +136,7 @@ def fields_or_field_count(self): return 'fields', StringArray return 'field_count', Int - def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): + def _parse_success(self, stream, fields: list): following = [ self.fields_or_field_count(), ('row_count', Int), @@ -140,9 +144,8 @@ def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): if self.has_cursor: following.insert(0, ('cursor', Long)) body_struct = Struct(following) - body_class, body_buffer = body_struct.parse(conn) - body = body_class.from_buffer_copy(body_buffer) - buffer += body_buffer + body_class, positions = body_struct.parse(stream) + body = body_class.from_buffer_copy(stream.mem_view(*positions)) if self.include_field_names: field_count = body.fields.length @@ -153,9 +156,8 @@ def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): for i in range(body.row_count): row_fields = [] for j in range(field_count): - field_class, field_buffer = AnyDataObject.parse(conn) + field_class, field_positions = AnyDataObject.parse(stream) row_fields.append(('column_{}'.format(j), field_class)) - buffer += field_buffer row_class = type( 'SQLResponseRow', @@ -180,7 +182,7 @@ def _parse_success(self, conn: Connection, buffer: bytearray, fields: list): ('more', ctypes.c_byte), ] - def _create_parse_result(self, stream, header_class, fields: list, buffer: bytearray): + def _create_response_class(self, stream, header_class, fields: list): final_class = type( 'SQLResponse', (header_class,), @@ -189,8 +191,7 @@ def _create_parse_result(self, stream, header_class, fields: list, buffer: bytea '_fields_': fields, } ) - buffer += stream.read(ctypes.sizeof(final_class) - len(buffer)) - return final_class, buffer + return final_class def to_python(self, ctype_object, *args, **kwargs): if getattr(ctype_object, 'status_code', 0) == 0: diff --git a/pyignite/stream/binary_stream.py b/pyignite/stream/binary_stream.py index 25c2c15..b192303 100644 --- a/pyignite/stream/binary_stream.py +++ b/pyignite/stream/binary_stream.py @@ -48,6 +48,9 @@ def seek(self, *args, **kwargs): def getvalue(self): return self.stream.getvalue() + def mem_view(self, start, offset): + return self.stream.getbuffer()[start:start+offset] + def hashcode(self, start, bytes_len): return ignite_utils.hashcode(self.stream.getbuffer()[start:start+bytes_len]) diff --git a/pyignite/utils.py b/pyignite/utils.py index 3ba5a5f..79e2dc8 100644 --- a/pyignite/utils.py +++ b/pyignite/utils.py @@ -97,11 +97,11 @@ def unwrap_binary(client: 'Client', wrapped: tuple) -> object: blob, offset = wrapped with BinaryStream(blob, client.random_node) as stream: - data_class, data_bytes = BinaryObject.parse(stream) - result = BinaryObject.to_python( - data_class.from_buffer_copy(data_bytes), - client, - ) + data_class, data_positions = BinaryObject.parse(stream) + result = BinaryObject.to_python( + data_class.from_buffer_copy(stream.mem_view(*data_positions)), + client, + ) return result diff --git a/tests/config/ignite-config.xml.jinja2 b/tests/config/ignite-config.xml.jinja2 index 4191f3f..322a958 100644 --- a/tests/config/ignite-config.xml.jinja2 +++ b/tests/config/ignite-config.xml.jinja2 @@ -93,7 +93,7 @@ - + diff --git a/tests/config/log4j.xml.jinja2 b/tests/config/log4j.xml.jinja2 index 99ce765..1f1008f 100644 --- a/tests/config/log4j.xml.jinja2 +++ b/tests/config/log4j.xml.jinja2 @@ -17,40 +17,26 @@ limitations under the License. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_affinity_request_routing.py b/tests/test_affinity_request_routing.py index fd06e5f..cd0c015 100644 --- a/tests/test_affinity_request_routing.py +++ b/tests/test_affinity_request_routing.py @@ -48,11 +48,11 @@ def wait_for_affinity_distribution(cache, key, node_idx, timeout=30): def check_grid_idx(): nonlocal real_node_idx - cache.get(key) - # conn = cache.get_best_node(key) - # real_node_idx = conn.port % 100 - # return real_node_idx == node_idx - real_node_idx = requests.pop() + try: + cache.get(key) + real_node_idx = requests.pop() + except (OSError, IOError): + return False return real_node_idx == node_idx res = wait_for_condition(check_grid_idx, timeout=timeout) diff --git a/tests/test_cache_composite_key_class_sql.py b/tests/test_cache_composite_key_class_sql.py index df1cac3..2f1705f 100644 --- a/tests/test_cache_composite_key_class_sql.py +++ b/tests/test_cache_composite_key_class_sql.py @@ -117,7 +117,7 @@ def validate_query_result(student_key, student_val, query_result): assert len(query_result) == 2 sql_row = dict(zip(query_result[0], query_result[1])) - #assert sql_row["_KEY"][0] == student_key._buffer + assert sql_row["_KEY"][0] == student_key._buffer assert sql_row['ID'] == student_key.ID assert sql_row['DEPT'] == student_key.DEPT assert sql_row['NAME'] == student_val.NAME From 3bd079aa0c45b99125e983f665dedcc9dca5c5be Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Fri, 5 Feb 2021 19:54:28 +0300 Subject: [PATCH 5/8] IGNITE-13967 Fix python 3.6 --- pyignite/cache.py | 4 ++-- pyignite/datatypes/__init__.py | 23 ++++++++++++++++++++++- pyignite/utils.py | 24 ++---------------------- tests/test_sql.py | 4 +++- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/pyignite/cache.py b/pyignite/cache.py index 64093e8..dd7dac4 100644 --- a/pyignite/cache.py +++ b/pyignite/cache.py @@ -17,7 +17,7 @@ from typing import Any, Dict, Iterable, Optional, Tuple, Union from .constants import * -from .binary import GenericObjectMeta +from .binary import GenericObjectMeta, unwrap_binary from .datatypes import prop_codes from .datatypes.internal import AnyDataObject from .exceptions import ( @@ -26,7 +26,7 @@ ) from .utils import ( cache_id, get_field_by_id, is_wrapped, - status_to_exception, unsigned, unwrap_binary, + status_to_exception, unsigned ) from .api.cache_config import ( cache_create, cache_create_with_config, diff --git a/pyignite/datatypes/__init__.py b/pyignite/datatypes/__init__.py index 87f8c6b..553c3f2 100644 --- a/pyignite/datatypes/__init__.py +++ b/pyignite/datatypes/__init__.py @@ -24,4 +24,25 @@ from .primitive import * from .primitive_arrays import * from .primitive_objects import * -from .standard import * \ No newline at end of file +from .standard import * +from ..stream import BinaryStream + + +def unwrap_binary(client: 'Client', wrapped: tuple) -> object: + """ + Unwrap wrapped BinaryObject and convert it to Python data. + + :param client: connection to Ignite cluster, + :param wrapped: `WrappedDataObject` value, + :return: dict representing wrapped BinaryObject. + """ + from pyignite.datatypes.complex import BinaryObject + + blob, offset = wrapped + with BinaryStream(blob, client.random_node) as stream: + data_class, data_positions = BinaryObject.parse(stream) + result = BinaryObject.to_python( + data_class.from_buffer_copy(stream.mem_view(*data_positions)), + client, + ) + return result \ No newline at end of file diff --git a/pyignite/utils.py b/pyignite/utils.py index 79e2dc8..3d0378f 100644 --- a/pyignite/utils.py +++ b/pyignite/utils.py @@ -19,11 +19,11 @@ from functools import wraps from threading import Event, Thread -from typing import Any, Callable, Optional, Type, Tuple, Union +from typing import Any, Optional, Type, Tuple, Union from pyignite.datatypes.base import IgniteDataType from .constants import * -from .stream import BinaryStream + LONG_MASK = 0xffffffff DIGITS_PER_INT = 9 @@ -85,26 +85,6 @@ def int_overflow(value: int) -> int: return ((value ^ 0x80000000) & 0xffffffff) - 0x80000000 -def unwrap_binary(client: 'Client', wrapped: tuple) -> object: - """ - Unwrap wrapped BinaryObject and convert it to Python data. - - :param client: connection to Ignite cluster, - :param wrapped: `WrappedDataObject` value, - :return: dict representing wrapped BinaryObject. - """ - from pyignite.datatypes.complex import BinaryObject - - blob, offset = wrapped - with BinaryStream(blob, client.random_node) as stream: - data_class, data_positions = BinaryObject.parse(stream) - result = BinaryObject.to_python( - data_class.from_buffer_copy(stream.mem_view(*data_positions)), - client, - ) - return result - - def hashcode(data: Union[str, bytes, bytearray, memoryview]) -> int: """ Calculate hash code used for identifying objects in Ignite binary API. diff --git a/tests/test_sql.py b/tests/test_sql.py index 15f84ee..c896afb 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -22,7 +22,9 @@ ) from pyignite.datatypes.prop_codes import * from pyignite.exceptions import SQLError -from pyignite.utils import entity_id, unwrap_binary +from pyignite.utils import entity_id +from pyignite.binary import unwrap_binary + initial_data = [ ('John', 'Doe', 5), From 5ce8d75f6ec0c004d113ac3dcc789a0779b531d7 Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Sun, 7 Feb 2021 15:45:15 +0300 Subject: [PATCH 6/8] IGNITE-13967 Refactor, optimize allocations. --- pyignite/api/affinity.py | 8 +- pyignite/api/binary.py | 21 +-- pyignite/binary.py | 6 +- pyignite/connection/__init__.py | 12 +- pyignite/datatypes/__init__.py | 12 +- pyignite/datatypes/cache_properties.py | 6 +- pyignite/datatypes/complex.py | 129 ++++---------- pyignite/datatypes/internal.py | 43 ++--- pyignite/datatypes/null_object.py | 44 ++++- pyignite/datatypes/primitive.py | 2 +- pyignite/datatypes/primitive_arrays.py | 32 +--- pyignite/datatypes/primitive_objects.py | 32 +--- pyignite/datatypes/standard.py | 187 +++++--------------- pyignite/queries/query.py | 8 +- pyignite/queries/response.py | 18 +- pyignite/stream/__init__.py | 2 +- pyignite/stream/binary_stream.py | 50 +++++- tests/test_cache_composite_key_class_sql.py | 5 +- 18 files changed, 254 insertions(+), 363 deletions(-) diff --git a/pyignite/api/affinity.py b/pyignite/api/affinity.py index 584f73c..16148a1 100644 --- a/pyignite/api/affinity.py +++ b/pyignite/api/affinity.py @@ -55,12 +55,12 @@ partition_mapping = StructArray([ ('is_applicable', Bool), - ('cache_mapping', Conditional(lambda stream, ctx: stream.mem_view(*ctx['is_applicable']) == b'\x01', - lambda ctx: ctx['is_applicable'] is True, + ('cache_mapping', Conditional(lambda ctx: ctx['is_applicable'] and ctx['is_applicable'].value == 1, + lambda ctx: ctx['is_applicable'], cache_mapping, empty_cache_mapping)), - ('node_mapping', Conditional(lambda stream, ctx: stream.mem_view(*ctx['is_applicable']) == b'\x01', - lambda ctx: ctx['is_applicable'] is True, + ('node_mapping', Conditional(lambda ctx: ctx['is_applicable'] and ctx['is_applicable'].value == 1, + lambda ctx: ctx['is_applicable'], node_mapping, empty_node_mapping)), ]) diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py index ff85856..faf01a0 100644 --- a/pyignite/api/binary.py +++ b/pyignite/api/binary.py @@ -24,7 +24,7 @@ from pyignite.queries.op_codes import * from pyignite.utils import int_overflow, entity_id from .result import APIResult -from ..stream import BinaryStream +from ..stream import BinaryStream, READ_BACKWARD from ..queries.response import Response @@ -57,24 +57,21 @@ def get_binary_type(conn: 'Connection', binary_type: Union[str, int], query_id=N following=[('type_exists', Bool)]) with BinaryStream(conn.recv(), conn) as stream: - response_head_type, response_positions = response_head_struct.parse(stream) - response_head = response_head_type.from_buffer_copy(stream.mem_view(*response_positions)) - init_pos, total_len = response_positions + init_pos = stream.tell() + response_head_type = response_head_struct.parse(stream) + response_head = stream.read_ctype(response_head_type, direction=READ_BACKWARD) response_parts = [] if response_head.type_exists: - resp_body_type, resp_body_positions = body_struct.parse(stream) + resp_body_type = body_struct.parse(stream) response_parts.append(('body', resp_body_type)) - resp_body = resp_body_type.from_buffer_copy(stream.mem_view(*resp_body_positions)) - total_len += resp_body_positions[1] + resp_body = stream.read_ctype(resp_body_type, direction=READ_BACKWARD) if resp_body.is_enum: - resp_enum, resp_enum_positions = enum_struct.parse(stream) + resp_enum = enum_struct.parse(stream) response_parts.append(('enums', resp_enum)) - total_len += resp_enum_positions[1] - resp_schema_type, resp_schema_positions = schema_struct.parse(stream) + resp_schema_type = schema_struct.parse(stream) response_parts.append(('schema', resp_schema_type)) - total_len += resp_schema_positions[1] response_class = type( 'GetBinaryTypeResponse', @@ -84,7 +81,7 @@ def get_binary_type(conn: 'Connection', binary_type: Union[str, int], query_id=N '_fields_': response_parts, } ) - response = response_class.from_buffer_copy(stream.mem_view(init_pos, total_len)) + response = stream.read_ctype(response_class, position=init_pos) result = APIResult(response) if result.status != 0: diff --git a/pyignite/binary.py b/pyignite/binary.py index dfc18ef..da62bb5 100644 --- a/pyignite/binary.py +++ b/pyignite/binary.py @@ -102,13 +102,14 @@ def __new__( mcs, name, (GenericObjectProps, )+base_classes, namespace ) - def _from_python(self, stream): + def _from_python(self, stream, save_to_buf=False): """ Method for building binary representation of the Generic object and calculating a hashcode from it. :param self: Generic object instance, :param stream: BinaryStream + :param save_to_buf: Optional. If True, save serialized data to buffer. """ compact_footer = stream.compact_footer @@ -169,7 +170,8 @@ def _from_python(self, stream): stream.seek(initial_pos + header.schema_offset) stream.write(schema) - self._buffer = bytes(stream.mem_view(initial_pos, stream.tell() - initial_pos)) + if save_to_buf: + self._buffer = bytes(stream.mem_view(initial_pos, stream.tell() - initial_pos)) self._hashcode = header.hash_code def _setattr(self, attr_name: str, attr_value: Any): diff --git a/pyignite/connection/__init__.py b/pyignite/connection/__init__.py index 2389f47..ea82d2d 100644 --- a/pyignite/connection/__init__.py +++ b/pyignite/connection/__init__.py @@ -52,7 +52,7 @@ __all__ = ['Connection'] -from ..stream import BinaryStream +from ..stream import BinaryStream, READ_BACKWARD class Connection: @@ -206,8 +206,8 @@ def read_response(self) -> Union[dict, OrderedDict]: ('op_code', Byte), ]) with BinaryStream(self.recv(), self) as stream: - start_class, start_positions = response_start.parse(stream) - start = start_class.from_buffer_copy(stream.mem_view(*start_positions)) + start_class = response_start.parse(stream) + start = stream.read_ctype(start_class, direction=READ_BACKWARD) data = response_start.to_python(start) response_end = None if data['op_code'] == 0: @@ -222,8 +222,8 @@ def read_response(self) -> Union[dict, OrderedDict]: ('node_uuid', UUIDObject), ]) if response_end: - end_class, end_positions = response_end.parse(stream) - end = end_class.from_buffer_copy(stream.mem_view(*end_positions)) + end_class = response_end.parse(stream) + end = stream.read_ctype(end_class, direction=READ_BACKWARD) data.update(response_end.to_python(end)) return data @@ -295,7 +295,7 @@ def _connect_version( self.password ) - with BinaryStream(None, self) as stream: + with BinaryStream(self) as stream: hs_request.from_python(stream) self.send(stream.getvalue()) diff --git a/pyignite/datatypes/__init__.py b/pyignite/datatypes/__init__.py index 553c3f2..179cbc3 100644 --- a/pyignite/datatypes/__init__.py +++ b/pyignite/datatypes/__init__.py @@ -25,7 +25,7 @@ from .primitive_arrays import * from .primitive_objects import * from .standard import * -from ..stream import BinaryStream +from ..stream import BinaryStream, READ_BACKWARD def unwrap_binary(client: 'Client', wrapped: tuple) -> object: @@ -40,9 +40,7 @@ def unwrap_binary(client: 'Client', wrapped: tuple) -> object: blob, offset = wrapped with BinaryStream(blob, client.random_node) as stream: - data_class, data_positions = BinaryObject.parse(stream) - result = BinaryObject.to_python( - data_class.from_buffer_copy(stream.mem_view(*data_positions)), - client, - ) - return result \ No newline at end of file + data_class = BinaryObject.parse(stream) + result = BinaryObject.to_python(stream.read_ctype(data_class, direction=READ_BACKWARD), client) + + return result diff --git a/pyignite/datatypes/cache_properties.py b/pyignite/datatypes/cache_properties.py index 2443577..eadaef9 100644 --- a/pyignite/datatypes/cache_properties.py +++ b/pyignite/datatypes/cache_properties.py @@ -94,10 +94,8 @@ def build_header(cls): @classmethod def parse(cls, stream): init_pos = stream.tell() - header_class = cls.build_header() - header_len = ctypes.sizeof(header_class) - data_class, data_buf = cls.prop_data_class.parse(stream.mem_view(init_pos, header_len)) + data_class = cls.prop_data_class.parse(stream) prop_class = type( cls.__name__, @@ -111,7 +109,7 @@ def parse(cls, stream): ) stream.seek(init_pos + ctypes.sizeof(prop_class)) - return prop_class, (init_pos, stream.tell() - init_pos) + return prop_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py index 971408c..aed3cda 100644 --- a/pyignite/datatypes/complex.py +++ b/pyignite/datatypes/complex.py @@ -25,8 +25,7 @@ from .type_codes import * from .type_ids import * from .type_names import * -from .null_object import Null - +from .null_object import Null, Nullable __all__ = [ 'Map', 'ObjectArrayObject', 'CollectionObject', 'MapObject', @@ -36,7 +35,7 @@ from ..stream import BinaryStream -class ObjectArrayObject(IgniteDataType): +class ObjectArrayObject(IgniteDataType, Nullable): """ Array of Ignite objects of any consistent type. Its Python representation is tuple(type_id, iterable of any type). The only type ID that makes sense @@ -70,21 +69,14 @@ def build_header(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_header() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) - stream.seek(init_pos + header_len) + header = stream.read_ctype(header_class) + stream.seek(ctypes.sizeof(header_class), SEEK_CUR) fields = [] for i in range(header.length): - c_type, _ = AnyDataObject.parse(stream) + c_type = AnyDataObject.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -96,16 +88,12 @@ def parse(cls, stream): } ) - stream.seek(init_pos + ctypes.sizeof(final_class)) - return final_class, (init_pos, stream.tell() - init_pos) + return final_class @classmethod - def to_python(cls, ctype_object, *args, **kwargs): + def to_python_not_null(cls, ctype_object, *args, **kwargs): result = [] - length = getattr(ctype_object, "length", None) - if length is None: - return None - for i in range(length): + for i in range(ctype_object.length): result.append( AnyDataObject.to_python( getattr(ctype_object, 'element_{}'.format(i)), @@ -115,11 +103,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return ctype_object.type_id, result @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): type_or_id, value = value header_class = cls.build_header() header = header_class() @@ -140,7 +124,7 @@ def from_python(cls, stream, value): infer_from_python(stream, x) -class WrappedDataObject(IgniteDataType): +class WrappedDataObject(IgniteDataType, Nullable): """ One or more binary objects can be wrapped in an array. This allows reading, storing, passing and writing objects efficiently without understanding @@ -166,17 +150,9 @@ def build_header(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_header() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) - stream.seek(init_pos + header_len) + header = stream.read_ctype(header_class) final_class = type( cls.__name__, @@ -190,8 +166,8 @@ def parse(cls, stream): } ) - stream.seek(init_pos + ctypes.sizeof(final_class)) - return final_class, (init_pos, stream.tell() - init_pos) + stream.seek(ctypes.sizeof(final_class), SEEK_CUR) + return final_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -202,7 +178,7 @@ def from_python(cls, stream, value): raise ParseError('Send unwrapped data.') -class CollectionObject(IgniteDataType): +class CollectionObject(IgniteDataType, Nullable): """ Similar to object array, but contains platform-agnostic deserialization type hint instead of type ID. @@ -265,22 +241,14 @@ def build_header(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - type_ = stream.mem_view(init_pos, type_len) - if type_ == TC_NULL: - stream.seek(type_len, SEEK_CUR) - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_header() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) - stream.seek(init_pos + header_len) + header = stream.read_ctype(header_class) + stream.seek(ctypes.sizeof(header_class), SEEK_CUR) fields = [] for i in range(header.length): - c_type, _ = AnyDataObject.parse(stream) + c_type = AnyDataObject.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -291,7 +259,7 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, (init_pos, stream.tell() - init_pos) + return final_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -309,11 +277,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return ctype_object.type, result @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): type_or_id, value = value header_class = cls.build_header() header = header_class() @@ -334,7 +298,7 @@ def from_python(cls, stream, value): infer_from_python(stream, x) -class Map(IgniteDataType): +class Map(IgniteDataType, Nullable): """ Dictionary type, payload-only. @@ -365,21 +329,14 @@ def build_header(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_header() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) - stream.seek(init_pos + header_len) + header = stream.read_ctype(header_class) + stream.seek(ctypes.sizeof(header_class), SEEK_CUR) fields = [] for i in range(header.length << 1): - c_type, _ = AnyDataObject.parse(stream) + c_type = AnyDataObject.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -390,7 +347,7 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, (init_pos, stream.tell() - init_pos) + return final_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -478,7 +435,7 @@ def from_python(cls, stream, value): super().from_python(stream, value, type_id) -class BinaryObject(IgniteDataType): +class BinaryObject(IgniteDataType, Nullable): _type_id = TYPE_BINARY_OBJ type_code = TC_COMPLEX_OBJECT @@ -496,8 +453,8 @@ def hashcode(value: object, client: None) -> int: # its hashcode if not value._hashcode and client : - with BinaryStream(None, client.random_node) as stream: - value._from_python(stream) + with BinaryStream(client.random_node) as stream: + value._from_python(stream, save_to_buf=True) return value._hashcode @@ -546,27 +503,19 @@ def schema_type(cls, flags: int): ) @classmethod - def parse(cls, stream): + def parse_not_null(cls, stream): from pyignite.datatypes import Struct - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - type_ = stream.mem_view(init_pos, type_len) - if type_ == TC_NULL: - stream.seek(type_len, SEEK_CUR) - return Null.build_c_type(), (init_pos, type_len) - header_class = cls.build_header() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) - stream.seek(init_pos + header_len) + header = stream.read_ctype(header_class) + stream.seek(ctypes.sizeof(header_class), SEEK_CUR) # ignore full schema, always retrieve fields' types and order # from complex types registry data_class = stream.get_dataclass(header) fields = data_class.schema.items() object_fields_struct = Struct(fields) - object_fields, _ = object_fields_struct.parse(stream) + object_fields = object_fields_struct.parse(stream) final_class_fields = [('object_fields', object_fields)] if header.flags & cls.HAS_SCHEMA: @@ -584,7 +533,7 @@ def parse(cls, stream): ) # register schema encoding approach stream.compact_footer = bool(header.flags & cls.COMPACT_FOOTER) - return final_class, (init_pos, stream.tell() - init_pos) + return final_class @classmethod def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs): @@ -614,11 +563,7 @@ def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs): return result @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): stream.register_binary_type(value.__class__) if getattr(value, '_buffer', None): stream.write(value._buffer) diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py index 139a55b..0111a22 100644 --- a/pyignite/datatypes/internal.py +++ b/pyignite/datatypes/internal.py @@ -34,6 +34,8 @@ 'infer_from_python', ] +from ..stream import READ_BACKWARD + def tc_map(key: bytes, _memo_map: dict = {}): """ @@ -114,14 +116,14 @@ def tc_map(key: bytes, _memo_map: dict = {}): class Conditional: - def __init__(self, predicate1: Callable[['BinaryStream', any], bool], predicate2: Callable[[any], bool], var1, var2): + def __init__(self, predicate1: Callable[[any], bool], predicate2: Callable[[any], bool], var1, var2): self.predicate1 = predicate1 self.predicate2 = predicate2 self.var1 = var1 self.var2 = var2 def parse(self, stream, context): - return self.var1.parse(stream) if self.predicate1(stream, context) else self.var2.parse(stream) + return self.var1.parse(stream) if self.predicate1(context) else self.var2.parse(stream) def to_python(self, ctype_object, context, *args, **kwargs): return self.var1.to_python(ctype_object, *args, **kwargs) if self.predicate2(context)\ @@ -147,13 +149,16 @@ def build_header_class(self): ) def parse(self, stream): - init_pos = stream.tell() - buffer = stream.read(ctypes.sizeof(self.counter_type)) - length = int.from_bytes(buffer, byteorder=PROTOCOL_BYTE_ORDER) - fields = [] + counter_type_len = ctypes.sizeof(self.counter_type) + length = int.from_bytes( + stream.mem_view(offset=counter_type_len), + byteorder=PROTOCOL_BYTE_ORDER + ) + stream.seek(counter_type_len, SEEK_CUR) + fields = [] for i in range(length): - c_type, _ = Struct(self.following).parse(stream) + c_type = Struct(self.following).parse(stream) fields.append(('element_{}'.format(i), c_type)) data_class = type( @@ -165,7 +170,7 @@ def parse(self, stream): }, ) - return data_class, (init_pos, stream.tell() - init_pos) + return data_class def to_python(self, ctype_object, *args, **kwargs): result = [] @@ -204,14 +209,12 @@ class Struct: defaults = attr.ib(type=dict, default={}) def parse(self, stream): - init_pos = stream.tell() - fields, values = [], {} for name, c_type in self.fields: is_cond = isinstance(c_type, Conditional) - c_type, value_position = c_type.parse(stream, values) if is_cond else c_type.parse(stream) + c_type = c_type.parse(stream, values) if is_cond else c_type.parse(stream) fields.append((name, c_type)) - values[name] = value_position + values[name] = stream.read_ctype(c_type, direction=READ_BACKWARD) data_class = type( 'Struct', @@ -222,7 +225,7 @@ def parse(self, stream): }, ) - return data_class, (init_pos, stream.tell() - init_pos) + return data_class def to_python( self, ctype_object, *args, **kwargs @@ -291,12 +294,11 @@ def get_subtype(iterable, allow_none=False): @classmethod def parse(cls, stream): - type_code = bytes(stream.read(ctypes.sizeof(ctypes.c_byte))) + type_code = bytes(stream.mem_view(offset=ctypes.sizeof(ctypes.c_byte))) try: - data_class = tc_map(bytes(type_code)) + data_class = tc_map(type_code) except KeyError: raise ParseError('Unknown type code: `{}`'.format(type_code)) - stream.seek(-ctypes.sizeof(ctypes.c_byte), SEEK_CUR) return data_class.parse(stream) @classmethod @@ -450,13 +452,12 @@ def build_header(self): def parse(self, stream): header_class = self.build_header() - header_len, initial_pos = ctypes.sizeof(header_class), stream.tell() - header = header_class.from_buffer_copy(stream.mem_view(initial_pos, header_len)) - stream.seek(header_len, SEEK_CUR) + header = stream.read_ctype(header_class) + stream.seek(ctypes.sizeof(header_class), SEEK_CUR) fields = [] for i in range(header.length): - c_type, _ = super().parse(stream) + c_type = super().parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -467,7 +468,7 @@ def parse(self, stream): '_fields_': fields, } ) - return final_class, (initial_pos, stream.tell() - initial_pos) + return final_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/null_object.py b/pyignite/datatypes/null_object.py index 4db0c9f..912ded8 100644 --- a/pyignite/datatypes/null_object.py +++ b/pyignite/datatypes/null_object.py @@ -29,6 +29,8 @@ __all__ = ['Null'] +from ..constants import PROTOCOL_BYTE_ORDER + class Null(IgniteDataType): default = None @@ -59,7 +61,7 @@ def build_c_type(cls): def parse(cls, stream): init_pos, offset = stream.tell(), ctypes.sizeof(ctypes.c_byte) stream.seek(offset, SEEK_CUR) - return cls.build_c_type(), (init_pos, offset) + return cls.build_c_type() @staticmethod def to_python(*args, **kwargs): @@ -69,3 +71,43 @@ def to_python(*args, **kwargs): def from_python(stream, *args): stream.write(TC_NULL) + +class Nullable: + @classmethod + def parse_not_null(cls, stream): + raise NotImplementedError + + @classmethod + def parse(cls, stream): + type_len = ctypes.sizeof(ctypes.c_byte) + + if stream.mem_view(offset=type_len) == TC_NULL: + stream.seek(type_len, SEEK_CUR) + return Null.build_c_type() + + return cls.parse_not_null(stream) + + @classmethod + def to_python_not_null(cls, ctypes_object, *args, **kwargs): + raise NotImplementedError + + @classmethod + def to_python(cls, ctypes_object, *args, **kwargs): + if ctypes_object.type_code == int.from_bytes( + TC_NULL, + byteorder=PROTOCOL_BYTE_ORDER + ): + return None + + return cls.to_python_not_null(ctypes_object, *args, **kwargs) + + @classmethod + def from_python_not_null(cls, stream, value): + raise NotImplementedError + + @classmethod + def from_python(cls, stream, value): + if value is None: + Null.from_python(stream) + else: + cls.from_python_not_null(stream, value) diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py index dde49ff..ffa2e32 100644 --- a/pyignite/datatypes/primitive.py +++ b/pyignite/datatypes/primitive.py @@ -50,7 +50,7 @@ class Primitive(IgniteDataType): def parse(cls, stream): init_pos, offset = stream.tell(), ctypes.sizeof(cls.c_type) stream.seek(offset, SEEK_CUR) - return cls.c_type, (init_pos, offset) + return cls.c_type @classmethod def to_python(cls, ctype_object, *args, **kwargs): diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py index 65ed9c3..7cb5b20 100644 --- a/pyignite/datatypes/primitive_arrays.py +++ b/pyignite/datatypes/primitive_arrays.py @@ -14,11 +14,13 @@ # limitations under the License. import ctypes +from io import SEEK_CUR from typing import Any from pyignite.constants import * from . import Null from .base import IgniteDataType +from .null_object import Nullable from .primitive import * from .type_codes import * from .type_ids import * @@ -33,7 +35,7 @@ ] -class PrimitiveArray(IgniteDataType): +class PrimitiveArray(IgniteDataType, Nullable): """ Base class for array of primitives. Payload-only. """ @@ -61,16 +63,9 @@ def build_header_class(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_header_class() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + header = stream.read_ctype(header_class) final_class = type( cls.__name__, @@ -82,9 +77,8 @@ def parse(cls, stream): ], } ) - data_len = ctypes.sizeof(final_class) - stream.seek(init_pos + data_len) - return final_class, (init_pos, data_len) + stream.seek(ctypes.sizeof(final_class), SEEK_CUR) + return final_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): @@ -94,11 +88,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return [ctype_object.data[i] for i in range(ctype_object.length)] @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): header_class = cls.build_header_class() header = header_class() if hasattr(header, 'type_code'): @@ -221,11 +211,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return ByteArray.to_python(ctype_object, *args, **kwargs) @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): header_class = cls.build_header_class() header = header_class() header.type_code = int.from_bytes( diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py index b4ea7e6..e942dd7 100644 --- a/pyignite/datatypes/primitive_objects.py +++ b/pyignite/datatypes/primitive_objects.py @@ -14,6 +14,7 @@ # limitations under the License. import ctypes +from io import SEEK_CUR from pyignite.constants import * from pyignite.utils import unsigned @@ -21,8 +22,7 @@ from .type_codes import * from .type_ids import * from .type_names import * -from .null_object import Null - +from .null_object import Null, Nullable __all__ = [ 'DataObject', 'ByteObject', 'ShortObject', 'IntObject', 'LongObject', @@ -30,7 +30,7 @@ ] -class DataObject(IgniteDataType): +class DataObject(IgniteDataType, Nullable): """ Base class for primitive data objects. @@ -60,29 +60,17 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): data_type = cls.build_c_type() - data_len = ctypes.sizeof(data_type) - stream.seek(init_pos + data_len) - - return data_type, (init_pos, data_len) + stream.seek(ctypes.sizeof(data_type), SEEK_CUR) + return data_type @staticmethod def to_python(ctype_object, *args, **kwargs): return getattr(ctype_object, "value", None) @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): data_type = cls.build_c_type() data_object = data_type() data_object.type_code = int.from_bytes( @@ -207,11 +195,7 @@ def to_python(cls, ctype_object, *args, **kwargs): ).decode(PROTOCOL_CHAR_ENCODING) @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): if type(value) is str: value = value.encode(PROTOCOL_CHAR_ENCODING) # assuming either a bytes or an integer diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py index 6b33b82..af50a8e 100644 --- a/pyignite/datatypes/standard.py +++ b/pyignite/datatypes/standard.py @@ -27,8 +27,7 @@ from .type_codes import * from .type_ids import * from .type_names import * -from .null_object import Null - +from .null_object import Null, Nullable __all__ = [ 'String', 'DecimalObject', 'UUIDObject', 'TimestampObject', 'DateObject', @@ -45,7 +44,7 @@ ] -class StandardObject(IgniteDataType): +class StandardObject(IgniteDataType, Nullable): _type_name = None _type_id = None type_code = None @@ -55,21 +54,13 @@ def build_c_type(cls): raise NotImplementedError('This object is generic') @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): data_type = cls.build_c_type() - data_len = ctypes.sizeof(data_type) - stream.seek(init_pos + data_len) + stream.seek(ctypes.sizeof(data_type), SEEK_CUR) + return data_type - return data_type, (init_pos, data_len) - -class String(IgniteDataType): +class String(IgniteDataType, Nullable): """ Pascal-style string: `c_int` counter, followed by count*bytes. UTF-8-encoded, so that one character may take 1 to 4 bytes. @@ -99,40 +90,25 @@ def build_c_type(cls, length: int): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - buffer = stream.read(type_len) - if buffer == TC_NULL: - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): length = int.from_bytes( - stream.read(ctypes.sizeof(ctypes.c_int)), + stream.mem_view(stream.tell() + ctypes.sizeof(ctypes.c_byte), ctypes.sizeof(ctypes.c_int)), byteorder=PROTOCOL_BYTE_ORDER ) data_type = cls.build_c_type(length) - data_len = ctypes.sizeof(data_type) - stream.seek(init_pos + data_len) + stream.seek(ctypes.sizeof(data_type), SEEK_CUR) + return data_type - return data_type, (init_pos, data_len) - - @staticmethod - def to_python(ctype_object, *args, **kwargs): - length = getattr(ctype_object, 'length', None) - if length is None: - return None - elif length > 0: + @classmethod + def to_python_not_null(cls, ctype_object, *args, **kwargs): + if ctype_object.length > 0: return ctype_object.data.decode(PROTOCOL_STRING_ENCODING) - else: - return '' - @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return + return '' + @classmethod + def from_python_not_null(cls, stream, value): if isinstance(value, str): value = value.encode(PROTOCOL_STRING_ENCODING) length = len(value) @@ -148,7 +124,7 @@ def from_python(cls, stream, value): stream.write(data_object) -class DecimalObject(IgniteDataType): +class DecimalObject(IgniteDataType, Nullable): _type_name = NAME_DECIMAL _type_id = TYPE_DECIMAL type_code = TC_DECIMAL @@ -175,17 +151,9 @@ def build_c_header(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - type_ = stream.mem_view(init_pos, type_len) - if type_ == TC_NULL: - stream.seek(type_len, SEEK_CUR) - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_c_header() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + header = stream.read_ctype(header_class) data_type = type( cls.__name__, @@ -198,16 +166,11 @@ def parse(cls, stream): } ) - data_len = ctypes.sizeof(data_type) - stream.seek(init_pos + data_len) - - return data_type, (init_pos, data_len) + stream.seek(ctypes.sizeof(data_type), SEEK_CUR) + return data_type @classmethod - def to_python(cls, ctype_object, *args, **kwargs): - if getattr(ctype_object, 'length', None) is None: - return None - + def to_python_not_null(cls, ctype_object, *args, **kwargs): sign = 1 if ctype_object.data[0] & 0x80 else 0 data = ctype_object.data[1:] data.insert(0, ctype_object.data[0] & 0x7f) @@ -228,11 +191,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return result @classmethod - def from_python(cls, stream, value: decimal.Decimal): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value: decimal.Decimal): sign, digits, scale = value.normalize().as_tuple() integer = int(''.join([str(d) for d in digits])) # calculate number of bytes (at least one, and not forget the sign bit) @@ -312,11 +271,7 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def from_python(cls, stream, value: uuid.UUID): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value: uuid.UUID): data_type = cls.build_c_type() data_object = data_type() data_object.type_code = int.from_bytes( @@ -329,12 +284,7 @@ def from_python(cls, stream, value: uuid.UUID): stream.write(data_object) @classmethod - def to_python(cls, ctypes_object, *args, **kwargs): - if ctypes_object.type_code == int.from_bytes( - TC_NULL, - byteorder=PROTOCOL_BYTE_ORDER - ): - return None + def to_python_not_null(cls, ctypes_object, *args, **kwargs): uuid_array = bytearray(ctypes_object.value) return uuid.UUID( bytes=bytes([uuid_array[i] for i in cls.UUID_BYTE_ORDER]) @@ -381,11 +331,7 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def from_python(cls, stream, value: tuple): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value: tuple): data_type = cls.build_c_type() data_object = data_type() data_object.type_code = int.from_bytes( @@ -398,12 +344,7 @@ def from_python(cls, stream, value: tuple): stream.write(data_object) @classmethod - def to_python(cls, ctypes_object, *args, **kwargs): - if ctypes_object.type_code == int.from_bytes( - TC_NULL, - byteorder=PROTOCOL_BYTE_ORDER - ): - return None + def to_python_not_null(cls, ctypes_object, *args, **kwargs): return ( datetime.fromtimestamp(ctypes_object.epoch/1000), ctypes_object.fraction @@ -445,11 +386,7 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def from_python(cls, stream, value: [date, datetime]): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value: [date, datetime]): if type(value) is date: value = datetime.combine(value, time()) data_type = cls.build_c_type() @@ -463,12 +400,7 @@ def from_python(cls, stream, value: [date, datetime]): stream.write(data_object) @classmethod - def to_python(cls, ctypes_object, *args, **kwargs): - if ctypes_object.type_code == int.from_bytes( - TC_NULL, - byteorder=PROTOCOL_BYTE_ORDER - ): - return None + def to_python_not_null(cls, ctypes_object, *args, **kwargs): return datetime.fromtimestamp(ctypes_object.epoch/1000) @@ -506,11 +438,7 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def from_python(cls, stream, value: timedelta): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value: timedelta): data_type = cls.build_c_type() data_object = data_type() data_object.type_code = int.from_bytes( @@ -522,12 +450,7 @@ def from_python(cls, stream, value: timedelta): stream.write(data_object) @classmethod - def to_python(cls, ctypes_object, *args, **kwargs): - if ctypes_object.type_code == int.from_bytes( - TC_NULL, - byteorder=PROTOCOL_BYTE_ORDER - ): - return None + def to_python_not_null(cls, ctypes_object, *args, **kwargs): return timedelta(milliseconds=ctypes_object.value) @@ -562,11 +485,7 @@ def build_c_type(cls): return cls._object_c_type @classmethod - def from_python(cls, stream, value: tuple): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value: tuple): data_type = cls.build_c_type() data_object = data_type() data_object.type_code = int.from_bytes( @@ -578,12 +497,7 @@ def from_python(cls, stream, value: tuple): stream.write(data_object) @classmethod - def to_python(cls, ctypes_object, *args, **kwargs): - if ctypes_object.type_code == int.from_bytes( - TC_NULL, - byteorder=PROTOCOL_BYTE_ORDER - ): - return None + def to_python_not_null(cls, ctypes_object, *args, **kwargs): return ctypes_object.type_id, ctypes_object.ordinal @@ -596,7 +510,7 @@ class BinaryEnumObject(EnumObject): type_code = TC_BINARY_ENUM -class StandardArray(IgniteDataType): +class StandardArray(IgniteDataType, Nullable): """ Base class for array of primitives. Payload-only. """ @@ -624,22 +538,14 @@ def build_header_class(cls): ) @classmethod - def parse(cls, stream): - init_pos, type_len = stream.tell(), ctypes.sizeof(ctypes.c_byte) - - type_ = stream.mem_view(init_pos, type_len) - if type_ == TC_NULL: - stream.seek(type_len, SEEK_CUR) - return Null.build_c_type(), (init_pos, type_len) - + def parse_not_null(cls, stream): header_class = cls.build_header_class() - header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) - stream.seek(init_pos + header_len) + header = stream.read_ctype(header_class) + stream.seek(ctypes.sizeof(header_class), SEEK_CUR) fields = [] for i in range(header.length): - c_type, _ = cls.standard_type.parse(stream) + c_type = cls.standard_type.parse(stream) fields.append(('element_{}'.format(i), c_type)) final_class = type( @@ -650,14 +556,15 @@ def parse(cls, stream): '_fields_': fields, } ) - return final_class, (init_pos, stream.tell() - init_pos) + return final_class @classmethod def to_python(cls, ctype_object, *args, **kwargs): - result = [] length = getattr(ctype_object, "length", None) if length is None: return None + + result = [] for i in range(length): result.append( cls.standard_type.to_python( @@ -668,11 +575,7 @@ def to_python(cls, ctype_object, *args, **kwargs): return result @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): header_class = cls.build_header_class() header = header_class() if hasattr(header, 'type_code'): @@ -833,11 +736,7 @@ def build_header_class(cls): ) @classmethod - def from_python(cls, stream, value): - if value is None: - Null.from_python(stream) - return - + def from_python_not_null(cls, stream, value): type_id, value = value header_class = cls.build_header_class() header = header_class() diff --git a/pyignite/queries/query.py b/pyignite/queries/query.py index 56d1ff1..57505c7 100644 --- a/pyignite/queries/query.py +++ b/pyignite/queries/query.py @@ -21,7 +21,7 @@ from pyignite.connection import Connection from pyignite.constants import MIN_LONG, MAX_LONG, RHF_TOPOLOGY_CHANGED from pyignite.queries.response import Response, SQLResponse -from pyignite.stream import BinaryStream +from pyignite.stream import BinaryStream, READ_BACKWARD @attr.s @@ -70,7 +70,7 @@ def _build_header(self, stream, values: dict): def from_python(self, conn: Connection, values: dict = None): if values is None: values = {} - stream = BinaryStream(None, conn) + stream = BinaryStream(conn) header = self._build_header(stream, values) stream.write(header) return stream.getvalue() @@ -102,8 +102,8 @@ def perform( following=response_config) with BinaryStream(conn.recv(), conn) as stream: - response_ctype, response_positions = response_struct.parse(stream) - response = response_ctype.from_buffer_copy(stream.mem_view(*response_positions)) + response_ctype = response_struct.parse(stream) + response = stream.read_ctype(response_ctype, direction=READ_BACKWARD) # this test depends on protocol version if getattr(response, 'flags', False) & RHF_TOPOLOGY_CHANGED: diff --git a/pyignite/queries/response.py b/pyignite/queries/response.py index 48f4adc..016f577 100644 --- a/pyignite/queries/response.py +++ b/pyignite/queries/response.py @@ -22,6 +22,7 @@ from pyignite.connection import Connection from pyignite.datatypes import AnyDataObject, Bool, Int, Long, String, StringArray, Struct from pyignite.queries.op_codes import OP_SUCCESS +from pyignite.stream import READ_BACKWARD @attr.s @@ -58,10 +59,9 @@ def build_header(self): def parse(self, stream): init_pos = stream.tell() - header_class = self.build_header() header_len = ctypes.sizeof(header_class) - header = header_class.from_buffer_copy(stream.mem_view(init_pos, header_len)) + header = stream.read_ctype(header_class) stream.seek(header_len, SEEK_CUR) fields = [] @@ -80,17 +80,17 @@ def parse(self, stream): has_error = header.status_code != OP_SUCCESS if fields: - stream.seek(sum([ctypes.sizeof(c_type) for _, c_type in fields]), SEEK_CUR) + stream.seek(sum(ctypes.sizeof(c_type) for _, c_type in fields), SEEK_CUR) if has_error: - msg_type, _ = String.parse(stream) + msg_type = String.parse(stream) fields.append(('error_message', msg_type)) else: self._parse_success(stream, fields) response_class = self._create_response_class(stream, header_class, fields) stream.seek(init_pos + ctypes.sizeof(response_class)) - return self._create_response_class(stream, header_class, fields), (init_pos, stream.tell() - init_pos) + return self._create_response_class(stream, header_class, fields) def _create_response_class(self, stream, header_class, fields: list): response_class = type( @@ -105,7 +105,7 @@ def _create_response_class(self, stream, header_class, fields: list): def _parse_success(self, stream, fields: list): for name, ignite_type in self.following: - c_type, _ = ignite_type.parse(stream) + c_type = ignite_type.parse(stream) fields.append((name, c_type)) def to_python(self, ctype_object, *args, **kwargs): @@ -144,8 +144,8 @@ def _parse_success(self, stream, fields: list): if self.has_cursor: following.insert(0, ('cursor', Long)) body_struct = Struct(following) - body_class, positions = body_struct.parse(stream) - body = body_class.from_buffer_copy(stream.mem_view(*positions)) + body_class = body_struct.parse(stream) + body = stream.read_ctype(body_class, direction=READ_BACKWARD) if self.include_field_names: field_count = body.fields.length @@ -156,7 +156,7 @@ def _parse_success(self, stream, fields: list): for i in range(body.row_count): row_fields = [] for j in range(field_count): - field_class, field_positions = AnyDataObject.parse(stream) + field_class = AnyDataObject.parse(stream) row_fields.append(('column_{}'.format(j), field_class)) row_class = type( diff --git a/pyignite/stream/__init__.py b/pyignite/stream/__init__.py index f5b931b..94153b4 100644 --- a/pyignite/stream/__init__.py +++ b/pyignite/stream/__init__.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .binary_stream import BinaryStream \ No newline at end of file +from .binary_stream import BinaryStream, READ_FORWARD, READ_BACKWARD \ No newline at end of file diff --git a/pyignite/stream/binary_stream.py b/pyignite/stream/binary_stream.py index b192303..48b4770 100644 --- a/pyignite/stream/binary_stream.py +++ b/pyignite/stream/binary_stream.py @@ -12,16 +12,39 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import ctypes from io import BytesIO import pyignite.utils as ignite_utils +READ_FORWARD = 0 +READ_BACKWARD = 1 + class BinaryStream: - def __init__(self, stream, conn): - self.stream = BytesIO(stream) if stream else BytesIO() - self.conn = conn + def __init__(self, *args): + """ + Initialize binary stream around buffers. + + :param buf: Buffer, optional parameter. If not passed, creates empty BytesIO. + :param conn: Connection instance, required. + """ + from pyignite.connection import Connection + + buf = None + if len(args) == 1: + self.conn = args[0] + elif len(args) == 2: + buf = args[0] + self.conn = args[1] + + if not isinstance(self.conn, Connection): + raise TypeError(f"invalid parameter: expected instance of {Connection}") + + if buf and not isinstance(buf, (bytearray, bytes, memoryview)): + raise TypeError(f"invalid parameter: expected bytes-like object") + + self.stream = BytesIO(buf) if buf else BytesIO() @property def compact_footer(self) -> bool: @@ -36,6 +59,22 @@ def read(self, size): self.stream.readinto(buf) return buf + def read_ctype(self, ctype_class, position=None, direction=READ_FORWARD): + ctype_len = ctypes.sizeof(ctype_class) + + if position is not None and position >= 0: + init_position = position + else: + init_position = self.tell() + + if direction == READ_FORWARD: + start, end = init_position, init_position + ctype_len + else: + start, end = init_position - ctype_len, init_position + + buf = self.stream.getbuffer()[start:end] + return ctype_class.from_buffer_copy(buf) + def write(self, buf): return self.stream.write(buf) @@ -48,7 +87,8 @@ def seek(self, *args, **kwargs): def getvalue(self): return self.stream.getvalue() - def mem_view(self, start, offset): + def mem_view(self, start=-1, offset=0): + start = start if start >= 0 else self.tell() return self.stream.getbuffer()[start:start+offset] def hashcode(self, start, bytes_len): diff --git a/tests/test_cache_composite_key_class_sql.py b/tests/test_cache_composite_key_class_sql.py index 2f1705f..989a229 100644 --- a/tests/test_cache_composite_key_class_sql.py +++ b/tests/test_cache_composite_key_class_sql.py @@ -111,13 +111,12 @@ def test_python_sql_finds_inserted_value_with_composite_key(client): def validate_query_result(student_key, student_val, query_result): - ''' + """ Compare query result with expected key and value. - ''' + """ assert len(query_result) == 2 sql_row = dict(zip(query_result[0], query_result[1])) - assert sql_row["_KEY"][0] == student_key._buffer assert sql_row['ID'] == student_key.ID assert sql_row['DEPT'] == student_key.DEPT assert sql_row['NAME'] == student_val.NAME From eb146ec1204055dac0d420451d281dd89d996bdc Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Mon, 8 Feb 2021 16:41:14 +0300 Subject: [PATCH 7/8] IGNITE-13967 Code review fixes. --- pyignite/api/binary.py | 11 ++++++----- pyignite/connection/__init__.py | 6 +++--- pyignite/datatypes/__init__.py | 2 +- pyignite/queries/query.py | 15 ++++++--------- pyignite/stream/binary_stream.py | 15 ++++++--------- 5 files changed, 22 insertions(+), 27 deletions(-) diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py index faf01a0..0e63c17 100644 --- a/pyignite/api/binary.py +++ b/pyignite/api/binary.py @@ -48,15 +48,16 @@ def get_binary_type(conn: 'Connection', binary_type: Union[str, int], query_id=N query_id=query_id, ) - send_buffer = query_struct.from_python(conn, { - 'type_id': entity_id(binary_type), - }) - conn.send(send_buffer) + with BinaryStream(conn) as stream: + query_struct.from_python(stream, { + 'type_id': entity_id(binary_type), + }) + conn.send(stream.getbuffer()) response_head_struct = Response(protocol_version=conn.get_protocol_version(), following=[('type_exists', Bool)]) - with BinaryStream(conn.recv(), conn) as stream: + with BinaryStream(conn, conn.recv()) as stream: init_pos = stream.tell() response_head_type = response_head_struct.parse(stream) response_head = stream.read_ctype(response_head_type, direction=READ_BACKWARD) diff --git a/pyignite/connection/__init__.py b/pyignite/connection/__init__.py index ea82d2d..0e793f8 100644 --- a/pyignite/connection/__init__.py +++ b/pyignite/connection/__init__.py @@ -205,7 +205,7 @@ def read_response(self) -> Union[dict, OrderedDict]: ('length', Int), ('op_code', Byte), ]) - with BinaryStream(self.recv(), self) as stream: + with BinaryStream(self, self.recv()) as stream: start_class = response_start.parse(stream) start = stream.read_ctype(start_class, direction=READ_BACKWARD) data = response_start.to_python(start) @@ -297,7 +297,7 @@ def _connect_version( with BinaryStream(self) as stream: hs_request.from_python(stream) - self.send(stream.getvalue()) + self.send(stream.getbuffer()) hs_response = self.read_response() if hs_response['op_code'] == 0: @@ -374,7 +374,7 @@ def _transfer_params(self, to: 'Connection'): to.host = self.host to.port = self.port - def send(self, data: bytes, flags=None): + def send(self, data: Union[bytes, bytearray, memoryview], flags=None): """ Send data down the socket. diff --git a/pyignite/datatypes/__init__.py b/pyignite/datatypes/__init__.py index 179cbc3..49860bd 100644 --- a/pyignite/datatypes/__init__.py +++ b/pyignite/datatypes/__init__.py @@ -39,7 +39,7 @@ def unwrap_binary(client: 'Client', wrapped: tuple) -> object: from pyignite.datatypes.complex import BinaryObject blob, offset = wrapped - with BinaryStream(blob, client.random_node) as stream: + with BinaryStream(client.random_node, blob) as stream: data_class = BinaryObject.parse(stream) result = BinaryObject.to_python(stream.read_ctype(data_class, direction=READ_BACKWARD), client) diff --git a/pyignite/queries/query.py b/pyignite/queries/query.py index 57505c7..5bd114b 100644 --- a/pyignite/queries/query.py +++ b/pyignite/queries/query.py @@ -67,13 +67,9 @@ def _build_header(self, stream, values: dict): return header - def from_python(self, conn: Connection, values: dict = None): - if values is None: - values = {} - stream = BinaryStream(conn) - header = self._build_header(stream, values) + def from_python(self, stream, values: dict = None): + header = self._build_header(stream, values if values else {}) stream.write(header) - return stream.getvalue() def perform( self, conn: Connection, query_params: dict = None, @@ -91,8 +87,9 @@ def perform( :return: instance of :class:`~pyignite.api.result.APIResult` with raw value (may undergo further processing in API functions). """ - send_buffer = self.from_python(conn, query_params) - conn.send(send_buffer) + with BinaryStream(conn) as stream: + self.from_python(stream, query_params) + conn.send(stream.getbuffer()) if sql: response_struct = SQLResponse(protocol_version=conn.get_protocol_version(), @@ -101,7 +98,7 @@ def perform( response_struct = Response(protocol_version=conn.get_protocol_version(), following=response_config) - with BinaryStream(conn.recv(), conn) as stream: + with BinaryStream(conn, conn.recv()) as stream: response_ctype = response_struct.parse(stream) response = stream.read_ctype(response_ctype, direction=READ_BACKWARD) diff --git a/pyignite/stream/binary_stream.py b/pyignite/stream/binary_stream.py index 48b4770..1ecdcfb 100644 --- a/pyignite/stream/binary_stream.py +++ b/pyignite/stream/binary_stream.py @@ -22,7 +22,7 @@ class BinaryStream: - def __init__(self, *args): + def __init__(self, conn, buf=None): """ Initialize binary stream around buffers. @@ -31,19 +31,13 @@ def __init__(self, *args): """ from pyignite.connection import Connection - buf = None - if len(args) == 1: - self.conn = args[0] - elif len(args) == 2: - buf = args[0] - self.conn = args[1] - - if not isinstance(self.conn, Connection): + if not isinstance(conn, Connection): raise TypeError(f"invalid parameter: expected instance of {Connection}") if buf and not isinstance(buf, (bytearray, bytes, memoryview)): raise TypeError(f"invalid parameter: expected bytes-like object") + self.conn = conn self.stream = BytesIO(buf) if buf else BytesIO() @property @@ -87,6 +81,9 @@ def seek(self, *args, **kwargs): def getvalue(self): return self.stream.getvalue() + def getbuffer(self): + return self.stream.getbuffer() + def mem_view(self, start=-1, offset=0): start = start if start >= 0 else self.tell() return self.stream.getbuffer()[start:start+offset] From 43ac33d930141910462d918cc309a1a64c1bbb02 Mon Sep 17 00:00:00 2001 From: Ivan Dashchinskiy Date: Mon, 8 Feb 2021 16:46:14 +0300 Subject: [PATCH 8/8] IGNITE-13967 Add jenkins profile for tox. --- .gitignore | 3 ++- tests/config/log4j.xml.jinja2 | 2 +- tox.ini | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5cc86d0..d28510c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .pytest_cache .tox tests/config/*.xml +junit*.xml pyignite.egg-info ignite-log-* -__pycache__ \ No newline at end of file +__pycache__ diff --git a/tests/config/log4j.xml.jinja2 b/tests/config/log4j.xml.jinja2 index 1f1008f..628f66c 100644 --- a/tests/config/log4j.xml.jinja2 +++ b/tests/config/log4j.xml.jinja2 @@ -39,4 +39,4 @@ - \ No newline at end of file + diff --git a/tox.ini b/tox.ini index 69db226..4361413 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,10 @@ usedevelop = True commands = pytest {env:PYTESTARGS:} {posargs} +[jenkins] +setenv: + PYTESTARGS = --junitxml=junit-{envname}.xml + [no-ssl] setenv: PYTEST_ADDOPTS = --examples @@ -54,3 +58,18 @@ setenv: {[ssl]setenv} [testenv:py{36,37,38}-ssl-password] setenv: {[ssl-password]setenv} + +[testenv:py{36,37,38}-jenkins-no-ssl] +setenv: + {[no-ssl]setenv} + {[jenkins]setenv} + +[testenv:py{36,37,38}-jenkins-ssl] +setenv: + {[ssl]setenv} + {[jenkins]setenv} + +[testenv:py{36,37,38}-jenkins-ssl-password] +setenv: + {[ssl-password]setenv} + {[jenkins]setenv}