Skip to content

IGNITE-13863: Fix Null reading and writing #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
isapego wants to merge 7 commits into from
Closed

IGNITE-13863: Fix Null reading and writing #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
999ea3f
IGNITE-13863: Fix Null reading and writing
isapego Dec 2, 2020
bf76b27
IGNITE-13863: Minor fix
isapego Jan 27, 2021
10f1fa5
IGNITE-13863: Review fixes
isapego Feb 2, 2021
09fecde
IGNITE-13863: Fix test
isapego Feb 2, 2021
7fd95bf
IGNITE-13863: Fix
isapego Feb 2, 2021
099f1fc
IGNITE-13863: Fix test
isapego Feb 2, 2021
38e8182
IGNITE-13863: Fix
isapego Feb 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 59 additions & 10 deletions pyignite/datatypes/complex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@

from pyignite.constants import *
from pyignite.exceptions import ParseError

from .base import IgniteDataType
from .internal import AnyDataObject, infer_from_python
from .type_codes import *
from .type_ids import *
from .type_names import *
from .null_object import Null


__all__ = [
Expand Down Expand Up @@ -68,8 +70,13 @@ def build_header(cls):

@classmethod
def parse(cls, client: 'Client'):
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))

if tc_type == TC_NULL:
return Null.build_c_type(), tc_type

header_class = cls.build_header()
buffer = client.recv(ctypes.sizeof(header_class))
buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []

Expand All @@ -91,7 +98,10 @@ def parse(cls, client: 'Client'):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
for i in range(ctype_object.length):
length = getattr(ctype_object, "length", None)
if length is None:
return None
for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
Expand All @@ -102,6 +112,9 @@ def to_python(cls, ctype_object, *args, **kwargs):

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()

type_or_id, value = value
header_class = cls.build_header()
header = header_class()
Expand Down Expand Up @@ -150,8 +163,13 @@ def build_header(cls):

@classmethod
def parse(cls, client: 'Client'):
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))

if tc_type == TC_NULL:
return Null.build_c_type(), tc_type

header_class = cls.build_header()
buffer = client.recv(ctypes.sizeof(header_class))
buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)

final_class = type(
Expand Down Expand Up @@ -243,8 +261,13 @@ def build_header(cls):

@classmethod
def parse(cls, client: 'Client'):
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))

if tc_type == TC_NULL:
return Null.build_c_type(), tc_type

header_class = cls.build_header()
buffer = client.recv(ctypes.sizeof(header_class))
buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []

Expand All @@ -266,7 +289,10 @@ def parse(cls, client: 'Client'):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
for i in range(ctype_object.length):
length = getattr(ctype_object, "length", None)
if length is None:
return None
for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
Expand All @@ -277,6 +303,9 @@ def to_python(cls, ctype_object, *args, **kwargs):

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()

type_or_id, value = value
header_class = cls.build_header()
header = header_class()
Expand Down Expand Up @@ -330,8 +359,13 @@ def build_header(cls):

@classmethod
def parse(cls, client: 'Client'):
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))

if tc_type == TC_NULL:
return Null.build_c_type(), tc_type

header_class = cls.build_header()
buffer = client.recv(ctypes.sizeof(header_class))
buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []

Expand Down Expand Up @@ -420,12 +454,18 @@ def build_header(cls):

@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
return ctype_object.type, super().to_python(
obj_type = getattr(ctype_object, "type", None)
if obj_type is None:
return None
return obj_type, super().to_python(
ctype_object, *args, **kwargs
)

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()

type_id, value = value
return super().from_python(value, type_id)

Expand Down Expand Up @@ -539,9 +579,13 @@ def get_dataclass(conn: 'Connection', header) -> OrderedDict:
@classmethod
def parse(cls, client: 'Client'):
from pyignite.datatypes import Struct
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))

if tc_type == TC_NULL:
return Null.build_c_type(), tc_type

header_class = cls.build_header()
buffer = client.recv(ctypes.sizeof(header_class))
buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)

# ignore full schema, always retrieve fields' types and order
Expand Down Expand Up @@ -572,14 +616,17 @@ def parse(cls, client: 'Client'):

@classmethod
def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs):
type_id = getattr(ctype_object, "type_id", None)
if type_id is None:
return None

if not client:
raise ParseError(
'Can not query binary type {}'.format(ctype_object.type_id)
'Can not query binary type {}'.format(type_id)
)

data_class = client.query_binary_type(
ctype_object.type_id,
type_id,
ctype_object.schema_id
)
result = data_class()
Expand All @@ -596,6 +643,8 @@ def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs):

@classmethod
def from_python(cls, value: object):
if value is None:
return Null.from_python()

if getattr(value, '_buffer', None) is None:
client = cls.find_client()
Expand Down
5 changes: 4 additions & 1 deletion pyignite/datatypes/internal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,10 @@ def parse(self, client: 'Client'):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
for i in range(ctype_object.length):
length = getattr(ctype_object, "length", None)
if length is None:
return None
for i in range(length):
result.append(
super().to_python(
getattr(ctype_object, 'element_{}'.format(i)),
Expand Down
33 changes: 28 additions & 5 deletions pyignite/datatypes/primitive_arrays.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from typing import Any

from pyignite.constants import *
from . import Null
from .base import IgniteDataType
from .primitive import *
from .type_codes import *
Expand Down Expand Up @@ -61,8 +62,13 @@ def build_header_class(cls):

@classmethod
def parse(cls, client: 'Client'):
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))

if tc_type == TC_NULL:
return Null.build_c_type(), tc_type

header_class = cls.build_header_class()
buffer = client.recv(ctypes.sizeof(header_class))
buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
cls.__name__,
Expand All @@ -82,12 +88,18 @@ def parse(cls, client: 'Client'):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
for i in range(ctype_object.length):
length = getattr(ctype_object, "length", None)
if length is None:
return None
for i in range(length):
result.append(ctype_object.data[i])
return result

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()

header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
Expand All @@ -112,7 +124,10 @@ class ByteArray(PrimitiveArray):

@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
return bytearray(ctype_object.data)
data = getattr(ctype_object, "data", None)
if data is None:
return None
return bytearray(data)

@classmethod
def from_python(cls, value):
Expand Down Expand Up @@ -210,6 +225,9 @@ def to_python(cls, ctype_object, *args, **kwargs):

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()

header_class = cls.build_header_class()
header = header_class()
header.type_code = int.from_bytes(
Expand Down Expand Up @@ -282,6 +300,8 @@ class CharArrayObject(PrimitiveArrayObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
values = super().to_python(ctype_object, *args, **kwargs)
if values is None:
return None
return [
v.to_bytes(
ctypes.sizeof(cls.primitive_type.c_type),
Expand All @@ -302,7 +322,10 @@ class BoolArrayObject(PrimitiveArrayObject):
def to_python(cls, ctype_object, *args, **kwargs):
if not ctype_object:
return None
result = [False] * ctype_object.length
for i in range(ctype_object.length):
length = getattr(ctype_object, "length", None)
if length is None:
return None
result = [False] * length
for i in range(length):
result[i] = ctype_object.data[i] != 0
return result
23 changes: 19 additions & 4 deletions pyignite/datatypes/primitive_objects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@

from pyignite.constants import *
from pyignite.utils import unsigned

from .base import IgniteDataType
from .type_codes import *
from .type_ids import *
from .type_names import *
from .null_object import Null


__all__ = [
Expand Down Expand Up @@ -60,16 +62,21 @@ def build_c_type(cls):

@classmethod
def parse(cls, client: 'Client'):
tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
if tc_type == TC_NULL:
return Null.build_c_type(), tc_type
data_type = cls.build_c_type()
buffer = client.recv(ctypes.sizeof(data_type))
buffer = tc_type + client.recv(ctypes.sizeof(data_type) - len(tc_type))
return data_type, buffer

@staticmethod
def to_python(ctype_object, *args, **kwargs):
return ctype_object.value
return getattr(ctype_object, "value", None)

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
Expand Down Expand Up @@ -185,13 +192,18 @@ def hashcode(value: str, *args, **kwargs) -> int:

@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
return ctype_object.value.to_bytes(
value = getattr(ctype_object, "value", None)
if value is None:
return None
return value.to_bytes(
ctypes.sizeof(cls.c_type),
byteorder=PROTOCOL_BYTE_ORDER
).decode(PROTOCOL_CHAR_ENCODING)

@classmethod
def from_python(cls, value):
if value is None:
return Null.from_python()
if type(value) is str:
value = value.encode(PROTOCOL_CHAR_ENCODING)
# assuming either a bytes or an integer
Expand All @@ -218,5 +230,8 @@ def hashcode(value: bool, *args, **kwargs) -> int:

@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
return ctype_object.value != 0
value = getattr(ctype_object, "value", None)
if value is None:
return None
return value != 0

Loading