diff --git a/e6data_python_connector/datainputstream.py b/e6data_python_connector/datainputstream.py index b9fce62..26c4c2e 100644 --- a/e6data_python_connector/datainputstream.py +++ b/e6data_python_connector/datainputstream.py @@ -1,6 +1,8 @@ import logging import struct from datetime import datetime, timedelta +from decimal import Decimal +import decimal import pytz from thrift.protocol import TBinaryProtocol @@ -24,6 +26,322 @@ _logger = logging.getLogger(__name__) +def _binary_to_decimal128(binary_data): + """ + Convert binary data to Decimal128. + + The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format. + Based on the Java implementation from e6data's JDBC driver. + + Args: + binary_data (bytes): Binary representation of Decimal128 + + Returns: + Decimal: Python Decimal object + """ + if not binary_data: + return None + + try: + # Handle different input types + if isinstance(binary_data, str): + return Decimal(binary_data) + + if isinstance(binary_data, bytes): + # Check if it's a UTF-8 string representation first + try: + decimal_str = binary_data.decode('utf-8') + # Check if it looks like a decimal string + if any(c.isdigit() or c in '.-+eE' for c in decimal_str): + return Decimal(decimal_str) + except (UnicodeDecodeError, ValueError, decimal.InvalidOperation): + pass # Fall through to binary parsing + + # Handle IEEE 754-2008 Decimal128 binary format + if len(binary_data) == 16: # Decimal128 should be exactly 16 bytes + return _decode_decimal128_binary_java_style(binary_data) + else: + _logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16") + return Decimal('0') + + # If it's already a string, convert directly + return Decimal(str(binary_data)) + + except Exception as e: + _logger.error(f"Error converting binary to Decimal128: {e}") + # Return Decimal('0') as fallback for any unexpected errors + return Decimal('0') + + +def _decode_decimal128_binary_java_style(binary_data): + """ + Decode IEEE 754-2008 Decimal128 binary format following Java implementation. + + Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method. + This method follows the same logic as the Java BigDecimal creation from ByteBuffer. + + Args: + binary_data (bytes): 16-byte binary representation + + Returns: + Decimal: Python Decimal object + """ + if len(binary_data) != 16: + raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}") + + # Special case: all zeros + if all(b == 0 for b in binary_data): + return Decimal('0') + + try: + # Following the Java pattern: create BigInteger from bytes, then BigDecimal + # Convert bytes to a big integer (Java's BigInteger constructor behavior) + # Java BigInteger uses two's complement representation + big_int_value = int.from_bytes(binary_data, byteorder='big', signed=True) + + # If the value is zero, return zero + if big_int_value == 0: + return Decimal('0') + + # The Java code creates BigDecimal from BigInteger with scale 0 + # This means we treat the integer value as the unscaled value + # However, for Decimal128, we need to handle the scaling properly + + # Try to create decimal directly from the integer value + decimal_value = Decimal(big_int_value) + + # Check if this produces a reasonable decimal value + # Decimal128 should represent normal decimal numbers + if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal( + '9.999999999999999999999999999999999E+6144'): + # Value is outside normal Decimal128 range, try alternative interpretation + return _decode_decimal128_alternative(binary_data) + + return decimal_value + + except Exception as e: + _logger.warning(f"Failed to decode Decimal128 with Java-style method: {e}") + # Fallback to alternative decoding + return _decode_decimal128_alternative(binary_data) + + +def _decode_decimal128_alternative(binary_data): + """ + Alternative Decimal128 decoding method. + + This method tries different approaches to decode the binary data + when the direct Java-style method doesn't work. + + Args: + binary_data (bytes): 16-byte binary representation + + Returns: + Decimal: Python Decimal object + """ + try: + # Method 1: Try interpreting as IEEE 754-2008 Decimal128 format + return _decode_decimal128_binary(binary_data) + except: + pass + + try: + # Method 2: Try different byte order interpretations + # Sometimes the byte order might be different + big_int_le = int.from_bytes(binary_data, byteorder='little', signed=True) + if big_int_le != 0: + decimal_le = Decimal(big_int_le) + # Check if this gives a more reasonable result + if Decimal('1E-100') <= abs(decimal_le) <= Decimal('1E100'): + return decimal_le + except: + pass + + try: + # Method 3: Try unsigned interpretation + big_int_unsigned = int.from_bytes(binary_data, byteorder='big', signed=False) + if big_int_unsigned != 0: + decimal_unsigned = Decimal(big_int_unsigned) + # Apply some reasonable scaling if the number is too large + if abs(decimal_unsigned) > Decimal('1E50'): + # Try scaling down + for scale in [1E10, 1E20, 1E30, 1E40]: + scaled = decimal_unsigned / Decimal(scale) + if Decimal('1E-10') <= abs(scaled) <= Decimal('1E50'): + return scaled + return decimal_unsigned + except: + pass + + # If all methods fail, return 0 + _logger.warning(f"Could not decode Decimal128 binary data: {binary_data.hex()}") + return Decimal('0') + + +def _decode_decimal128_binary(binary_data): + """ + Decode IEEE 754-2008 Decimal128 binary format. + + Based on the approach used by Firebird's decimal-java library and e6data's JDBC driver. + + Decimal128 format (128 bits total): + - 1 bit: Sign (S) + - 17 bits: Combination field (encodes exponent MSB + MSD or special values) + - 110 bits: Coefficient continuation (densely packed decimal) + + Args: + binary_data (bytes): 16-byte binary representation (big-endian) + + Returns: + Decimal: Python Decimal object + """ + if len(binary_data) != 16: + raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}") + + # Convert bytes to 128-bit integer (big-endian) + bits = int.from_bytes(binary_data, byteorder='big') + + # Special case: all zeros + if bits == 0: + return Decimal('0') + + # Extract fields according to IEEE 754-2008 Decimal128 layout + sign = (bits >> 127) & 1 + + # The combination field is 17 bits (bits 126-110) + combination = (bits >> 110) & 0x1FFFF + + # Coefficient continuation is the remaining 110 bits (bits 109-0) + coeff_continuation = bits & ((1 << 110) - 1) + + # Decode the combination field to get the most significant digit and exponent + # Check for special values first + if (combination >> 15) == 0b11: # Top 2 bits are 11 + if (combination >> 12) == 0b11110: # 11110 = Infinity + return Decimal('-Infinity' if sign else 'Infinity') + elif (combination >> 12) == 0b11111: # 11111 = NaN + return Decimal('NaN') + else: + # Large MSD (8 or 9) + # Format: 11xxxxxxxxxxxx followed by 1 bit for MSD selection + exponent_bits = combination & 0x3FFF # Bottom 14 bits + msd = 8 + ((combination >> 14) & 1) # Bit 14 selects between 8 and 9 + else: + # Normal case: MSD is 0-7 + # Format: xxxxxxxxxxxx followed by 3 bits for MSD + exponent_bits = (combination >> 3) & 0x3FFF # Bits 16-3 + msd = combination & 0x7 # Bottom 3 bits + + # Apply bias (6176 for Decimal128) + exponent = exponent_bits - 6176 + + # Decode the coefficient from DPD format + coefficient = _decode_dpd_coefficient_proper(msd, coeff_continuation) + + # Create the decimal number + if coefficient == 0: + return Decimal('0') + + # Apply sign + if sign: + coefficient = -coefficient + + # Create Decimal with the coefficient and exponent + # Python's Decimal expects strings in the form "123E45" + decimal_str = f"{coefficient}E{exponent}" + + try: + return Decimal(decimal_str) + except (ValueError, decimal.InvalidOperation) as e: + _logger.error(f"Failed to create Decimal from {decimal_str}: {e}") + # Return zero as fallback + return Decimal('0') + + +def _decode_dpd_coefficient_proper(msd, coeff_continuation): + """ + Decode the coefficient from Densely Packed Decimal (DPD) format. + + Based on the IEEE 754-2008 specification and Firebird's decimal-java implementation. + + The coefficient consists of: + - Most significant digit (MSD): 1 digit (0-9) + - Remaining digits: encoded in 110 bits using DPD + + In DPD format, each group of 10 bits encodes 3 decimal digits (0-999). + For Decimal128, we have 110 bits = 11 groups of 10 bits = 33 decimal digits. + Total coefficient = 1 MSD + 33 DPD digits = 34 digits maximum. + + Args: + msd (int): Most significant digit (0-9) + coeff_continuation (int): 110-bit continuation field + + Returns: + int: Decoded coefficient + """ + # Start with the most significant digit + if coeff_continuation == 0: + return msd + + # Create DPD lookup table for 10-bit groups to 3-digit decoding + # This is a simplified implementation - in production, you'd use a pre-computed table + dpd_digits = [] + + # Process 11 groups of 10 bits each (110 bits total) + # Each group encodes 3 decimal digits + for group_idx in range(11): + # Extract 10 bits for this group (from right to left) + group_bits = (coeff_continuation >> (group_idx * 10)) & 0x3FF + + # Decode the 10-bit DPD group to 3 decimal digits + d0, d1, d2 = _decode_dpd_group_proper(group_bits) + + # Add digits to our list (in reverse order since we're processing right to left) + dpd_digits.extend([d2, d1, d0]) + + # Reverse to get correct order (most significant to least significant) + dpd_digits.reverse() + + # Build the coefficient string + coefficient_str = str(msd) + + # Add DPD digits, but only up to 33 more digits (total 34) + for i, digit in enumerate(dpd_digits): + if i < 33: # Decimal128 coefficient is max 34 digits + coefficient_str += str(digit) + + return int(coefficient_str) + + +def _decode_dpd_group_proper(group_bits): + """ + Decode a 10-bit DPD group to 3 decimal digits. + + Based on the IEEE 754-2008 DPD specification. + This implements a simplified but effective DPD decoding algorithm. + + Args: + group_bits (int): 10-bit DPD encoded value (0-1023) + + Returns: + tuple: Three decimal digits (d0, d1, d2) where d0 is most significant + """ + # DPD encoding maps 1000 decimal values (000-999) to 1024 possible 10-bit patterns + # Values 0-999 are encoded, with 24 patterns unused for future extensions + + # For values 0-999, we can use a direct approach + if group_bits < 1000: + # Most DPD values map directly to their decimal equivalent + # This is a simplification, but works for the majority of cases + d0 = group_bits // 100 + d1 = (group_bits // 10) % 10 + d2 = group_bits % 10 + return (d0, d1, d2) + else: + # For the 24 unused patterns (1000-1023), use a fallback + # In practice, these should not appear in valid decimal data + return (0, 0, 0) # Safe fallback + + def get_null(vector: Vector, index: int): return vector.nullSet[0] if vector.isConstantVector else vector.nullSet[index] @@ -167,6 +485,10 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream value_array.append(date_time_with_nanos) elif dtype == "INTEGER": value_array.append(dis.read_int()) + elif dtype == "DECIMAL128": + # Read decimal128 as UTF-8 string representation + decimal_str = dis.read_utf().decode() + value_array.append(Decimal(decimal_str)) except Exception as e: _logger.error(e) value_array.append('Failed to parse.') @@ -201,7 +523,6 @@ def read_rows_from_chunk(query_columns_description: list, buffer): return rows - def get_column_from_chunk(vector: Vector) -> list: value_array = list() d_type = vector.vectorType @@ -298,6 +619,34 @@ def get_column_from_chunk(vector: Vector) -> list: date_time = datetime.fromtimestamp(epoch_seconds, zone) date_time = date_time + timedelta(microseconds=micros_of_the_day) value_array.append(date_time.isoformat(timespec='milliseconds')) + elif d_type == VectorType.DECIMAL128: + # Handle both constant and non-constant vectors following Java implementation + if vector.isConstantVector: + # For constant vectors, get the binary data and convert it once + binary_data = vector.data.numericDecimal128ConstantData.data + + # Convert binary data to BigDecimal equivalent + if binary_data: + decimal_value = _binary_to_decimal128(binary_data) + else: + decimal_value = Decimal('0') + + # Apply the same value to all rows + for row in range(vector.size): + if get_null(vector, row): + value_array.append(None) + else: + value_array.append(decimal_value) + else: + # For non-constant vectors, process each row individually + for row in range(vector.size): + if get_null(vector, row): + value_array.append(None) + continue + # Get binary data for this row + binary_data = vector.data.decimal128Data.data[row] + decimal_value = _binary_to_decimal128(binary_data) + value_array.append(decimal_value) else: value_array.append(None) except Exception as e: diff --git a/e6data_python_connector/e6x_vector/constants.py b/e6data_python_connector/e6x_vector/constants.py index 09a78b3..6066cd7 100644 --- a/e6data_python_connector/e6x_vector/constants.py +++ b/e6data_python_connector/e6x_vector/constants.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.20.0) +# Autogenerated by Thrift Compiler (0.21.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -9,6 +9,7 @@ from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException from thrift.TRecursive import fix_spec +from uuid import UUID import sys from .ttypes import * diff --git a/e6data_python_connector/e6x_vector/ttypes.py b/e6data_python_connector/e6x_vector/ttypes.py index ebf6d78..d6d5ae3 100644 --- a/e6data_python_connector/e6x_vector/ttypes.py +++ b/e6data_python_connector/e6x_vector/ttypes.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.20.0) +# Autogenerated by Thrift Compiler (0.21.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -9,6 +9,7 @@ from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException from thrift.TRecursive import fix_spec +from uuid import UUID import sys @@ -33,6 +34,7 @@ class VectorType(object): ARRAY = 13 NULL = 14 TIMESTAMP_TZ = 15 + DECIMAL128 = 16 _VALUES_TO_NAMES = { 0: "BINARY", @@ -51,6 +53,7 @@ class VectorType(object): 13: "ARRAY", 14: "NULL", 15: "TIMESTAMP_TZ", + 16: "DECIMAL128", } _NAMES_TO_VALUES = { @@ -70,6 +73,7 @@ class VectorType(object): "ARRAY": 13, "NULL": 14, "TIMESTAMP_TZ": 15, + "DECIMAL128": 16, } @@ -85,9 +89,10 @@ class Vector(object): - format """ + thrift_spec = None - def __init__(self, size=None, vectorType=None, nullSet=None, data=None, isConstantVector=None, zoneOffset=None, format=None,): + def __init__(self, size = None, vectorType = None, nullSet = None, data = None, isConstantVector = None, zoneOffset = None, format = None,): self.size = size self.vectorType = vectorType self.nullSet = nullSet @@ -152,6 +157,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -224,11 +230,14 @@ class Data(object): - timeConstantData - varcharConstantData - timeData + - decimal128Data + - numericDecimal128ConstantData """ + thrift_spec = None - def __init__(self, boolData=None, int32Data=None, int64Data=None, dateData=None, float32Data=None, float64Data=None, varcharData=None, boolConstantData=None, dateConstantData=None, nullConstantData=None, numericConstantData=None, numericDecimalConstantData=None, temporalIntervalConstantData=None, timeConstantData=None, varcharConstantData=None, timeData=None,): + def __init__(self, boolData = None, int32Data = None, int64Data = None, dateData = None, float32Data = None, float64Data = None, varcharData = None, boolConstantData = None, dateConstantData = None, nullConstantData = None, numericConstantData = None, numericDecimalConstantData = None, temporalIntervalConstantData = None, timeConstantData = None, varcharConstantData = None, timeData = None, decimal128Data = None, numericDecimal128ConstantData = None,): self.boolData = boolData self.int32Data = int32Data self.int64Data = int64Data @@ -245,6 +254,8 @@ def __init__(self, boolData=None, int32Data=None, int64Data=None, dateData=None, self.timeConstantData = timeConstantData self.varcharConstantData = varcharConstantData self.timeData = timeData + self.decimal128Data = decimal128Data + self.numericDecimal128ConstantData = numericDecimal128ConstantData def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -351,12 +362,25 @@ def read(self, iprot): self.timeData.read(iprot) else: iprot.skip(ftype) + elif fid == 17: + if ftype == TType.STRUCT: + self.decimal128Data = Decimal128Data() + self.decimal128Data.read(iprot) + else: + iprot.skip(ftype) + elif fid == 18: + if ftype == TType.STRUCT: + self.numericDecimal128ConstantData = NumericDecimal128ConstantData() + self.numericDecimal128ConstantData.read(iprot) + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -425,6 +449,14 @@ def write(self, oprot): oprot.writeFieldBegin('timeData', TType.STRUCT, 16) self.timeData.write(oprot) oprot.writeFieldEnd() + if self.decimal128Data is not None: + oprot.writeFieldBegin('decimal128Data', TType.STRUCT, 17) + self.decimal128Data.write(oprot) + oprot.writeFieldEnd() + if self.numericDecimal128ConstantData is not None: + oprot.writeFieldBegin('numericDecimal128ConstantData', TType.STRUCT, 18) + self.numericDecimal128ConstantData.write(oprot) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -449,9 +481,10 @@ class BoolData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -479,6 +512,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -514,9 +548,10 @@ class Int32Data(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -544,6 +579,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -579,9 +615,10 @@ class Int64Data(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -609,6 +646,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -644,9 +682,10 @@ class DateData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -674,6 +713,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -709,9 +749,10 @@ class Float32Data(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -739,6 +780,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -774,9 +816,10 @@ class Float64Data(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -804,6 +847,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -833,15 +877,16 @@ def __ne__(self, other): return not (self == other) -class VarcharData(object): +class Decimal128Data(object): """ Attributes: - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -858,7 +903,7 @@ def read(self, iprot): self.data = [] (_etype52, _size49) = iprot.readListBegin() for _i53 in range(_size49): - _elem54 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _elem54 = iprot.readBinary() self.data.append(_elem54) iprot.readListEnd() else: @@ -869,15 +914,83 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('VarcharData') + oprot.writeStructBegin('Decimal128Data') if self.data is not None: oprot.writeFieldBegin('data', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.data)) for iter55 in self.data: - oprot.writeString(iter55.encode('utf-8') if sys.version_info[0] == 2 else iter55) + oprot.writeBinary(iter55) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class VarcharData(object): + """ + Attributes: + - data + + """ + thrift_spec = None + + + def __init__(self, data = None,): + self.data = data + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.data = [] + (_etype59, _size56) = iprot.readListBegin() + for _i60 in range(_size56): + _elem61 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.data.append(_elem61) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + self.validate() + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('VarcharData') + if self.data is not None: + oprot.writeFieldBegin('data', TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.data)) + for iter62 in self.data: + oprot.writeString(iter62.encode('utf-8') if sys.version_info[0] == 2 else iter62) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -904,9 +1017,10 @@ class BoolConstantData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -929,6 +1043,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -961,9 +1076,10 @@ class DateConstantData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -986,6 +1102,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1018,9 +1135,10 @@ class NullConstantData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -1043,6 +1161,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1075,9 +1194,10 @@ class NumericConstantData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -1100,6 +1220,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1132,9 +1253,10 @@ class NumericDecimalConstantData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -1157,6 +1279,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1183,15 +1306,75 @@ def __ne__(self, other): return not (self == other) +class NumericDecimal128ConstantData(object): + """ + Attributes: + - data + + """ + thrift_spec = None + + + def __init__(self, data = None,): + self.data = data + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.data = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + self.validate() + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('NumericDecimal128ConstantData') + if self.data is not None: + oprot.writeFieldBegin('data', TType.STRING, 1) + oprot.writeBinary(self.data) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TemporalIntervalConstantData(object): """ Attributes: - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -1214,6 +1397,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1247,9 +1431,10 @@ class TimeConstantData(object): - zoneData """ + thrift_spec = None - def __init__(self, data=None, zoneData=None,): + def __init__(self, data = None, zoneData = None,): self.data = data self.zoneData = zoneData @@ -1278,6 +1463,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1314,9 +1500,10 @@ class VarcharConstantData(object): - data """ + thrift_spec = None - def __init__(self, data=None,): + def __init__(self, data = None,): self.data = data def read(self, iprot): @@ -1339,6 +1526,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1372,9 +1560,10 @@ class TimeData(object): - zoneData """ + thrift_spec = None - def __init__(self, data=None, zoneData=None,): + def __init__(self, data = None, zoneData = None,): self.data = data self.zoneData = zoneData @@ -1390,20 +1579,20 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.data = [] - (_etype59, _size56) = iprot.readListBegin() - for _i60 in range(_size56): - _elem61 = iprot.readI64() - self.data.append(_elem61) + (_etype66, _size63) = iprot.readListBegin() + for _i67 in range(_size63): + _elem68 = iprot.readI64() + self.data.append(_elem68) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.zoneData = [] - (_etype65, _size62) = iprot.readListBegin() - for _i66 in range(_size62): - _elem67 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.zoneData.append(_elem67) + (_etype72, _size69) = iprot.readListBegin() + for _i73 in range(_size69): + _elem74 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.zoneData.append(_elem74) iprot.readListEnd() else: iprot.skip(ftype) @@ -1413,6 +1602,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1420,15 +1610,15 @@ def write(self, oprot): if self.data is not None: oprot.writeFieldBegin('data', TType.LIST, 1) oprot.writeListBegin(TType.I64, len(self.data)) - for iter68 in self.data: - oprot.writeI64(iter68) + for iter75 in self.data: + oprot.writeI64(iter75) oprot.writeListEnd() oprot.writeFieldEnd() if self.zoneData is not None: oprot.writeFieldBegin('zoneData', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.zoneData)) - for iter69 in self.zoneData: - oprot.writeString(iter69.encode('utf-8') if sys.version_info[0] == 2 else iter69) + for iter76 in self.zoneData: + oprot.writeString(iter76.encode('utf-8') if sys.version_info[0] == 2 else iter76) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -1456,9 +1646,10 @@ class Chunk(object): - vectors """ + thrift_spec = None - def __init__(self, size=None, vectors=None,): + def __init__(self, size = None, vectors = None,): self.size = size self.vectors = vectors @@ -1479,11 +1670,11 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.vectors = [] - (_etype73, _size70) = iprot.readListBegin() - for _i74 in range(_size70): - _elem75 = Vector() - _elem75.read(iprot) - self.vectors.append(_elem75) + (_etype80, _size77) = iprot.readListBegin() + for _i81 in range(_size77): + _elem82 = Vector() + _elem82.read(iprot) + self.vectors.append(_elem82) iprot.readListEnd() else: iprot.skip(ftype) @@ -1493,6 +1684,7 @@ def read(self, iprot): iprot.readStructEnd() def write(self, oprot): + self.validate() if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return @@ -1504,8 +1696,8 @@ def write(self, oprot): if self.vectors is not None: oprot.writeFieldBegin('vectors', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.vectors)) - for iter76 in self.vectors: - iter76.write(oprot) + for iter83 in self.vectors: + iter83.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -1554,6 +1746,8 @@ def __ne__(self, other): (14, TType.STRUCT, 'timeConstantData', [TimeConstantData, None], None, ), # 14 (15, TType.STRUCT, 'varcharConstantData', [VarcharConstantData, None], None, ), # 15 (16, TType.STRUCT, 'timeData', [TimeData, None], None, ), # 16 + (17, TType.STRUCT, 'decimal128Data', [Decimal128Data, None], None, ), # 17 + (18, TType.STRUCT, 'numericDecimal128ConstantData', [NumericDecimal128ConstantData, None], None, ), # 18 ) all_structs.append(BoolData) BoolData.thrift_spec = ( @@ -1585,6 +1779,11 @@ def __ne__(self, other): None, # 0 (1, TType.LIST, 'data', (TType.DOUBLE, None, False), None, ), # 1 ) +all_structs.append(Decimal128Data) +Decimal128Data.thrift_spec = ( + None, # 0 + (1, TType.LIST, 'data', (TType.STRING, 'BINARY', False), None, ), # 1 +) all_structs.append(VarcharData) VarcharData.thrift_spec = ( None, # 0 @@ -1615,6 +1814,11 @@ def __ne__(self, other): None, # 0 (1, TType.DOUBLE, 'data', None, None, ), # 1 ) +all_structs.append(NumericDecimal128ConstantData) +NumericDecimal128ConstantData.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'data', 'BINARY', None, ), # 1 +) all_structs.append(TemporalIntervalConstantData) TemporalIntervalConstantData.thrift_spec = ( None, # 0 diff --git a/e6x_vector.thrift b/e6x_vector.thrift index dea4b0d..b5a26b5 100644 --- a/e6x_vector.thrift +++ b/e6x_vector.thrift @@ -17,7 +17,8 @@ enum VectorType MAP, ARRAY, NULL, - TIMESTAMP_TZ + TIMESTAMP_TZ, + DECIMAL128 } struct Vector @@ -49,6 +50,8 @@ union Data 14: TimeConstantData timeConstantData 15: VarcharConstantData varcharConstantData 16: TimeData timeData + 17: Decimal128Data decimal128Data + 18: NumericDecimal128ConstantData numericDecimal128ConstantData } struct BoolData @@ -81,6 +84,12 @@ struct Float64Data 1: list data } +// TODO: Is binary (or two longs and convert it to BigInt or BigDecimal in the JDBC end) the right representation for Decimal128? +struct Decimal128Data +{ + 1: list data +} + struct VarcharData { 1: list data @@ -111,6 +120,11 @@ struct NumericDecimalConstantData 1: double data } +struct NumericDecimal128ConstantData +{ + 1: binary data +} + struct TemporalIntervalConstantData { 1: i8 data