diff --git a/README.md b/README.md index cd0c609..bbdfe14 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ The `Connection` class supports the following parameters: | `secure` | bool | No | False | Enable SSL/TLS for secure connections | | `auto_resume` | bool | No | True | Automatically resume cluster if suspended | | `grpc_options` | dict | No | None | Additional gRPC configuration options | +| `debug` | bool | No | False | Enable debug logging for troubleshooting | #### Secure Connection Example @@ -598,4 +599,58 @@ conn.close() - Graceful connection recovery and retry logic - Blue-green deployment support with automatic failover +## Debugging and Troubleshooting + +### Enable Debug Mode + +Enable comprehensive debugging to troubleshoot connection and query issues: + +```python +from e6data_python_connector import Connection + +conn = Connection( + host=host, + port=port, + username=username, + password=password, + database=database, + debug=True # Enable debug logging +) +``` + +When `debug=True`, the following features are enabled: +- Python logging at DEBUG level for all operations +- Blue-green strategy transition logging +- Connection lifecycle logging +- Query execution detailed logging + +### gRPC Network Tracing + +For low-level gRPC network debugging (HTTP/2 frames, TCP events), set environment variables **before** running your Python script: + +```bash +# Enable gRPC network tracing +export GRPC_VERBOSITY=DEBUG +export GRPC_TRACE=client_channel,http2 + +# For comprehensive tracing +export GRPC_TRACE=api,call_error,channel,client_channel,connectivity_state,http,http2_stream,tcp,transport_security + +# Run your script +python your_script.py +``` + +**Note**: These environment variables must be set before Python starts, as the gRPC C++ core reads them at module import time. + +### Common Issues and Solutions + +| Issue | Solution | +|-------|----------| +| Connection timeout | Check network connectivity, firewall rules, and ensure port 80/443 is open | +| Authentication failure | Verify username (email) and access token are correct | +| 503 Service Unavailable | Cluster may be suspended; enable `auto_resume=True` | +| 456 Strategy Error | Automatic blue-green failover will handle this | +| Memory issues with large results | Use `fetchall_buffer()` instead of `fetchall()` | +| gRPC message size errors | Configure `grpc_options` with appropriate message size limits | + See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation. diff --git a/e6data_python_connector/datainputstream.py b/e6data_python_connector/datainputstream.py index 26c4c2e..d9c81f1 100644 --- a/e6data_python_connector/datainputstream.py +++ b/e6data_python_connector/datainputstream.py @@ -26,16 +26,18 @@ _logger = logging.getLogger(__name__) -def _binary_to_decimal128(binary_data): +def _binary_to_decimal128(binary_data, scale=None): """ Convert binary data to Decimal128. - + The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format. Based on the Java implementation from e6data's JDBC driver. - + Args: binary_data (bytes): Binary representation of Decimal128 - + scale (int, optional): Scale parameter for decimal precision. If None, + attempts to decode from IEEE format or defaults to 0. + Returns: Decimal: Python Decimal object """ @@ -59,7 +61,7 @@ def _binary_to_decimal128(binary_data): # Handle IEEE 754-2008 Decimal128 binary format if len(binary_data) == 16: # Decimal128 should be exactly 16 bytes - return _decode_decimal128_binary_java_style(binary_data) + return _decode_decimal128_binary_java_style(binary_data, scale) else: _logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16") return Decimal('0') @@ -73,16 +75,17 @@ def _binary_to_decimal128(binary_data): return Decimal('0') -def _decode_decimal128_binary_java_style(binary_data): +def _decode_decimal128_binary_java_style(binary_data, scale=None): """ Decode IEEE 754-2008 Decimal128 binary format following Java implementation. - + Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method. This method follows the same logic as the Java BigDecimal creation from ByteBuffer. - + Args: binary_data (bytes): 16-byte binary representation - + scale (int, optional): Scale parameter for decimal precision + Returns: Decimal: Python Decimal object """ @@ -103,17 +106,21 @@ def _decode_decimal128_binary_java_style(binary_data): if big_int_value == 0: return Decimal('0') - # The Java code creates BigDecimal from BigInteger with scale 0 - # This means we treat the integer value as the unscaled value - # However, for Decimal128, we need to handle the scaling properly - - # Try to create decimal directly from the integer value - decimal_value = Decimal(big_int_value) + # The Java code creates BigDecimal from BigInteger with the provided scale + # If scale is provided, use it to create the properly scaled decimal + if scale is not None: + # Create decimal with the specified scale (like Java's BigDecimal constructor) + # This treats big_int_value as the unscaled value + decimal_value = Decimal(big_int_value) / (Decimal(10) ** scale) + else: + # Fallback: try to create decimal directly from the integer value + decimal_value = Decimal(big_int_value) # Check if this produces a reasonable decimal value # Decimal128 should represent normal decimal numbers - if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal( - '9.999999999999999999999999999999999E+6144'): + # Only check range if scale was not provided (backward compatibility) + if scale is None and (abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal( + '9.999999999999999999999999999999999E+6144')): # Value is outside normal Decimal128 range, try alternative interpretation return _decode_decimal128_alternative(binary_data) @@ -624,10 +631,12 @@ def get_column_from_chunk(vector: Vector) -> list: if vector.isConstantVector: # For constant vectors, get the binary data and convert it once binary_data = vector.data.numericDecimal128ConstantData.data + # Get scale with backward compatibility for older engines + scale = getattr(vector.data.numericDecimal128ConstantData, 'scale', None) # Convert binary data to BigDecimal equivalent if binary_data: - decimal_value = _binary_to_decimal128(binary_data) + decimal_value = _binary_to_decimal128(binary_data, scale) else: decimal_value = Decimal('0') @@ -639,13 +648,16 @@ def get_column_from_chunk(vector: Vector) -> list: value_array.append(decimal_value) else: # For non-constant vectors, process each row individually + # Get scale from decimal128Data (with backward compatibility) + scale = getattr(vector.data.decimal128Data, 'scale', None) + for row in range(vector.size): if get_null(vector, row): value_array.append(None) continue # Get binary data for this row binary_data = vector.data.decimal128Data.data[row] - decimal_value = _binary_to_decimal128(binary_data) + decimal_value = _binary_to_decimal128(binary_data, scale) value_array.append(decimal_value) else: value_array.append(None) diff --git a/e6data_python_connector/e6data_grpc.py b/e6data_python_connector/e6data_grpc.py index d7e3a2e..d335a85 100644 --- a/e6data_python_connector/e6data_grpc.py +++ b/e6data_python_connector/e6data_grpc.py @@ -7,6 +7,9 @@ from __future__ import unicode_literals import datetime +import logging +import os + import re import sys import time @@ -396,8 +399,51 @@ def __init__( self._debug = debug if self._debug: _debug_connections.add(id(self)) + + # Enable comprehensive debugging if debug flag is set + if self._debug: + # Configure root logger for DEBUG level + logging.basicConfig( + level=logging.DEBUG, + format='[%(name)s] %(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + force=True # Force reconfiguration even if logging is already configured + ) + + # Note: gRPC C++ core tracing (GRPC_VERBOSITY and GRPC_TRACE) must be set + # BEFORE the gRPC module is imported to take effect. Setting them at runtime + # will not enable HTTP/2 frame logs or low-level tracing. + # + # To enable full gRPC network tracing, set these environment variables + # before starting your Python script: + # export GRPC_VERBOSITY=DEBUG + # export GRPC_TRACE=client_channel,http2 + # + # The following runtime settings only affect Python-level logging: + + # Enable gRPC Python logging (this works at runtime) + os.environ['GRPC_PYTHON_LOG_LEVEL'] = 'DEBUG' + os.environ['GRPC_PYTHON_LOG_STDERR'] = '1' + + # Ensure gRPC logger is at DEBUG level + grpc_logger = logging.getLogger('grpc') + grpc_logger.setLevel(logging.DEBUG) + + # Enable gRPC transport logger + grpc_transport_logger = logging.getLogger('grpc._channel') + grpc_transport_logger.setLevel(logging.DEBUG) + + # Enable gRPC server logger + grpc_server_logger = logging.getLogger('grpc._server') + grpc_server_logger.setLevel(logging.DEBUG) + + # Set e6data connector logger to DEBUG + e6data_logger = logging.getLogger('e6data_python_connector') + e6data_logger.setLevel(logging.DEBUG) + _strategy_debug_log(f"Debug mode enabled for connection {id(self)}") - + _strategy_debug_log(f"GRPC_TRACE={os.environ.get('GRPC_TRACE')}") + self._create_client() @property @@ -449,6 +495,7 @@ def _create_client(self): Raises: grpc.RpcError: If there is an error in creating the gRPC channel or client stub. """ + if self._secure_channel: self._channel = grpc.secure_channel( target='{}:{}'.format(self._host, self._port), diff --git a/e6data_python_connector/e6x_vector/ttypes.py b/e6data_python_connector/e6x_vector/ttypes.py index d6d5ae3..77afba0 100644 --- a/e6data_python_connector/e6x_vector/ttypes.py +++ b/e6data_python_connector/e6x_vector/ttypes.py @@ -881,13 +881,15 @@ class Decimal128Data(object): """ Attributes: - data + - scale """ thrift_spec = None - def __init__(self, data = None,): + def __init__(self, data = None, scale = None,): self.data = data + self.scale = scale def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -908,6 +910,11 @@ def read(self, iprot): iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.scale = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -926,6 +933,10 @@ def write(self, oprot): oprot.writeBinary(iter55) oprot.writeListEnd() oprot.writeFieldEnd() + if self.scale is not None: + oprot.writeFieldBegin('scale', TType.I32, 2) + oprot.writeI32(self.scale) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -1310,13 +1321,15 @@ class NumericDecimal128ConstantData(object): """ Attributes: - data + - scale """ thrift_spec = None - def __init__(self, data = None,): + def __init__(self, data = None, scale = None,): self.data = data + self.scale = scale def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -1332,6 +1345,11 @@ def read(self, iprot): self.data = iprot.readBinary() else: iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.scale = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -1347,6 +1365,10 @@ def write(self, oprot): oprot.writeFieldBegin('data', TType.STRING, 1) oprot.writeBinary(self.data) oprot.writeFieldEnd() + if self.scale is not None: + oprot.writeFieldBegin('scale', TType.I32, 2) + oprot.writeI32(self.scale) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -1783,6 +1805,7 @@ def __ne__(self, other): Decimal128Data.thrift_spec = ( None, # 0 (1, TType.LIST, 'data', (TType.STRING, 'BINARY', False), None, ), # 1 + (2, TType.I32, 'scale', None, None, ), # 2 ) all_structs.append(VarcharData) VarcharData.thrift_spec = ( @@ -1818,6 +1841,7 @@ def __ne__(self, other): NumericDecimal128ConstantData.thrift_spec = ( None, # 0 (1, TType.STRING, 'data', 'BINARY', None, ), # 1 + (2, TType.I32, 'scale', None, None, ), # 2 ) all_structs.append(TemporalIntervalConstantData) TemporalIntervalConstantData.thrift_spec = ( diff --git a/e6x_vector.thrift b/e6x_vector.thrift index b5a26b5..b2c85a0 100644 --- a/e6x_vector.thrift +++ b/e6x_vector.thrift @@ -88,6 +88,7 @@ struct Float64Data struct Decimal128Data { 1: list data + 2: i32 scale } struct VarcharData @@ -123,6 +124,7 @@ struct NumericDecimalConstantData struct NumericDecimal128ConstantData { 1: binary data + 2: i32 scale } struct TemporalIntervalConstantData