From 316900c6167d3e4dd6c94a193f058d8b5a596848 Mon Sep 17 00:00:00 2001 From: Ionut Negru Date: Thu, 26 Sep 2019 01:42:28 +0300 Subject: [PATCH 1/4] Support for python native logging from python wrapper --- pulsar-client-cpp/python/pulsar/__init__.py | 7 ++ pulsar-client-cpp/python/src/config.cc | 100 ++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/pulsar-client-cpp/python/pulsar/__init__.py b/pulsar-client-cpp/python/pulsar/__init__.py index af9c0d698f32f..3325a99ca2b2d 100644 --- a/pulsar-client-cpp/python/pulsar/__init__.py +++ b/pulsar-client-cpp/python/pulsar/__init__.py @@ -98,6 +98,7 @@ def send_callback(res, msg): client.close() """ +import logging import _pulsar @@ -329,6 +330,7 @@ def __init__(self, service_url, message_listener_threads=1, concurrent_lookup_requests=50000, log_conf_file_path=None, + logger=None, use_tls=False, tls_trust_certs_file_path=None, tls_allow_insecure_connection=False, @@ -362,6 +364,8 @@ def __init__(self, service_url, to prevent overload on the broker. * `log_conf_file_path`: Initialize log4cxx from a configuration file. + * `logger`: + Set a Python logger for this Pulsar client. * `use_tls`: Configure whether to use TLS encryption on the connection. This setting is deprecated. TLS will be automatically enabled if the `serviceUrl` is @@ -384,6 +388,7 @@ def __init__(self, service_url, _check_type(int, message_listener_threads, 'message_listener_threads') _check_type(int, concurrent_lookup_requests, 'concurrent_lookup_requests') _check_type_or_none(str, log_conf_file_path, 'log_conf_file_path') + _check_type_or_none(logging.Logger, logger, 'logger') _check_type(bool, use_tls, 'use_tls') _check_type_or_none(str, tls_trust_certs_file_path, 'tls_trust_certs_file_path') _check_type(bool, tls_allow_insecure_connection, 'tls_allow_insecure_connection') @@ -398,6 +403,8 @@ def __init__(self, service_url, conf.concurrent_lookup_requests(concurrent_lookup_requests) if log_conf_file_path: conf.log_conf_file_path(log_conf_file_path) + if logger: + conf.set_logger(logger) if use_tls or service_url.startswith('pulsar+ssl://') or service_url.startswith('https://'): conf.use_tls(True) if tls_trust_certs_file_path: diff --git a/pulsar-client-cpp/python/src/config.cc b/pulsar-client-cpp/python/src/config.cc index 4b1454be8afe4..ea4a76f4ed00d 100644 --- a/pulsar-client-cpp/python/src/config.cc +++ b/pulsar-client-cpp/python/src/config.cc @@ -18,6 +18,99 @@ */ #include "utils.h" +class LoggerWrapper: public Logger { + std::string _logger; + PyObject* _pyLogger; + +public: + + LoggerWrapper(const std::string &logger, PyObject* pyLogger) : _logger(logger) { + _pyLogger = pyLogger; + Py_XINCREF(_pyLogger); + } + + LoggerWrapper(const LoggerWrapper& other) { + _pyLogger = other._pyLogger; + Py_XINCREF(_pyLogger); + } + + LoggerWrapper& operator=(const LoggerWrapper& other) { + _pyLogger = other._pyLogger; + Py_XINCREF(_pyLogger); + return *this; + } + + virtual ~LoggerWrapper() { + Py_XDECREF(_pyLogger); + } + + bool isEnabled(Level level) { + PyGILState_STATE state = PyGILState_Ensure(); + bool isEnabled = true; + + try { + // Python levels are: DEBUGGING=10, INFO=20, WARNING=30 and ERROR=40 + isEnabled = py::call_method(_pyLogger, "isEnabledFor", 10 + (level*10)); + } catch (py::error_already_set e) { + PyErr_Print(); + } + + PyGILState_Release(state); + + return isEnabled; + }; + + void log(Level level, int line, const std::string& message) { + PyGILState_STATE state = PyGILState_Ensure(); + + try { + switch (level) { + case Logger::LEVEL_DEBUG: + py::call_method(_pyLogger, "debug", message.c_str()); + break; + case Logger::LEVEL_INFO: + py::call_method(_pyLogger, "info", message.c_str()); + break; + case Logger::LEVEL_WARN: + py::call_method(_pyLogger, "warning", message.c_str()); + break; + case Logger::LEVEL_ERROR: + py::call_method(_pyLogger, "error", message.c_str()); + break; + } + + } catch (py::error_already_set e) { + PyErr_Print(); + } + + PyGILState_Release(state); + } +}; + +class LoggerWrapperFactory : public LoggerFactory { + static LoggerWrapperFactory* _instance; + PyObject* _pyLogger; + + LoggerWrapperFactory(py::object pyLogger) { + _pyLogger = pyLogger.ptr(); + Py_XINCREF(_pyLogger); + } + +public: + virtual ~LoggerWrapperFactory() { + Py_XDECREF(_pyLogger); + } + + Logger* getLogger(const std::string &fileName) { + return new LoggerWrapper(fileName, _pyLogger); + } + + static LoggerFactoryPtr create(py::object pyLogger) { + return LoggerFactoryPtr(new LoggerWrapperFactory(pyLogger)); + } +}; + + template struct ListenerWrapper { PyObject* _pyListener; @@ -74,6 +167,12 @@ static ClientConfiguration& ClientConfiguration_setAuthentication(ClientConfigur return conf; } +static ClientConfiguration& ClientConfiguration_setLogger(ClientConfiguration& conf, + py::object logger) { + conf.setLogger(LoggerWrapperFactory::create(logger)); + return conf; +} + void export_config() { using namespace boost::python; @@ -89,6 +188,7 @@ void export_config() { .def("concurrent_lookup_requests", &ClientConfiguration::setConcurrentLookupRequest, return_self<>()) .def("log_conf_file_path", &ClientConfiguration::getLogConfFilePath, return_value_policy()) .def("log_conf_file_path", &ClientConfiguration::setLogConfFilePath, return_self<>()) + .def("set_logger", &ClientConfiguration_setLogger, return_self<>()) .def("use_tls", &ClientConfiguration::isUseTls) .def("use_tls", &ClientConfiguration::setUseTls, return_self<>()) .def("tls_trust_certs_file_path", &ClientConfiguration::getTlsTrustCertsFilePath) From 295b62149e0b0591fe8ab503861de2ee50007e89 Mon Sep 17 00:00:00 2001 From: Ionut Negru Date: Thu, 26 Sep 2019 10:20:42 +0300 Subject: [PATCH 2/4] Optimizations for isEnabled check on LoggerWrapper --- pulsar-client-cpp/python/src/config.cc | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pulsar-client-cpp/python/src/config.cc b/pulsar-client-cpp/python/src/config.cc index ea4a76f4ed00d..6e7983e55c21a 100644 --- a/pulsar-client-cpp/python/src/config.cc +++ b/pulsar-client-cpp/python/src/config.cc @@ -21,12 +21,27 @@ class LoggerWrapper: public Logger { std::string _logger; PyObject* _pyLogger; + int _currentPythonLogLevel = 10 + (Logger::LEVEL_INFO*10); + + void _updateCurrentPythonLogLevel() { + PyGILState_STATE state = PyGILState_Ensure(); + + try { + _currentPythonLogLevel = py::call_method(_pyLogger, "getEffectiveLevel"); + } catch (py::error_already_set e) { + PyErr_Print(); + } + + PyGILState_Release(state); + }; public: LoggerWrapper(const std::string &logger, PyObject* pyLogger) : _logger(logger) { _pyLogger = pyLogger; Py_XINCREF(_pyLogger); + + _updateCurrentPythonLogLevel(); } LoggerWrapper(const LoggerWrapper& other) { @@ -45,20 +60,8 @@ class LoggerWrapper: public Logger { } bool isEnabled(Level level) { - PyGILState_STATE state = PyGILState_Ensure(); - bool isEnabled = true; - - try { - // Python levels are: DEBUGGING=10, INFO=20, WARNING=30 and ERROR=40 - isEnabled = py::call_method(_pyLogger, "isEnabledFor", 10 + (level*10)); - } catch (py::error_already_set e) { - PyErr_Print(); - } - - PyGILState_Release(state); - - return isEnabled; - }; + return 10 + (level*10) >= _currentPythonLogLevel; + } void log(Level level, int line, const std::string& message) { PyGILState_STATE state = PyGILState_Ensure(); From 8bb6a81494d9e1d42240aac9ff82c2cae970d366 Mon Sep 17 00:00:00 2001 From: Ionut Negru Date: Thu, 26 Sep 2019 01:42:28 +0300 Subject: [PATCH 3/4] Support for python native logging from python wrapper --- pulsar-client-cpp/python/pulsar/__init__.py | 7 ++ pulsar-client-cpp/python/src/config.cc | 100 ++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/pulsar-client-cpp/python/pulsar/__init__.py b/pulsar-client-cpp/python/pulsar/__init__.py index c6b213d974239..2a1155630ca31 100644 --- a/pulsar-client-cpp/python/pulsar/__init__.py +++ b/pulsar-client-cpp/python/pulsar/__init__.py @@ -98,6 +98,7 @@ def send_callback(res, msg): client.close() """ +import logging import _pulsar @@ -335,6 +336,7 @@ def __init__(self, service_url, message_listener_threads=1, concurrent_lookup_requests=50000, log_conf_file_path=None, + logger=None, use_tls=False, tls_trust_certs_file_path=None, tls_allow_insecure_connection=False, @@ -368,6 +370,8 @@ def __init__(self, service_url, to prevent overload on the broker. * `log_conf_file_path`: Initialize log4cxx from a configuration file. + * `logger`: + Set a Python logger for this Pulsar client. * `use_tls`: Configure whether to use TLS encryption on the connection. This setting is deprecated. TLS will be automatically enabled if the `serviceUrl` is @@ -390,6 +394,7 @@ def __init__(self, service_url, _check_type(int, message_listener_threads, 'message_listener_threads') _check_type(int, concurrent_lookup_requests, 'concurrent_lookup_requests') _check_type_or_none(str, log_conf_file_path, 'log_conf_file_path') + _check_type_or_none(logging.Logger, logger, 'logger') _check_type(bool, use_tls, 'use_tls') _check_type_or_none(str, tls_trust_certs_file_path, 'tls_trust_certs_file_path') _check_type(bool, tls_allow_insecure_connection, 'tls_allow_insecure_connection') @@ -404,6 +409,8 @@ def __init__(self, service_url, conf.concurrent_lookup_requests(concurrent_lookup_requests) if log_conf_file_path: conf.log_conf_file_path(log_conf_file_path) + if logger: + conf.set_logger(logger) if use_tls or service_url.startswith('pulsar+ssl://') or service_url.startswith('https://'): conf.use_tls(True) if tls_trust_certs_file_path: diff --git a/pulsar-client-cpp/python/src/config.cc b/pulsar-client-cpp/python/src/config.cc index 306886e1bea9e..2b1531311846a 100644 --- a/pulsar-client-cpp/python/src/config.cc +++ b/pulsar-client-cpp/python/src/config.cc @@ -18,6 +18,99 @@ */ #include "utils.h" +class LoggerWrapper: public Logger { + std::string _logger; + PyObject* _pyLogger; + +public: + + LoggerWrapper(const std::string &logger, PyObject* pyLogger) : _logger(logger) { + _pyLogger = pyLogger; + Py_XINCREF(_pyLogger); + } + + LoggerWrapper(const LoggerWrapper& other) { + _pyLogger = other._pyLogger; + Py_XINCREF(_pyLogger); + } + + LoggerWrapper& operator=(const LoggerWrapper& other) { + _pyLogger = other._pyLogger; + Py_XINCREF(_pyLogger); + return *this; + } + + virtual ~LoggerWrapper() { + Py_XDECREF(_pyLogger); + } + + bool isEnabled(Level level) { + PyGILState_STATE state = PyGILState_Ensure(); + bool isEnabled = true; + + try { + // Python levels are: DEBUGGING=10, INFO=20, WARNING=30 and ERROR=40 + isEnabled = py::call_method(_pyLogger, "isEnabledFor", 10 + (level*10)); + } catch (py::error_already_set e) { + PyErr_Print(); + } + + PyGILState_Release(state); + + return isEnabled; + }; + + void log(Level level, int line, const std::string& message) { + PyGILState_STATE state = PyGILState_Ensure(); + + try { + switch (level) { + case Logger::LEVEL_DEBUG: + py::call_method(_pyLogger, "debug", message.c_str()); + break; + case Logger::LEVEL_INFO: + py::call_method(_pyLogger, "info", message.c_str()); + break; + case Logger::LEVEL_WARN: + py::call_method(_pyLogger, "warning", message.c_str()); + break; + case Logger::LEVEL_ERROR: + py::call_method(_pyLogger, "error", message.c_str()); + break; + } + + } catch (py::error_already_set e) { + PyErr_Print(); + } + + PyGILState_Release(state); + } +}; + +class LoggerWrapperFactory : public LoggerFactory { + static LoggerWrapperFactory* _instance; + PyObject* _pyLogger; + + LoggerWrapperFactory(py::object pyLogger) { + _pyLogger = pyLogger.ptr(); + Py_XINCREF(_pyLogger); + } + +public: + virtual ~LoggerWrapperFactory() { + Py_XDECREF(_pyLogger); + } + + Logger* getLogger(const std::string &fileName) { + return new LoggerWrapper(fileName, _pyLogger); + } + + static LoggerFactoryPtr create(py::object pyLogger) { + return LoggerFactoryPtr(new LoggerWrapperFactory(pyLogger)); + } +}; + + template struct ListenerWrapper { PyObject* _pyListener; @@ -74,6 +167,12 @@ static ClientConfiguration& ClientConfiguration_setAuthentication(ClientConfigur return conf; } +static ClientConfiguration& ClientConfiguration_setLogger(ClientConfiguration& conf, + py::object logger) { + conf.setLogger(LoggerWrapperFactory::create(logger)); + return conf; +} + void export_config() { using namespace boost::python; @@ -89,6 +188,7 @@ void export_config() { .def("concurrent_lookup_requests", &ClientConfiguration::setConcurrentLookupRequest, return_self<>()) .def("log_conf_file_path", &ClientConfiguration::getLogConfFilePath, return_value_policy()) .def("log_conf_file_path", &ClientConfiguration::setLogConfFilePath, return_self<>()) + .def("set_logger", &ClientConfiguration_setLogger, return_self<>()) .def("use_tls", &ClientConfiguration::isUseTls) .def("use_tls", &ClientConfiguration::setUseTls, return_self<>()) .def("tls_trust_certs_file_path", &ClientConfiguration::getTlsTrustCertsFilePath) From 153f4c81a08bce8a4c0f54354448e0dc554a1a00 Mon Sep 17 00:00:00 2001 From: Ionut Negru Date: Thu, 26 Sep 2019 10:20:42 +0300 Subject: [PATCH 4/4] Optimizations for isEnabled check on LoggerWrapper --- pulsar-client-cpp/python/src/config.cc | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pulsar-client-cpp/python/src/config.cc b/pulsar-client-cpp/python/src/config.cc index 2b1531311846a..44d981d552bd6 100644 --- a/pulsar-client-cpp/python/src/config.cc +++ b/pulsar-client-cpp/python/src/config.cc @@ -21,12 +21,27 @@ class LoggerWrapper: public Logger { std::string _logger; PyObject* _pyLogger; + int _currentPythonLogLevel = 10 + (Logger::LEVEL_INFO*10); + + void _updateCurrentPythonLogLevel() { + PyGILState_STATE state = PyGILState_Ensure(); + + try { + _currentPythonLogLevel = py::call_method(_pyLogger, "getEffectiveLevel"); + } catch (py::error_already_set e) { + PyErr_Print(); + } + + PyGILState_Release(state); + }; public: LoggerWrapper(const std::string &logger, PyObject* pyLogger) : _logger(logger) { _pyLogger = pyLogger; Py_XINCREF(_pyLogger); + + _updateCurrentPythonLogLevel(); } LoggerWrapper(const LoggerWrapper& other) { @@ -45,20 +60,8 @@ class LoggerWrapper: public Logger { } bool isEnabled(Level level) { - PyGILState_STATE state = PyGILState_Ensure(); - bool isEnabled = true; - - try { - // Python levels are: DEBUGGING=10, INFO=20, WARNING=30 and ERROR=40 - isEnabled = py::call_method(_pyLogger, "isEnabledFor", 10 + (level*10)); - } catch (py::error_already_set e) { - PyErr_Print(); - } - - PyGILState_Release(state); - - return isEnabled; - }; + return 10 + (level*10) >= _currentPythonLogLevel; + } void log(Level level, int line, const std::string& message) { PyGILState_STATE state = PyGILState_Ensure();