diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a81c8ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/README.md b/README.md index aab508d..2699987 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# python3-logstash +# logstash-sync -[![PyPI version](https://badge.fury.io/py/python3-logstash.svg)](https://pypi.org/project/python3-logstash/) +[![PyPI version](https://badge.fury.io/py/logstash-sync.svg)](https://pypi.org/project/logstash-sync/) ## Python logging handler for Logstash. @@ -12,7 +12,9 @@ That has been update to work with python 3. ### Installation Using pip: -`pip install python3-logstash` +```bash +pip install logstash-sync +``` ### Usage @@ -20,7 +22,7 @@ Using pip: #### For example: -``` +```python import logging import logstash import sys @@ -52,34 +54,34 @@ When using `extra` field make sure you don't use reserved names. From `Python do | "The keys in the dictionary passed in extra should not clash with the keys used by the logging system. (See the `Formatter `_ documentation for more information on which keys are used by the logging system.)" To use the AMQPLogstashHandler you will need to install pika first. -``` +```bash pip install pika ``` For example:: -``` - import logging - import logstash - - test_logger = logging.getLogger('python-logstash-logger') - test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1)) - - test_logger.info('python-logstash: test logstash info message.') - try: - 1/0 - except: - test_logger.exception('python-logstash-logger: Exception with stack trace!') +```python +import logging +import logstash + +test_logger = logging.getLogger('python-logstash-logger') +test_logger.setLevel(logging.INFO) +test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1)) + +test_logger.info('python-logstash: test logstash info message.') +try: + 1/0 +except: + test_logger.exception('python-logstash-logger: Exception with stack trace!') ``` ### Using with Django Modify your `settings.py` to integrate `python3-logstash` with Django's logging:: -``` - LOGGING = { - ... - 'handlers': { +```python +LOGGING = { + # ... + 'handlers': { 'logstash': { 'level': 'DEBUG', 'class': 'logstash.LogstashHandler', @@ -90,21 +92,21 @@ Modify your `settings.py` to integrate `python3-logstash` with Django's logging: 'fqdn': False, # Fully qualified domain name. Default value: false. 'tags': ['tag1', 'tag2'], # list of tags. Default: None. }, - }, - 'loggers': { + }, + 'loggers': { 'django.request': { 'handlers': ['logstash'], 'level': 'DEBUG', 'propagate': True, }, - }, - ... - } + }, + # ... +} ``` ### Using with Gunicorn Create a logging.conf similar to this: -``` +```ini [loggers] keys=root, logstash.error, logstash.access diff --git a/README.rst b/README.rst deleted file mode 100644 index 57587d1..0000000 --- a/README.rst +++ /dev/null @@ -1,176 +0,0 @@ -python3-logstash -=================== - -Python logging handler for Logstash. -https://www.elastic.co/products/logstash - -Notes: -========= -This is a copy of python3-logstash: https://pypi.python.org/pypi/python3-logstash -That has been update to work with python 3. - -Installation -============ - -Using pip:: - - pip install python3-logstash - -Usage -===== - -``LogstashHandler`` is a custom logging handler which sends Logstash messages using UDP. - -For example:: - - import logging - import logstash - import sys - - host = 'localhost' - - test_logger = logging.getLogger('python3-logstash-logger') - test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1)) - # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1)) - - test_logger.error('python3-logstash: test logstash error message.') - test_logger.info('python3-logstash: test logstash info message.') - test_logger.warning('python3-logstash: test logstash warning message.') - - # add extra field to logstash message - extra = { - 'test_string': 'python version: ' + repr(sys.version_info), - 'test_boolean': True, - 'test_dict': {'a': 1, 'b': 'c'}, - 'test_float': 1.23, - 'test_integer': 123, - 'test_list': [1, 2, '3'], - } - test_logger.info('python3-logstash: test extra fields', extra=extra) - -When using ``extra`` field make sure you don't use reserved names. From `Python documentation `_. - | "The keys in the dictionary passed in extra should not clash with the keys used by the logging system. (See the `Formatter `_ documentation for more information on which keys are used by the logging system.)" - -To use the AMQPLogstashHandler you will need to install pika first. - - pip install pika - -For example:: - - import logging - import logstash - - test_logger = logging.getLogger('python3-logstash-logger') - test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1)) - - test_logger.info('python3-logstash: test logstash info message.') - try: - 1/0 - except: - test_logger.exception('python3-logstash-logger: Exception with stack trace!') - - - -Using with Django -================= - -Modify your ``settings.py`` to integrate ``python3-logstash`` with Django's logging:: - - LOGGING = { - ... - 'handlers': { - 'logstash': { - 'level': 'DEBUG', - 'class': 'logstash.LogstashHandler', - 'host': 'localhost', - 'port': 5959, # Default value: 5959 - 'version': 1, # Version of logstash event schema. Default value: 0 (for backward compatibility of the library) - 'message_type': 'logstash', # 'type' field in logstash message. Default value: 'logstash'. - 'fqdn': False, # Fully qualified domain name. Default value: false. - 'tags': ['tag1', 'tag2'], # list of tags. Default: None. - }, - }, - 'loggers': { - 'django.request': { - 'handlers': ['logstash'], - 'level': 'DEBUG', - 'propagate': True, - }, - }, - ... - } - - -Using with Gunicorn -=================== - -Create a logging.conf similar to this: - -[loggers] -keys=root, logstash.error, logstash.access - -[handlers] -keys=console , logstash - -[formatters] -keys=generic, access, json - -[logger_root] -level=INFO -handlers=console - -[logger_logstash.error] -level=INFO -handlers=logstash -propagate=1 -qualname=gunicorn.error - -[logger_logstash.access] -level=INFO -handlers=logstash -propagate=0 -qualname=gunicorn.access - -[handler_console] -class=logging.StreamHandler -formatter=generic -args=(sys.stdout, ) - -[handler_logstash] -class=logstash.TCPLogstashHandler -formatter=json -args=('localhost',5959) - -[formatter_generic] -format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s -datefmt=%Y-%m-%d %H:%M:%S -class=logging.Formatter - -[formatter_access] -format=%(message)s -class=logging.Formatter - -[formatter_json] -class=jsonlogging.JSONFormatter - -** Note that I am using the jsonlogging module to parse the gunicorn logs ** - -Sample Logstash Configuration: -============================== - -``logstash.conf`` for Receiving Events from python3-logstash is:: - - input { - tcp { - port => 5000 - codec => json - } - } - output { - stdout { - codec => rubydebug - } - } - diff --git a/logstash/__init__.py b/logstash/__init__.py index 60f10b3..02821d8 100644 --- a/logstash/__init__.py +++ b/logstash/__init__.py @@ -1,10 +1,12 @@ - from logstash.formatter import LogstashFormatterVersion0, LogstashFormatterVersion1 - +from logstash.handler_http import HTTPLogstashHandler from logstash.handler_tcp import TCPLogstashHandler -from logstash.handler_udp import UDPLogstashHandler, LogstashHandler +from logstash.handler_udp import UDPLogstashHandler + try: from logstash.handler_amqp import AMQPLogstashHandler except: - # you need to install AMQP support to enable this handler. - pass + # you need to install AMQP support to enable this handler. + pass + +__version__ = "0.5.1" diff --git a/logstash/formatter.py b/logstash/formatter.py index adb7416..ac130e8 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -1,3 +1,4 @@ +import inspect import logging import socket import traceback @@ -12,12 +13,31 @@ class LogstashFormatterBase(logging.Formatter): # The list contains all the attributes listed in # http://docs.python.org/library/logging.html#logrecord-attributes - skip_list = ( - "args", "asctime", "created", "exc_info", "exc_text", "filename", - "funcName", "id", "levelname", "levelno", "lineno", "module", - "msecs", "msecs", "message", "msg", "name", "pathname", "process", - "processName", "relativeCreated", "thread", "threadName", "extra" - ) + skip_list = { + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "id", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "thread", + "threadName", + "extra", + } easy_types = (str, bool, float, int, type(None)) @@ -47,8 +67,30 @@ def get_extra_fields(self, record): if key not in self.skip_list: fields[key] = self.simplify(value) + frame = self.get_frame(record) + if frame: + cls = self.get_class(frame) + if cls: + fields["class_name"] = cls.__module__ + "." + cls.__name__ + return fields + @staticmethod + def get_frame(record: logging.LogRecord): + frame = inspect.currentframe() + while frame: + frame = frame.f_back + frameinfo = inspect.getframeinfo(frame) + if frameinfo.filename == record.pathname: + return frame + + @staticmethod + def get_class(frame): + if "self" in frame.f_locals: + return type(frame.f_locals["self"]) + elif "cls" in frame.f_locals: + return frame.f_locals["cls"] + def get_debug_fields(self, record): fields = { "stack_trace": self.format_exception(record.exc_info), @@ -87,11 +129,16 @@ def format_exception(cls, exc_info): def serialize(cls, message): return json.dumps(message) + def get_message(self, record: logging.LogRecord) -> dict: + raise NotImplementedError() + + def format(self, record: logging.LogRecord) -> str: + message = self.get_message(record) + return self.serialize(message) -class LogstashFormatterVersion0(LogstashFormatterBase): - version = 0 - def format(self, record): +class LogstashFormatterVersion0(LogstashFormatterBase): + def get_message(self, record): # Create message dict message = { "@timestamp": self.format_timestamp(record.created), @@ -116,15 +163,14 @@ def format(self, record): if record.exc_info: message["@fields"].update(self.get_debug_fields(record)) - return self.serialize(message) + return message class LogstashFormatterVersion1(LogstashFormatterBase): - def format(self, record): + def get_message(self, record): # Create message dict message = { "@timestamp": self.format_timestamp(record.created), - "@version": "1", "message": record.getMessage(), "host": self.host, "path": record.pathname, @@ -142,4 +188,7 @@ def format(self, record): if record.exc_info: message.update(self.get_debug_fields(record)) - return self.serialize(message) + return message + + +versions = {0: LogstashFormatterVersion0, 1: LogstashFormatterVersion1} diff --git a/logstash/handler_amqp.py b/logstash/handler_amqp.py index 930fc88..9a37fcc 100644 --- a/logstash/handler_amqp.py +++ b/logstash/handler_amqp.py @@ -39,12 +39,24 @@ class AMQPLogstashHandler(SocketHandler): record.name will be passed as `logger` parameter. """ - def __init__(self, host='localhost', port=5672, username='guest', - password='guest', exchange='logstash', exchange_type='fanout', - virtual_host='/', message_type='logstash', tags=None, - durable=False, version=0, extra_fields=True, fqdn=False, - facility=None, exchange_routing_key=''): - + def __init__( + self, + host="localhost", + port=5672, + username="guest", + password="guest", + exchange="logstash", + exchange_type="fanout", + virtual_host="/", + message_type="logstash", + tags=None, + durable=False, + version=0, + extra_fields=True, + fqdn=False, + facility=None, + exchange_routing_key="", + ): # AMQP parameters self.host = host @@ -61,8 +73,7 @@ def __init__(self, host='localhost', port=5672, username='guest', # Extract Logstash paramaters self.tags = tags or [] - fn = formatter.LogstashFormatterVersion1 if version == 1 \ - else formatter.LogstashFormatterVersion0 + fn = formatter.versions[version] self.formatter = fn(message_type, tags, fqdn) # Standard logging parameters @@ -72,51 +83,58 @@ def __init__(self, host='localhost', port=5672, username='guest', def makeSocket(self, **kwargs): - return PikaSocket(self.host, - self.port, - self.username, - self.password, - self.virtual_host, - self.exchange, - self.routing_key, - self.exchange_is_durable, - self.exchange_type) + return PikaSocket( + self.host, + self.port, + self.username, + self.password, + self.virtual_host, + self.exchange, + self.routing_key, + self.exchange_is_durable, + self.exchange_type, + ) def makePickle(self, record): return self.formatter.format(record) class PikaSocket(object): - - def __init__(self, host, port, username, password, virtual_host, exchange, - routing_key, durable, exchange_type): + def __init__( + self, + host, + port, + username, + password, + virtual_host, + exchange, + routing_key, + durable, + exchange_type, + ): # create connection parameters credentials = pika.PlainCredentials(username, password) - parameters = pika.ConnectionParameters(host, port, virtual_host, - credentials) + parameters = pika.ConnectionParameters(host, port, virtual_host, credentials) # create connection & channel self.connection = pika.BlockingConnection(parameters) self.channel = self.connection.channel() # create an exchange, if needed - self.channel.exchange_declare(exchange=exchange, - exchange_type=exchange_type, - durable=durable) + self.channel.exchange_declare( + exchange=exchange, exchange_type=exchange_type, durable=durable + ) # needed when publishing self.spec = pika.spec.BasicProperties(delivery_mode=2) self.routing_key = routing_key self.exchange = exchange - def sendall(self, data): - - self.channel.basic_publish(self.exchange, - self.routing_key, - data, - properties=self.spec) + self.channel.basic_publish( + self.exchange, self.routing_key, data, properties=self.spec + ) def close(self): try: diff --git a/logstash/handler_http.py b/logstash/handler_http.py new file mode 100644 index 0000000..2241c7c --- /dev/null +++ b/logstash/handler_http.py @@ -0,0 +1,67 @@ +import logging +import urllib +from base64 import b64encode +from urllib.request import Request + +from logstash import formatter + + +class HTTPLogstashHandler(logging.Handler): + """Python logging handler for Logstash. Sends events over TCP. + :param url: Logstash url. + :param message_type: The type of the message (default logstash). + :param fqdn; Indicates whether to show fully qualified domain name or not (default False). + :param version: version of logstash event schema (default is 0). + :param tags: list of tags for a logger (default is None). + """ + + def __init__( + self, + url: str, + message_type="logstash", + tags=None, + fqdn=False, + version=0, + username=None, + password=None, + ): + super().__init__() + self.url = url + self.username = username + self.password = password + self.formatter = formatter.versions[version](message_type, tags, fqdn) + + def makePickle(self, record): + return b'json='+str.encode(self.formatter.format(record)) + + def emit(self, record): + """ + Emit a record. + + Pickles the record and writes it to the socket in binary format. + If there is an error with the socket, silently drop the packet. + If there was a problem with the socket, re-establishes the + socket. + """ + try: + s = self.makePickle(record) + self.send(s) + except Exception: + self.handleError(record) + + def send(self, data: bytes): + headers = {"Content-Type": "application/x-www-form-urlencoded"} + if self.username and self.password: + basic = b64encode(f"{self.username}:{self.password}".encode()).decode("utf-8") + headers["authorization"] = f"Basic {basic}" + + httprequest = Request(self.url, data=data, headers=headers, method="POST") + + with urllib.request.urlopen(httprequest) as httpresponse: + pass + # status = httpresponse.status + # body = httpresponse.read().decode( + # httpresponse.headers.get_content_charset("utf-8") + # ) + # if status != 200: + # raise Exception(body) diff --git a/logstash/handler_tcp.py b/logstash/handler_tcp.py index 476a767..ec57894 100644 --- a/logstash/handler_tcp.py +++ b/logstash/handler_tcp.py @@ -12,12 +12,11 @@ class TCPLogstashHandler(SocketHandler): :param tags: list of tags for a logger (default is None). """ - def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0): + def __init__( + self, host, port=5959, message_type="logstash", tags=None, fqdn=False, version=0 + ): super(TCPLogstashHandler, self).__init__(host, port) - if version == 1: - self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn) - else: - self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn) + self.formatter = formatter.versions[version](message_type, tags, fqdn) def makePickle(self, record): - return str.encode(self.formatter.format(record)) + b'\n' + return str.encode(self.formatter.format(record)) + b"\n" diff --git a/logstash/handler_udp.py b/logstash/handler_udp.py index ec87a79..144eebd 100644 --- a/logstash/handler_udp.py +++ b/logstash/handler_udp.py @@ -1,4 +1,5 @@ from logging.handlers import DatagramHandler + from logstash.handler_tcp import TCPLogstashHandler @@ -14,7 +15,3 @@ class UDPLogstashHandler(TCPLogstashHandler, DatagramHandler): def makePickle(self, record): return self.formatter.format(record) - - -# For backward compatibility -LogstashHandler = UDPLogstashHandler diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 7982e2e..ac99ad5 --- a/setup.py +++ b/setup.py @@ -1,22 +1,37 @@ +#!/usr/bin/env python3 +import os from distutils.core import setup + +from logstash import __version__ + +readme_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md") +try: + from m2r import parse_from_file + + long_description = parse_from_file(readme_file) +except ImportError: + # m2r may not be installed in user environment + with open(readme_file) as f: + long_description = f.read() + setup( - name='python3-logstash', - packages=['logstash'], - version='0.4.81', - description='Python logging handler for Logstash.', - long_description=open('README.md').read(), - author='Israel Flores', - author_email='jobs@israelfl.com', - url='https://github.com/israel-fl/python3-logstash', + name="logstash-sync", + packages=["logstash"], + version=__version__, + description="Python logging handler for Logstash.", + long_description=long_description, + long_description_content_type="text/markdown", + author="Sergey Yorsh", + author_email="myrik260138@tut.by", + url="https://github.com/MyrikLD/python3-logstash", classifiers=[ - 'Development Status :: 1 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Logging', - ] + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Logging", + ], )