From 9f3798d6bc8d6dfd287359793c5e1b4851558056 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 29 Jun 2021 10:54:11 -0600 Subject: [PATCH 1/9] Add test case --- tests/test_structlog_formatter.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_structlog_formatter.py b/tests/test_structlog_formatter.py index a0eb6b4..5064a7c 100644 --- a/tests/test_structlog_formatter.py +++ b/tests/test_structlog_formatter.py @@ -21,8 +21,17 @@ from .compat import StringIO +class NotSerializable: + def __repr__(self): + return "" + + def make_event_dict(): - return {"event": "test message", "log.logger": "logger-name"} + return { + "event": "test message", + "log.logger": "logger-name", + "baz": NotSerializable(), + } @mock.patch("time.time") @@ -32,7 +41,9 @@ def test_event_dict_formatted(time, spec_validator): formatter = ecs_logging.StructlogFormatter() assert spec_validator(formatter(None, "debug", make_event_dict())) == ( '{"@timestamp":"2020-03-20T16:16:37.187Z","log.level":"debug",' - '"message":"test message","ecs":{"version":"1.6.0"},' + '"message":"test message",' + '"baz":"",' + '"ecs":{"version":"1.6.0"},' '"log":{"logger":"logger-name"}}' ) From 2a788d77ab2a5c7eab391b571d64ca877c2e7b33 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 29 Jun 2021 10:54:34 -0600 Subject: [PATCH 2/9] Add default/fallback behavior for json.dumps --- ecs_logging/_utils.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/ecs_logging/_utils.py b/ecs_logging/_utils.py index df49ac1..a7a5d48 100644 --- a/ecs_logging/_utils.py +++ b/ecs_logging/_utils.py @@ -151,17 +151,46 @@ def json_dumps(value): # case the given values aren't strings (even though # they should be according to the spec) ordered_json = ",".join( - '"%s":%s' % (k, json.dumps(v, sort_keys=True, separators=(",", ":"))) + '"%s":%s' + % ( + k, + json.dumps( + v, + sort_keys=True, + separators=(",", ":"), + default=_json_dumps_fallback, + ), + ) for k, v in ordered_fields ) if value: return "{%s,%s" % ( ordered_json, - json.dumps(value, sort_keys=True, separators=(",", ":"))[1:], + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + default=_json_dumps_fallback, + )[1:], ) else: return "{%s}" % ordered_json # If there are no fields with ordering requirements we # pass everything into json.dumps() else: - return json.dumps(value, sort_keys=True, separators=(",", ":")) + return json.dumps( + value, sort_keys=True, separators=(",", ":"), default=_json_dumps_fallback + ) + + +def _json_dumps_fallback(value): + # type: (Any) -> Any + """ + Fallback handler for json.dumps to handle objects json doesn't know how to + serialize. + """ + try: + # This is what structlog's json fallback does + return value.__structlog__() + except AttributeError: + return repr(value) From c6ad909bbe8b111280c681010de2675975045e6f Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 29 Jun 2021 11:00:38 -0600 Subject: [PATCH 3/9] CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6867735..9e3390e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +* Add default/fallback handling for json.dumps ([#47](https://github.com/elastic/ecs-logging-python/pull/47)) + ## 1.0.0 (2021-02-08) * Remove "beta" designation From 627a10cd76c188bf71a0021f9dd0f6963751fda5 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 29 Jun 2021 11:27:38 -0600 Subject: [PATCH 4/9] Collapse duplicate json.dumps kwargs --- ecs_logging/_utils.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/ecs_logging/_utils.py b/ecs_logging/_utils.py index a7a5d48..1629cda 100644 --- a/ecs_logging/_utils.py +++ b/ecs_logging/_utils.py @@ -144,6 +144,12 @@ def json_dumps(value): except KeyError: pass + json_dumps_kwargs = { + "sort_keys": True, + "separators": (",", ":"), + "default": _json_dumps_fallback, + } + # Because we want to use 'sorted_keys=True' we manually build # the first three keys and then build the rest with json.dumps() if ordered_fields: @@ -154,33 +160,21 @@ def json_dumps(value): '"%s":%s' % ( k, - json.dumps( - v, - sort_keys=True, - separators=(",", ":"), - default=_json_dumps_fallback, - ), + json.dumps(v, **json_dumps_kwargs), ) for k, v in ordered_fields ) if value: return "{%s,%s" % ( ordered_json, - json.dumps( - value, - sort_keys=True, - separators=(",", ":"), - default=_json_dumps_fallback, - )[1:], + json.dumps(value, **json_dumps_kwargs)[1:], ) else: return "{%s}" % ordered_json # If there are no fields with ordering requirements we # pass everything into json.dumps() else: - return json.dumps( - value, sort_keys=True, separators=(",", ":"), default=_json_dumps_fallback - ) + return json.dumps(value, **json_dumps_kwargs) def _json_dumps_fallback(value): From 41b5f9225fb9af43202e479cf8cf4f85f14c05a0 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Thu, 1 Jul 2021 09:50:34 -0600 Subject: [PATCH 5/9] Update precommit --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18fb3c5..c75db66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 + rev: v4.0.1 hooks: - id: check-case-conflict - id: check-executables-have-shebangs diff --git a/CHANGELOG.md b/CHANGELOG.md index e92dfae..c3bb16b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ ## Unreleased -* Add default/fallback handling for json.dumps ([#47](https://github.com/elastic/ecs-logging-python/pull/47)) * Fixed an issue in `StructlogFormatter` caused by a conflict with `event` (used for the log `message`) and `event.dataset` (a field provided by the `elasticapm` integration) ([#46](https://github.com/elastic/ecs-logging-python/pull/46)) +* Add default/fallback handling for json.dumps ([#47](https://github.com/elastic/ecs-logging-python/pull/47)) ## 1.0.0 (2021-02-08) From ce2c2c59169935220b99646368a5a598ace579a0 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Thu, 1 Jul 2021 09:53:43 -0600 Subject: [PATCH 6/9] Remove whitespace --- tests/test_structlog_formatter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_structlog_formatter.py b/tests/test_structlog_formatter.py index ccb0df4..1e3e04c 100644 --- a/tests/test_structlog_formatter.py +++ b/tests/test_structlog_formatter.py @@ -31,7 +31,6 @@ def __repr__(self): def make_event_dict(): return { "event": "test message", - "event.dataset": "agent.log", "log.logger": "logger-name", "foo": "bar", From b78216ed393c23d8e9a58c167df9eab129982bf5 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Thu, 1 Jul 2021 10:16:11 -0600 Subject: [PATCH 7/9] Add more pre-commit (black, flake8, mypy) --- .pre-commit-config.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c75db66..2ff7af0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,10 +5,24 @@ repos: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - - repo: git@github.com:elastic/apm-pipeline-library rev: current hooks: - id: check-bash-syntax - id: check-jenkins-pipelines - id: check-jjbb +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + args: [--strict] +- repo: https://github.com/ambv/black + rev: 20.8b1 + hooks: + - id: black + language_version: python3 +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + exclude: tests|conftest.py|setup.py \ No newline at end of file From 86a0cf2aec93be7be51036506fce12afff1b12ab Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Thu, 1 Jul 2021 10:23:19 -0600 Subject: [PATCH 8/9] Fix mypy --- ecs_logging/_utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ecs_logging/_utils.py b/ecs_logging/_utils.py index c1f230d..244986f 100644 --- a/ecs_logging/_utils.py +++ b/ecs_logging/_utils.py @@ -16,6 +16,7 @@ # under the License. import json +import functools try: import typing @@ -152,11 +153,9 @@ def json_dumps(value): except KeyError: pass - json_dumps_kwargs = { - "sort_keys": True, - "separators": (",", ":"), - "default": _json_dumps_fallback, - } + json_dumps = functools.partial( + json.dumps, sort_keys=True, separators=(",", ":"), default=_json_dumps_fallback + ) # Because we want to use 'sorted_keys=True' we manually build # the first three keys and then build the rest with json.dumps() @@ -168,21 +167,21 @@ def json_dumps(value): '"%s":%s' % ( k, - json.dumps(v, **json_dumps_kwargs), + json_dumps(v), ) for k, v in ordered_fields ) if value: return "{%s,%s" % ( ordered_json, - json.dumps(value, **json_dumps_kwargs)[1:], + json_dumps(value)[1:], ) else: return "{%s}" % ordered_json # If there are no fields with ordering requirements we # pass everything into json.dumps() else: - return json.dumps(value, **json_dumps_kwargs) + return json_dumps(value) def _json_dumps_fallback(value): From 8cfd4b3aae82d939f9e6802fbe10de587acf815f Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Thu, 1 Jul 2021 10:36:16 -0600 Subject: [PATCH 9/9] Update pre-commit --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ff7af0..19d0369 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,12 +17,12 @@ repos: - id: mypy args: [--strict] - repo: https://github.com/ambv/black - rev: 20.8b1 + rev: 21.6b0 hooks: - id: black language_version: python3 - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + rev: 3.9.2 hooks: - id: flake8 exclude: tests|conftest.py|setup.py \ No newline at end of file