From 412c89a6c403c3150cf8f18d29abf23a2fc94260 Mon Sep 17 00:00:00 2001 From: Joshua Zenn Date: Wed, 6 Apr 2022 16:22:41 -0400 Subject: [PATCH 1/3] #46 Stdout logging module --- test/console_me_tests.py | 60 +++++++++++++++++++++ watergrid/metrics/ConsoleMetricsExporter.py | 25 +++++++++ 2 files changed, 85 insertions(+) create mode 100644 test/console_me_tests.py create mode 100644 watergrid/metrics/ConsoleMetricsExporter.py diff --git a/test/console_me_tests.py b/test/console_me_tests.py new file mode 100644 index 0000000..af827d2 --- /dev/null +++ b/test/console_me_tests.py @@ -0,0 +1,60 @@ +import unittest + +from watergrid.context import DataContext +from watergrid.metrics.ConsoleMetricsExporter import ConsoleMetricsExporter +from watergrid.pipelines.pipeline import Pipeline +from watergrid.steps import Step + + +class MockStep(Step): + def __init__(self, throw_exception=False): + self.throw_exception = throw_exception + super().__init__(self.__class__.__name__) + + def run(self, context: DataContext): + if self.throw_exception: + raise Exception("MockStep failed") + + +class ConsoleMetricsExporterTestCase(unittest.TestCase): + def test_outputs_pipeline_start(self): + pipeline = Pipeline("test_pipeline") + pipeline.add_metrics_exporter(ConsoleMetricsExporter()) + with self.assertLogs() as captured: + pipeline.run() + self.assertIn("INFO:root:Starting pipeline: test_pipeline", captured.output) + + def test_outputs_pipeline_end(self): + pipeline = Pipeline("test_pipeline") + pipeline.add_metrics_exporter(ConsoleMetricsExporter()) + with self.assertLogs() as captured: + pipeline.run() + self.assertIn("INFO:root:Ending pipeline", captured.output) + + def test_outputs_pipeline_end_with_error(self): + pipeline = Pipeline("test_pipeline") + pipeline.add_metrics_exporter(ConsoleMetricsExporter()) + pipeline.add_step(MockStep(throw_exception=True)) + with self.assertLogs() as captured: + pipeline.run() + self.assertIn("ERROR:root:Exception: MockStep failed", captured.output) + + def test_outputs_step_start(self): + pipeline = Pipeline("test_pipeline") + pipeline.add_metrics_exporter(ConsoleMetricsExporter()) + pipeline.add_step(MockStep()) + with self.assertLogs() as captured: + pipeline.run() + self.assertIn("INFO:root:Starting step: MockStep", captured.output) + + def test_outputs_step_end(self): + pipeline = Pipeline("test_pipeline") + pipeline.add_metrics_exporter(ConsoleMetricsExporter()) + pipeline.add_step(MockStep()) + with self.assertLogs() as captured: + pipeline.run() + self.assertIn("INFO:root:Ending step", captured.output) + + +if __name__ == "__main__": + unittest.main() diff --git a/watergrid/metrics/ConsoleMetricsExporter.py b/watergrid/metrics/ConsoleMetricsExporter.py new file mode 100644 index 0000000..11a0921 --- /dev/null +++ b/watergrid/metrics/ConsoleMetricsExporter.py @@ -0,0 +1,25 @@ +import logging + +from watergrid.metrics.MetricsExporter import MetricsExporter + + +class ConsoleMetricsExporter(MetricsExporter): + def __init__(self): + super().__init__() + logging.basicConfig(level=logging.INFO) + self._logger = logging.getLogger(__name__) + + def start_pipeline(self, pipeline_name): + logging.info("Starting pipeline: " + pipeline_name) + + def end_pipeline(self): + logging.info("Ending pipeline") + + def start_step(self, step_name): + logging.info("Starting step: " + step_name) + + def end_step(self): + logging.info("Ending step") + + def capture_exception(self, exception: Exception): + logging.error("Exception: " + str(exception)) \ No newline at end of file From 88013e4414b458ac16f0570d208f569b2f1b478c Mon Sep 17 00:00:00 2001 From: Joshua Zenn Date: Wed, 6 Apr 2022 16:24:47 -0400 Subject: [PATCH 2/3] #46 ConsoleMetricsExporter documentation --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1369e20..51b4d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Added +- `ConsoleMetricsExporter` for locally debugging pipelines without an APM service. ### Changed - Bumped redis dependency to 4.2.2. From 588e67c13de1f7117ebd7ab67804ee7210c3b134 Mon Sep 17 00:00:00 2001 From: Joshua Zenn Date: Thu, 7 Apr 2022 14:18:34 -0400 Subject: [PATCH 3/3] #46 Linter fixes --- watergrid/metrics/ConsoleMetricsExporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watergrid/metrics/ConsoleMetricsExporter.py b/watergrid/metrics/ConsoleMetricsExporter.py index 11a0921..50060ab 100644 --- a/watergrid/metrics/ConsoleMetricsExporter.py +++ b/watergrid/metrics/ConsoleMetricsExporter.py @@ -22,4 +22,4 @@ def end_step(self): logging.info("Ending step") def capture_exception(self, exception: Exception): - logging.error("Exception: " + str(exception)) \ No newline at end of file + logging.error("Exception: " + str(exception))