From cb7f2cc71a5a2e2ddddcb57f8a68da7f2b7e022f Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 6 Jun 2019 14:26:08 -0700 Subject: [PATCH 01/21] Add metrics exporter to azure --- README.rst | 5 ++ contrib/opencensus-ext-azure/CHANGELOG.md | 2 + contrib/opencensus-ext-azure/README.rst | 50 ++++++++++++++ .../examples/metrics/metric_exporter.py | 67 +++++++++++++++++++ .../ext/azure/metric_exporter/__init__.py | 48 +++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py create mode 100644 contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py diff --git a/README.rst b/README.rst index 377357978..e22f0901c 100644 --- a/README.rst +++ b/README.rst @@ -237,6 +237,11 @@ Stats Exporter .. _threading: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-threading .. _Zipkin: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-zipkin +Metrics Exporter +-------------- + +- `Azure`_ + Log Exporter -------------- diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index e8241ac9c..f139a7489 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## Unreleased +- Added metrics exporter + ([#](https://github.com/census-instrumentation/opencensus-python/pull/#) ## 0.2.0 Released 2019-05-31 diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 71bc8570c..87e568bc9 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -114,6 +114,56 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr logger.warning('In the span') logger.warning('After the span') +Metric +~~~ + +The **OpenCensus Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Monitor`_. + +Metrics Exporter Usage +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Metrics Exporter Import +************************ + +.. code:: python + + from opencensus.ext.azure import metric_exporter + from opencensus.stats import stats as stats_module + +Metrics Exporter Prerequisites +************************ + +* OpenCensus Python libraries require Python 2.7 or later. +* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. +* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. + +Register the Metrics exporter +********************************** + + .. code:: python + + stats = stats_module.stats + view_manager = stats.view_manager + + exporter = metric_exporter.new_metrics_exporter() + view_manager.register_exporter(exporter) + ... + +Metrics Exporter Code Reference +****************************** + +In the *examples* folder, you can find all the necessary steps to get the exporter, register a view, put tags on the measure, and see the values in `Azure Monitor`_ + +For further details for the Stackdriver implementation, see the folder *stackdriver/stats_exporter/*. + ++---------------------------------------------------------------------------------+-----------------------------------------+ +| Path & File | Short Description | ++=================================================================================+=========================================+ +| contrib/opencensus-ext-azure/examples/metrics | End to end example | ++---------------------------------------------------------------------------------+-----------------------------------------+ +| contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/ | Metrics implementation for Azure Monitor| ++---------------------------------------------------------------------------------+-----------------------------------------+ + References ---------- diff --git a/contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py b/contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py new file mode 100644 index 000000000..fe865f155 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py @@ -0,0 +1,67 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import time + +from opencensus.ext.azure import metric_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module +from opencensus.tags import tag_map as tag_map_module + +# Create the measures +# The latency in milliseconds +m_latency_ms = measure_module.MeasureFloat( + "task_latency", "The task latency in milliseconds", "ms") + +# The stats recorder +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder + +# Create a view in which defines an aggregation and tag keys +latency_view = view_module.View( + "task_latency_distribution", + "The distribution of the task latencies", + [], + m_latency_ms, + # Latency in buckets: [>=0ms, >=100ms, >=200ms, >=400ms, >=1s, >=2s, >=4s] + aggregation_module.DistributionAggregation( + [100.0, 200.0, 400.0, 1000.0, 2000.0, 4000.0])) + + +def main(): + # Enable metrics + # Set the interval in seconds in which you want to send metrics + exporter = metric_exporter.new_metrics_exporter(interval=5) + view_manager.register_exporter(exporter) + + view_manager.register_view(latency_view) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + for i in range(100): + ms = random.random() * 5 * 1000 + print("Latency {0}:{1}".format(i, ms)) + mmap.measure_float_put(m_latency_ms, ms) + mmap.record(tmap) + time.sleep(1) + + print("Done recording metrics") + + +if __name__ == "__main__": + main() diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py new file mode 100644 index 000000000..b2ca2c26b --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import os +import threading + +from opencensus.ext.azure.common import Options +from opencensus.metrics import transport +from opencensus.stats import stats + +__all__ = ['MetricsExporter'] + +class MetricsExporter(object): + """Metrics exporter for Microsoft Azure Monitoring.""" + + def __init__(self, options): + self._options = options + self._md_cache = {} + self._md_lock = threading.Lock() + + @property + def options(self): + return self._options + + def export_metrics(self, metrics): + metrics = list(metrics) + for metric in metrics: + print(repr(metric)) + +def new_metrics_exporter(interval=None, **options): + options = Options(**options) + if not options.instrumentation_key and 1 == 2: + raise ValueError('The instrumentation_key is not provided.') + exporter = MetricsExporter(options=options) + transport.get_exporter_thread(stats.stats, exporter, interval=interval) + return exporter From 41e71a971d668e2dba2e17ca61116a42e2c54282 Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 6 Jun 2019 14:37:51 -0700 Subject: [PATCH 02/21] include changelog --- contrib/opencensus-ext-azure/CHANGELOG.md | 2 +- .../opencensus/ext/azure/metric_exporter/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index f139a7489..2728c486c 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased - Added metrics exporter - ([#](https://github.com/census-instrumentation/opencensus-python/pull/#) + ([678](https://github.com/census-instrumentation/opencensus-python/pull/678) ## 0.2.0 Released 2019-05-31 diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py index b2ca2c26b..23e67b192 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py @@ -41,7 +41,7 @@ def export_metrics(self, metrics): def new_metrics_exporter(interval=None, **options): options = Options(**options) - if not options.instrumentation_key and 1 == 2: + if not options.instrumentation_key: raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) transport.get_exporter_thread(stats.stats, exporter, interval=interval) From ec6f7ea281154e023d8c905b6d1e9e1b664827ac Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 6 Jun 2019 14:56:49 -0700 Subject: [PATCH 03/21] fix lint --- contrib/opencensus-ext-azure/README.rst | 10 +++++----- .../opencensus/ext/azure/metric_exporter/__init__.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 87e568bc9..1eb4f3ff5 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -115,12 +115,12 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr logger.warning('After the span') Metric -~~~ +~~~~~~ The **OpenCensus Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Monitor`_. Metrics Exporter Usage -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ Metrics Exporter Import ************************ @@ -131,14 +131,14 @@ Metrics Exporter Import from opencensus.stats import stats as stats_module Metrics Exporter Prerequisites -************************ +****************************** * OpenCensus Python libraries require Python 2.7 or later. * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. Register the Metrics exporter -********************************** +***************************** .. code:: python @@ -150,7 +150,7 @@ Register the Metrics exporter ... Metrics Exporter Code Reference -****************************** +******************************* In the *examples* folder, you can find all the necessary steps to get the exporter, register a view, put tags on the measure, and see the values in `Azure Monitor`_ diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py index 23e67b192..8af1093de 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime -import os import threading from opencensus.ext.azure.common import Options @@ -25,6 +23,7 @@ class MetricsExporter(object): """Metrics exporter for Microsoft Azure Monitoring.""" + def __init__(self, options): self._options = options self._md_cache = {} @@ -39,6 +38,7 @@ def export_metrics(self, metrics): for metric in metrics: print(repr(metric)) + def new_metrics_exporter(interval=None, **options): options = Options(**options) if not options.instrumentation_key: From 12a0da6a6c249cc5d64a0fcd95e92607e21ec3f5 Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 6 Jun 2019 16:39:34 -0700 Subject: [PATCH 04/21] fix docs, add examples --- README.rst | 2 +- contrib/opencensus-ext-azure/README.rst | 48 +++++++++++------ .../{metric_exporter.py => distribution.py} | 3 +- .../examples/metrics/simple.py | 51 ++++++++++++++++++ .../examples/metrics/sum.py | 54 +++++++++++++++++++ .../ext/azure/metric_exporter/__init__.py | 4 +- 6 files changed, 142 insertions(+), 20 deletions(-) rename contrib/opencensus-ext-azure/examples/metrics/{metric_exporter.py => distribution.py} (96%) create mode 100644 contrib/opencensus-ext-azure/examples/metrics/simple.py create mode 100644 contrib/opencensus-ext-azure/examples/metrics/sum.py diff --git a/README.rst b/README.rst index e22f0901c..c00213ab5 100644 --- a/README.rst +++ b/README.rst @@ -238,7 +238,7 @@ Stats Exporter .. _Zipkin: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-zipkin Metrics Exporter --------------- +---------------- - `Azure`_ diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 1eb4f3ff5..156156993 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -137,38 +137,56 @@ Metrics Exporter Prerequisites * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. -Register the Metrics exporter +Using the Metrics exporter ***************************** .. code:: python + import time + + from opencensus.ext.azure import metric_exporter + from opencensus.stats import aggregation as aggregation_module + from opencensus.stats import measure as measure_module + from opencensus.stats import stats as stats_module + from opencensus.stats import view as view_module + from opencensus.tags import tag_map as tag_map_module + + + # The stats recorder stats = stats_module.stats view_manager = stats.view_manager + stats_recorder = stats.stats_recorder - exporter = metric_exporter.new_metrics_exporter() - view_manager.register_exporter(exporter) - ... + PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "chips") + PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems eaten", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) + + def main(): + # Enable metrics + # Set the interval in seconds in which you want to send metrics + exporter = metric_exporter.new_metrics_exporter(export_interval=5) + view_manager.register_exporter(exporter) -Metrics Exporter Code Reference -******************************* + view_manager.register_view(PROBLEMS_SOLVED_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() -In the *examples* folder, you can find all the necessary steps to get the exporter, register a view, put tags on the measure, and see the values in `Azure Monitor`_ + mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) + mmap.record(tmap) + time.sleep(5) -For further details for the Stackdriver implementation, see the folder *stackdriver/stats_exporter/*. + print("Done recording metrics") -+---------------------------------------------------------------------------------+-----------------------------------------+ -| Path & File | Short Description | -+=================================================================================+=========================================+ -| contrib/opencensus-ext-azure/examples/metrics | End to end example | -+---------------------------------------------------------------------------------+-----------------------------------------+ -| contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/ | Metrics implementation for Azure Monitor| -+---------------------------------------------------------------------------------+-----------------------------------------+ + + if __name__ == "__main__": + main() + ... References ---------- * `Azure Monitor `_ * `Examples `_ +* `Implementation `_ * `OpenCensus Project `_ .. _Azure Monitor: https://docs.microsoft.com/azure/azure-monitor/ diff --git a/contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py b/contrib/opencensus-ext-azure/examples/metrics/distribution.py similarity index 96% rename from contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py rename to contrib/opencensus-ext-azure/examples/metrics/distribution.py index fe865f155..458c3b870 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/metric_exporter.py +++ b/contrib/opencensus-ext-azure/examples/metrics/distribution.py @@ -42,11 +42,10 @@ aggregation_module.DistributionAggregation( [100.0, 200.0, 400.0, 1000.0, 2000.0, 4000.0])) - def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metric_exporter.new_metrics_exporter(interval=5) + exporter = metric_exporter.new_metrics_exporter(export_interval=5) view_manager.register_exporter(exporter) view_manager.register_view(latency_view) diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py new file mode 100644 index 000000000..7efbaf56f --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -0,0 +1,51 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from opencensus.ext.azure import metric_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module +from opencensus.tags import tag_map as tag_map_module + + +# The stats recorder +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder + +PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "chips") +PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems eaten", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) + +def main(): + # Enable metrics + # Set the interval in seconds in which you want to send metrics + exporter = metric_exporter.new_metrics_exporter() + view_manager.register_exporter(exporter) + + view_manager.register_view(PROBLEMS_SOLVED_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) + mmap.record(tmap) + time.sleep(5) + + print("Done recording metrics") + + +if __name__ == "__main__": + main() diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py new file mode 100644 index 000000000..be3ed4b64 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -0,0 +1,54 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import time + +from opencensus.ext.azure import metric_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module +from opencensus.tags import tag_map as tag_map_module + + +# The stats recorder +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder + +CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", "number of chips eaten", "chips") +CHIPS_EATEN_VIEW = view_module.View('chips_eaten_view', "number of chips eaten", [], CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) + +def main(): + # Enable metrics + # Set the interval in seconds in which you want to send metrics + exporter = metric_exporter.new_metrics_exporter(export_interval=5) + view_manager.register_exporter(exporter) + + view_manager.register_view(CHIPS_EATEN_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + for i in range(100): + print(i) + mmap.measure_int_put(CHIPS_EATEN_MEASURE, 1) + mmap.record(tmap) + time.sleep(1) + + print("Done recording metrics") + + +if __name__ == "__main__": + main() diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py index 8af1093de..262f03596 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py @@ -39,10 +39,10 @@ def export_metrics(self, metrics): print(repr(metric)) -def new_metrics_exporter(interval=None, **options): +def new_metrics_exporter(**options): options = Options(**options) if not options.instrumentation_key: raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) - transport.get_exporter_thread(stats.stats, exporter, interval=interval) + transport.get_exporter_thread(stats.stats, exporter, interval=options.export_interval) return exporter From 54f8945a1fa7d991b1bb404bc394650047d0d6f9 Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 6 Jun 2019 17:06:09 -0700 Subject: [PATCH 05/21] fix naming --- contrib/opencensus-ext-azure/README.rst | 11 +---------- .../opencensus/ext/azure/metric_exporter/__init__.py | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 156156993..c5c77aade 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -122,18 +122,9 @@ The **OpenCensus Azure Monitor Metrics Exporter** allows you to export metrics t Metrics Exporter Usage ~~~~~~~~~~~~~~~~~~~~~~ -Metrics Exporter Import -************************ - -.. code:: python - - from opencensus.ext.azure import metric_exporter - from opencensus.stats import stats as stats_module - Metrics Exporter Prerequisites ****************************** -* OpenCensus Python libraries require Python 2.7 or later. * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. @@ -179,7 +170,7 @@ Using the Metrics exporter if __name__ == "__main__": main() - ... + References ---------- diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py index 262f03596..55c7cbbc6 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py @@ -21,7 +21,7 @@ __all__ = ['MetricsExporter'] class MetricsExporter(object): - """Metrics exporter for Microsoft Azure Monitoring.""" + """Metrics exporter for Microsoft Azure Monitor.""" def __init__(self, options): From 93af5756e9b0cffb93a49a4c257d6e2e2d4a35ae Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 7 Jun 2019 10:58:23 -0700 Subject: [PATCH 06/21] Fix examples --- contrib/opencensus-ext-azure/README.rst | 4 ++-- contrib/opencensus-ext-azure/examples/metrics/simple.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index c5c77aade..6a992d063 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -148,8 +148,8 @@ Using the Metrics exporter view_manager = stats.view_manager stats_recorder = stats.stats_recorder - PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "chips") - PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems eaten", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) + PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") + PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) def main(): # Enable metrics diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 7efbaf56f..9feeb3b34 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -27,8 +27,8 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "chips") -PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems eaten", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) +PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") +PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) def main(): # Enable metrics From 7ef2ff17ca449a941baa4e1eb5e4b67771ab9d8a Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 10 Jun 2019 14:08:51 -0700 Subject: [PATCH 07/21] copy stats to metrics --- .../examples/metrics/distribution.py | 19 +- .../examples/metrics/simple.py | 22 +- .../examples/metrics/sum.py | 18 +- .../ext/azure/metric_exporter/__init__.py | 4 +- opencensus/metrics/export/aggregation.py | 190 ++++++++ opencensus/metrics/export/aggregation_data.py | 415 ++++++++++++++++++ .../metrics/export/bucket_boundaries.py | 41 ++ .../metrics/export/execution_context.py | 32 ++ opencensus/metrics/export/measure.py | 60 +++ .../metrics/export/measure_to_view_map.py | 148 +++++++ opencensus/metrics/export/measurement_map.py | 119 +++++ opencensus/metrics/export/metric_producer.py | 24 +- opencensus/metrics/export/metric_utils.py | 132 ++++++ opencensus/metrics/export/metrics_recorder.py | 34 ++ opencensus/metrics/export/view.py | 100 +++++ opencensus/metrics/export/view_data.py | 98 +++++ opencensus/metrics/export/view_manager.py | 55 +++ opencensus/stats/stats.py | 3 +- 18 files changed, 1473 insertions(+), 41 deletions(-) create mode 100644 opencensus/metrics/export/aggregation.py create mode 100644 opencensus/metrics/export/aggregation_data.py create mode 100644 opencensus/metrics/export/bucket_boundaries.py create mode 100644 opencensus/metrics/export/execution_context.py create mode 100644 opencensus/metrics/export/measure.py create mode 100644 opencensus/metrics/export/measure_to_view_map.py create mode 100644 opencensus/metrics/export/measurement_map.py create mode 100644 opencensus/metrics/export/metric_utils.py create mode 100644 opencensus/metrics/export/metrics_recorder.py create mode 100644 opencensus/metrics/export/view.py create mode 100644 opencensus/metrics/export/view_data.py create mode 100644 opencensus/metrics/export/view_manager.py diff --git a/contrib/opencensus-ext-azure/examples/metrics/distribution.py b/contrib/opencensus-ext-azure/examples/metrics/distribution.py index 458c3b870..256b2e894 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/distribution.py +++ b/contrib/opencensus-ext-azure/examples/metrics/distribution.py @@ -16,22 +16,21 @@ import time from opencensus.ext.azure import metric_exporter -from opencensus.stats import aggregation as aggregation_module -from opencensus.stats import measure as measure_module -from opencensus.stats import stats as stats_module -from opencensus.stats import view as view_module +from opencensus.metrics.export import aggregation as aggregation_module +from opencensus.metrics.export import measure as measure_module +from opencensus.metrics.export import metric_producer as metric_module +from opencensus.metrics.export import view as view_module from opencensus.tags import tag_map as tag_map_module +metrics = metric_module.metrics +view_manager = metrics.view_manager +metrics_recorder = metrics.metrics_recorder + # Create the measures # The latency in milliseconds m_latency_ms = measure_module.MeasureFloat( "task_latency", "The task latency in milliseconds", "ms") -# The stats recorder -stats = stats_module.stats -view_manager = stats.view_manager -stats_recorder = stats.stats_recorder - # Create a view in which defines an aggregation and tag keys latency_view = view_module.View( "task_latency_distribution", @@ -49,7 +48,7 @@ def main(): view_manager.register_exporter(exporter) view_manager.register_view(latency_view) - mmap = stats_recorder.new_measurement_map() + mmap = metrics_recorder.new_measurement_map() tmap = tag_map_module.TagMap() for i in range(100): diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 9feeb3b34..8df4884df 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -15,17 +15,15 @@ import time from opencensus.ext.azure import metric_exporter -from opencensus.stats import aggregation as aggregation_module -from opencensus.stats import measure as measure_module -from opencensus.stats import stats as stats_module -from opencensus.stats import view as view_module +from opencensus.metrics.export import aggregation as aggregation_module +from opencensus.metrics.export import measure as measure_module +from opencensus.metrics.export import metric_producer as metric_module +from opencensus.metrics.export import view as view_module from opencensus.tags import tag_map as tag_map_module - -# The stats recorder -stats = stats_module.stats -view_manager = stats.view_manager -stats_recorder = stats.stats_recorder +metrics = metric_module.metrics +view_manager = metrics.view_manager +metrics_recorder = metrics.metrics_recorder PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) @@ -33,16 +31,16 @@ def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metric_exporter.new_metrics_exporter() + exporter = metric_exporter.new_metrics_exporter(export_interval = 2) view_manager.register_exporter(exporter) view_manager.register_view(PROBLEMS_SOLVED_VIEW) - mmap = stats_recorder.new_measurement_map() + mmap = metrics_recorder.new_measurement_map() tmap = tag_map_module.TagMap() mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) mmap.record(tmap) - time.sleep(5) + time.sleep(10) print("Done recording metrics") diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index be3ed4b64..46c4fcd80 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -16,17 +16,15 @@ import time from opencensus.ext.azure import metric_exporter -from opencensus.stats import aggregation as aggregation_module -from opencensus.stats import measure as measure_module -from opencensus.stats import stats as stats_module -from opencensus.stats import view as view_module +from opencensus.metrics.export import aggregation as aggregation_module +from opencensus.metrics.export import measure as measure_module +from opencensus.metrics.export import metric_producer as metric_module +from opencensus.metrics.export import view as view_module from opencensus.tags import tag_map as tag_map_module - -# The stats recorder -stats = stats_module.stats -view_manager = stats.view_manager -stats_recorder = stats.stats_recorder +metrics = metric_module.metrics +view_manager = metrics.view_manager +metrics_recorder = metrics.metrics_recorder CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", "number of chips eaten", "chips") CHIPS_EATEN_VIEW = view_module.View('chips_eaten_view', "number of chips eaten", [], CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) @@ -38,7 +36,7 @@ def main(): view_manager.register_exporter(exporter) view_manager.register_view(CHIPS_EATEN_VIEW) - mmap = stats_recorder.new_measurement_map() + mmap = metrics_recorder.new_measurement_map() tmap = tag_map_module.TagMap() for i in range(100): diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py index 55c7cbbc6..552370baf 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py @@ -16,7 +16,7 @@ from opencensus.ext.azure.common import Options from opencensus.metrics import transport -from opencensus.stats import stats +from opencensus.metrics.export import metric_producer __all__ = ['MetricsExporter'] @@ -44,5 +44,5 @@ def new_metrics_exporter(**options): if not options.instrumentation_key: raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) - transport.get_exporter_thread(stats.stats, exporter, interval=options.export_interval) + transport.get_exporter_thread(metric_producer.metrics, exporter, interval=options.export_interval) return exporter diff --git a/opencensus/metrics/export/aggregation.py b/opencensus/metrics/export/aggregation.py new file mode 100644 index 000000000..184e81142 --- /dev/null +++ b/opencensus/metrics/export/aggregation.py @@ -0,0 +1,190 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from opencensus.metrics.export import bucket_boundaries +from opencensus.metrics.export import aggregation_data + + +logger = logging.getLogger(__name__) + + +class Type(object): + """ The type of aggregation function used on a View. + + Attributes: + NONE (int): The aggregation type of the view is 'unknown'. + SUM (int): The aggregation type of the view is 'sum'. + COUNT (int): The aggregation type of the view is 'count'. + DISTRIBUTION (int): The aggregation type of the view is 'distribution'. + LASTVALUE (int): The aggregation type of the view is 'lastvalue'. + """ + NONE = 0 + SUM = 1 + COUNT = 2 + DISTRIBUTION = 3 + LASTVALUE = 4 + + +class BaseAggregation(object): + """Aggregation describes how the data collected is aggregated by type of + aggregation and buckets + + :type buckets: list(:class: '~opencensus.stats.bucket_boundaries. + BucketBoundaries') + :param buckets: list of endpoints if the aggregation represents a + distribution + + :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` + :param aggregation_type: represents the type of this aggregation + + """ + def __init__(self, buckets=None, aggregation_type=Type.NONE): + self._aggregation_type = aggregation_type + self._buckets = buckets or [] + + @property + def aggregation_type(self): + """The aggregation type of the current aggregation""" + return self._aggregation_type + + @property + def buckets(self): + """The buckets of the current aggregation""" + return self._buckets + + +class SumAggregation(BaseAggregation): + """Sum Aggregation escribes that data collected and aggregated with this + method will be summed + + :type sum: int or float + :param sum: the sum of the data collected and aggregated + + + :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` + :param aggregation_type: represents the type of this aggregation + + """ + def __init__(self, sum=None, aggregation_type=Type.SUM): + super(SumAggregation, self).__init__(aggregation_type=aggregation_type) + self._sum = aggregation_data.SumAggregationDataFloat( + sum_data=float(sum or 0)) + self.aggregation_data = self._sum + + @property + def sum(self): + """The sum of the current aggregation""" + return self._sum + + +class CountAggregation(BaseAggregation): + """Describes that the data collected and aggregated with this method will + be turned into a count value + + :type count: int + :param count: represents the count of this aggregation + + :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` + :param aggregation_type: represents the type of this aggregation + + """ + def __init__(self, count=0, aggregation_type=Type.COUNT): + super(CountAggregation, self).__init__( + aggregation_type=aggregation_type) + self._count = aggregation_data.CountAggregationData(count) + self.aggregation_data = self._count + + @property + def count(self): + """The count of the current aggregation""" + return self._count + + +class DistributionAggregation(BaseAggregation): + """Distribution Aggregation indicates that the desired aggregation is a + histogram distribution + + :type boundaries: list(:class:'~opencensus.metrics.export.bucket_boundaries. + BucketBoundaries') + :param boundaries: the bucket endpoints + + :type distribution: histogram + :param distribution: histogram of the values of the population + + :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` + :param aggregation_type: represents the type of this aggregation + + """ + + def __init__(self, + boundaries=None, + distribution=None, + aggregation_type=Type.DISTRIBUTION): + if boundaries: + if not all(boundaries[ii] < boundaries[ii + 1] + for ii in range(len(boundaries) - 1)): + raise ValueError("bounds must be sorted in increasing order") + for ii, bb in enumerate(boundaries): + if bb > 0: + break + else: + ii += 1 + if ii: + logger.warning("Dropping %s non-positive bucket boundaries", + ii) + boundaries = boundaries[ii:] + + super(DistributionAggregation, self).__init__( + buckets=boundaries, aggregation_type=aggregation_type) + self._boundaries = bucket_boundaries.BucketBoundaries(boundaries) + self._distribution = distribution or {} + self.aggregation_data = aggregation_data.DistributionAggregationData( + 0, 0, 0, None, boundaries) + + @property + def boundaries(self): + """The boundaries of the current aggregation""" + return self._boundaries + + @property + def distribution(self): + """The distribution of the current aggregation""" + return self._distribution + + +class LastValueAggregation(BaseAggregation): + """Describes that the data collected with this method will + overwrite the last recorded value + + :type value: long + :param value: represents the value of this aggregation + + :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` + :param aggregation_type: represents the type of this aggregation + + """ + def __init__(self, value=0, aggregation_type=Type.LASTVALUE): + super(LastValueAggregation, self).__init__( + aggregation_type=aggregation_type) + self.aggregation_data = aggregation_data.LastValueAggregationData( + value=value) + self._value = value + + @property + def value(self): + """The current recorded value + """ + return self._value diff --git a/opencensus/metrics/export/aggregation_data.py b/opencensus/metrics/export/aggregation_data.py new file mode 100644 index 000000000..160258df4 --- /dev/null +++ b/opencensus/metrics/export/aggregation_data.py @@ -0,0 +1,415 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging + +from opencensus.metrics.export import point +from opencensus.metrics.export import value +from opencensus.metrics.export import bucket_boundaries + + +logger = logging.getLogger(__name__) + + +class BaseAggregationData(object): + """Aggregation Data represents an aggregated value from a collection + + :type aggregation_data: aggregated value + :param aggregation_data: represents the aggregated value from a collection + + """ + + def __init__(self, aggregation_data): + self._aggregation_data = aggregation_data + + @property + def aggregation_data(self): + """The current aggregation data""" + return self._aggregation_data + + def to_point(self, timestamp): + """Get a Point conversion of this aggregation. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The time to report the point as having been recorded. + + :rtype: :class: `opencensus.metrics.export.point.Point` + :return: a Point with with this aggregation's value and appropriate + value type. + """ + raise NotImplementedError # pragma: NO COVER + + +class SumAggregationDataFloat(BaseAggregationData): + """Sum Aggregation Data is the aggregated data for the Sum aggregation + + :type sum_data: float + :param sum_data: represents the aggregated sum + + """ + + def __init__(self, sum_data): + super(SumAggregationDataFloat, self).__init__(sum_data) + self._sum_data = sum_data + + def __repr__(self): + return ("{}({})" + .format( + type(self).__name__, + self.sum_data, + )) + + def add_sample(self, value, timestamp=None, attachments=None): + """Allows the user to add a sample to the Sum Aggregation Data + The value of the sample is then added to the current sum data + """ + self._sum_data += value + + @property + def sum_data(self): + """The current sum data""" + return self._sum_data + + def to_point(self, timestamp): + """Get a Point conversion of this aggregation. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The time to report the point as having been recorded. + + :rtype: :class: `opencensus.metrics.export.point.Point` + :return: a :class: `opencensus.metrics.export.value.ValueDouble`-valued + Point with value equal to `sum_data`. + """ + return point.Point(value.ValueDouble(self.sum_data), timestamp) + + +class CountAggregationData(BaseAggregationData): + """Count Aggregation Data is the count value of aggregated data + + :type count_data: long + :param count_data: represents the aggregated count + + """ + + def __init__(self, count_data): + super(CountAggregationData, self).__init__(count_data) + self._count_data = count_data + + def __repr__(self): + return ("{}({})" + .format( + type(self).__name__, + self.count_data, + )) + + def add_sample(self, value, timestamp=None, attachments=None): + """Adds a sample to the current Count Aggregation Data and adds 1 to + the count data""" + self._count_data = self._count_data + 1 + + @property + def count_data(self): + """The current count data""" + return self._count_data + + def to_point(self, timestamp): + """Get a Point conversion of this aggregation. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The time to report the point as having been recorded. + + :rtype: :class: `opencensus.metrics.export.point.Point` + :return: a :class: `opencensus.metrics.export.value.ValueLong`-valued + Point with value equal to `count_data`. + """ + return point.Point(value.ValueLong(self.count_data), timestamp) + + +class DistributionAggregationData(BaseAggregationData): + """Distribution Aggregation Data refers to the distribution stats of + aggregated data + + :type mean_data: float + :param mean_data: the mean value of the distribution + + :type count_data: int + :param count_data: the count value of the distribution + + :type sum_of_sqd_deviations: float + :param sum_of_sqd_deviations: the sum of the sqd deviations from the mean + + :type counts_per_bucket: list(int) + :param counts_per_bucket: the number of occurrences per bucket + + :type exemplars: list(Exemplar) + :param: exemplars: the exemplars associated with histogram buckets. + + :type bounds: list(float) + :param bounds: the histogram distribution of the values + + """ + + def __init__(self, + mean_data, + count_data, + sum_of_sqd_deviations, + counts_per_bucket=None, + bounds=None, + exemplars=None): + if bounds is None and exemplars is not None: + raise ValueError + if exemplars is not None and len(exemplars) != len(bounds) + 1: + raise ValueError + + super(DistributionAggregationData, self).__init__(mean_data) + self._mean_data = mean_data + self._count_data = count_data + self._sum_of_sqd_deviations = sum_of_sqd_deviations + + if bounds is None: + bounds = [] + self._exemplars = None + else: + assert bounds == list(sorted(set(bounds))) + assert all(bb > 0 for bb in bounds) + if exemplars is None: + self._exemplars = {ii: None for ii in range(len(bounds) + 1)} + else: + self._exemplars = {ii: ex for ii, ex in enumerate(exemplars)} + self._bounds = (bucket_boundaries.BucketBoundaries(boundaries=bounds) + .boundaries) + + if counts_per_bucket is None: + counts_per_bucket = [0 for ii in range(len(bounds) + 1)] + else: + assert all(cc >= 0 for cc in counts_per_bucket) + assert len(counts_per_bucket) == len(bounds) + 1 + self._counts_per_bucket = counts_per_bucket + + def __repr__(self): + return ("{}({})" + .format( + type(self).__name__, + self.count_data, + )) + + @property + def mean_data(self): + """The current mean data""" + return self._mean_data + + @property + def count_data(self): + """The current count data""" + return self._count_data + + @property + def sum_of_sqd_deviations(self): + """The current sum of squared deviations from the mean""" + return self._sum_of_sqd_deviations + + @property + def counts_per_bucket(self): + """The current counts per bucket for the distribution""" + return self._counts_per_bucket + + @property + def exemplars(self): + """The current counts per bucket for the distribution""" + return self._exemplars + + @property + def bounds(self): + """The current bounds for the distribution""" + return self._bounds + + @property + def sum(self): + """The sum of the current distribution""" + return self._mean_data * self._count_data + + @property + def variance(self): + """The variance of the current distribution""" + if self._count_data <= 1: + return 0 + return self.sum_of_sqd_deviations / (self._count_data - 1) + + def add_sample(self, value, timestamp, attachments): + """Adding a sample to Distribution Aggregation Data""" + self._count_data += 1 + bucket = self.increment_bucket_count(value) + + if attachments is not None and self.exemplars is not None: + self.exemplars[bucket] = Exemplar(value, timestamp, attachments) + if self.count_data == 1: + self._mean_data = value + return + + old_mean = self._mean_data + self._mean_data = self._mean_data + ( + (value - self._mean_data) / self._count_data) + self._sum_of_sqd_deviations = self._sum_of_sqd_deviations + ( + (value - old_mean) * (value - self._mean_data)) + + def increment_bucket_count(self, value): + """Increment the bucket count based on a given value from the user""" + if len(self._bounds) == 0: + self._counts_per_bucket[0] += 1 + return 0 + + for ii, bb in enumerate(self._bounds): + if value < bb: + self._counts_per_bucket[ii] += 1 + return ii + else: + last_bucket_index = len(self._bounds) + self._counts_per_bucket[last_bucket_index] += 1 + return last_bucket_index + + def to_point(self, timestamp): + """Get a Point conversion of this aggregation. + + This method creates a :class: `opencensus.metrics.export.point.Point` + with a :class: `opencensus.metrics.export.value.ValueDistribution` + value, and creates buckets and exemplars for that distribution from the + appropriate classes in the `metrics` package. If the distribution + doesn't have a histogram (i.e. `bounds` is empty) the converted point's + `buckets` attribute will be null. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The time to report the point as having been recorded. + + :rtype: :class: `opencensus.metrics.export.point.Point` + :return: a :class: `opencensus.metrics.export.value.ValueDistribution` + -valued Point. + """ + if self.bounds: + bucket_options = value.BucketOptions(value.Explicit(self.bounds)) + buckets = [None] * len(self.counts_per_bucket) + for ii, count in enumerate(self.counts_per_bucket): + stat_ex = self.exemplars.get(ii) if self.exemplars else None + if stat_ex is not None: + metric_ex = value.Exemplar(stat_ex.value, + stat_ex.timestamp, + copy.copy(stat_ex.attachments)) + buckets[ii] = value.Bucket(count, metric_ex) + else: + buckets[ii] = value.Bucket(count) + + else: + bucket_options = value.BucketOptions() + buckets = None + return point.Point( + value.ValueDistribution( + count=self.count_data, + sum_=self.sum, + sum_of_squared_deviation=self.sum_of_sqd_deviations, + bucket_options=bucket_options, + buckets=buckets + ), + timestamp + ) + + +class LastValueAggregationData(BaseAggregationData): + """ + LastValue Aggregation Data is the value of aggregated data + + :type value: long + :param value: represents the current value + + """ + + def __init__(self, value): + super(LastValueAggregationData, self).__init__(value) + self._value = value + + def __repr__(self): + return ("{}({})" + .format( + type(self).__name__, + self.value, + )) + + def add_sample(self, value, timestamp=None, attachments=None): + """Adds a sample to the current + LastValue Aggregation Data and overwrite + the current recorded value""" + self._value = value + + @property + def value(self): + """The current value recorded""" + return self._value + + def to_point(self, timestamp): + """Get a Point conversion of this aggregation. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The time to report the point as having been recorded. + + :rtype: :class: `opencensus.metrics.export.point.Point` + :return: a :class: `opencensus.metrics.export.value.ValueDouble`-valued + Point. + """ + return point.Point(value.ValueDouble(self.value), timestamp) + + +class Exemplar(object): + """ Exemplar represents an example point that may be used to annotate + aggregated distribution values, associated with a histogram bucket. + + :type value: double + :param value: value of the Exemplar point. + + :type timestamp: time + :param timestamp: the time that this Exemplar's value was recorded. + + :type attachments: dict + :param attachments: the contextual information about the example value. + """ + + def __init__(self, value, timestamp, attachments): + self._value = value + + self._timestamp = timestamp + + if attachments is None: + raise TypeError('attachments should not be empty') + + for key, value in attachments.items(): + if key is None or not isinstance(key, str): + raise TypeError('attachment key should not be ' + 'empty and should be a string') + if value is None or not isinstance(value, str): + raise TypeError('attachment value should not be ' + 'empty and should be a string') + self._attachments = attachments + + @property + def value(self): + """The current value of the Exemplar point""" + return self._value + + @property + def timestamp(self): + """The time that this Exemplar's value was recorded""" + return self._timestamp + + @property + def attachments(self): + """The contextual information about the example value""" + return self._attachments diff --git a/opencensus/metrics/export/bucket_boundaries.py b/opencensus/metrics/export/bucket_boundaries.py new file mode 100644 index 000000000..b03be3f71 --- /dev/null +++ b/opencensus/metrics/export/bucket_boundaries.py @@ -0,0 +1,41 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BucketBoundaries(object): + """The bucket boundaries for a histogram + + :type boundaries: list(float) + :param boundaries: boundaries for the buckets in the underlying histogram + + """ + def __init__(self, boundaries=None): + self._boundaries = list(boundaries or []) + + @property + def boundaries(self): + """the current boundaries""" + return self._boundaries + + def is_valid_boundaries(self, boundaries): + """checks if the boundaries are in ascending order""" + if boundaries is not None: + min_ = boundaries[0] + for value in boundaries: + if value < min_: + return False + else: + min_ = value + return True + return False diff --git a/opencensus/metrics/export/execution_context.py b/opencensus/metrics/export/execution_context.py new file mode 100644 index 000000000..90c76eb36 --- /dev/null +++ b/opencensus/metrics/export/execution_context.py @@ -0,0 +1,32 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opencensus.common.runtime_context import RuntimeContext + +_measure_to_view_map_slot = RuntimeContext.register_slot( + 'measure_to_view_map_metrics', + lambda: {}) + + +def get_measure_to_view_map(): + return RuntimeContext.measure_to_view_map_metrics + + +def set_measure_to_view_map(measure_to_view_map_metrics): + RuntimeContext.measure_to_view_map_metrics = measure_to_view_map_metrics + + +def clear(): + """Clear the context, used in test.""" + _measure_to_view_map_slot.clear() diff --git a/opencensus/metrics/export/measure.py b/opencensus/metrics/export/measure.py new file mode 100644 index 000000000..f17179457 --- /dev/null +++ b/opencensus/metrics/export/measure.py @@ -0,0 +1,60 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseMeasure(object): + """ A measure is the type of metric that is being recorded with + a name, description, and unit + + :type name: str + :param name: string representing the name of the measure + + :type description: str + :param description: a string representing the description of the measure + + :type unit: str + :param unit: the units in which the measure values are measured + + """ + def __init__(self, name, description, unit=None): + self._name = name + self._description = description + self._unit = unit + + @property + def name(self): + """The name of the current measure""" + return self._name + + @property + def description(self): + """The description of the current measure""" + return self._description + + @property + def unit(self): + """The unit of the current measure""" + return self._unit + + +class MeasureInt(BaseMeasure): + """Creates an Integer Measure""" + def __init__(self, name, description, unit=None): + super(MeasureInt, self).__init__(name, description, unit) + + +class MeasureFloat(BaseMeasure): + """Creates a Float Measure""" + def __init__(self, name, description, unit=None): + super(MeasureFloat, self).__init__(name, description, unit) diff --git a/opencensus/metrics/export/measure_to_view_map.py b/opencensus/metrics/export/measure_to_view_map.py new file mode 100644 index 000000000..e90963492 --- /dev/null +++ b/opencensus/metrics/export/measure_to_view_map.py @@ -0,0 +1,148 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +import copy +import logging + +from opencensus.metrics.export import metric_utils +from opencensus.metrics.export import view_data as view_data_module + + +class MeasureToViewMap(object): + """Measure To View Map stores a map from names of Measures to + specific View Datas + + """ + + def __init__(self): + # stores the one-to-many mapping from Measures to View Datas + self._measure_to_view_data_list_map = defaultdict(list) + # stores a map from the registered View names to the Views + self._registered_views = {} + # stores a map from the registered Measure names to the Measures + self._registered_measures = {} + # stores the set of the exported views + self._exported_views = set() + # Stores the registered exporters + self._exporters = [] + + @property + def exported_views(self): + """the current exported views""" + return self._exported_views + + @property + def exporters(self): + """registered exporters""" + return self._exporters + + def get_view(self, view_name, timestamp): + """get the View Data from the given View name""" + view = self._registered_views.get(view_name) + if view is None: + return None + + view_data_list = self._measure_to_view_data_list_map.get( + view.measure.name) + + if not view_data_list: + return None + + for view_data in view_data_list: + if view_data.view.name == view_name: + break + else: + return None + + return self.copy_and_finalize_view_data(view_data) + + def filter_exported_views(self, all_views): + """returns the subset of the given view that should be exported""" + views = set(all_views) + return views + + # TODO: deprecate + def register_view(self, view, timestamp): + """registers the view's measure name to View Datas given a view""" + if len(self.exporters) > 0: + try: + for e in self.exporters: + e.on_register_view(view) + except AttributeError: + pass + + self._exported_views = None + existing_view = self._registered_views.get(view.name) + if existing_view is not None: + if existing_view == view: + # ignore the views that are already registered + return + else: + logging.warning( + "A different view with the same name is already registered" + ) # pragma: NO COVER + measure = view.measure + registered_measure = self._registered_measures.get(measure.name) + if registered_measure is not None and registered_measure != measure: + logging.warning( + "A different measure with the same name is already registered") + self._registered_views[view.name] = view + if registered_measure is None: + self._registered_measures[measure.name] = measure + self._measure_to_view_data_list_map[view.measure.name].append( + view_data_module.ViewData(view=view, start_time=timestamp, + end_time=timestamp)) + + def record(self, tags, measurement_map, timestamp, attachments=None): + """records stats with a set of tags""" + assert all(vv >= 0 for vv in measurement_map.values()) + for measure, value in measurement_map.items(): + if measure != self._registered_measures.get(measure.name): + return + view_datas = [] + for measure_name, view_data_list \ + in self._measure_to_view_data_list_map.items(): + if measure_name == measure.name: + view_datas.extend(view_data_list) + for view_data in view_datas: + view_data.record( + context=tags, value=value, timestamp=timestamp, + attachments=attachments) + + def get_metrics(self, timestamp): + """Get a Metric for each registered view. + + Convert each registered view's associated `ViewData` into a `Metric` to + be exported. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The timestamp to use for metric conversions, usually + the current time. + + :rtype: Iterator[:class: `opencensus.metrics.export.metric.Metric`] + """ + for vdl in self._measure_to_view_data_list_map.values(): + for vd in vdl: + metric = metric_utils.view_data_to_metric(vd, timestamp) + if metric is not None: + yield metric + + # TODO(issue #470): remove this method once we export immutable metrics. + def copy_and_finalize_view_data(self, view_data): + view_data_copy = copy.copy(view_data) + tvdam_copy = copy.deepcopy(view_data.tag_value_aggregation_data_map) + view_data_copy._tag_value_aggregation_data_map = tvdam_copy + view_data_copy.end() + return view_data_copy diff --git a/opencensus/metrics/export/measurement_map.py b/opencensus/metrics/export/measurement_map.py new file mode 100644 index 000000000..1f6e5cfd4 --- /dev/null +++ b/opencensus/metrics/export/measurement_map.py @@ -0,0 +1,119 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from opencensus.common import utils +from opencensus.tags import TagContext + + +logger = logging.getLogger(__name__) + + +class MeasurementMap(object): + """Measurement Map is a map from Measures to measured values + to be recorded at the same time + + :type measure_to_view_map: :class: '~opencensus.metrics.export.measure_to_view_map. + MeasureToViewMap' + :param measure_to_view_map: the measure to view map that will store the + recorded metrics with tags + + :type: attachments: dict + :param attachments: the contextual information about the attachment value. + + """ + def __init__(self, measure_to_view_map, attachments=None): + self._measurement_map = {} + self._measure_to_view_map = measure_to_view_map + self._attachments = attachments + # If the user tries to record a negative value for any measurement, + # refuse to record all measurements from this map. Recording negative + # measurements will become an error in a later release. + self._invalid = False + + @property + def measurement_map(self): + """the current measurement map""" + return self._measurement_map + + @property + def measure_to_view_map(self): + """the current measure to view map for the measurement map""" + return self._measure_to_view_map + + @property + def attachments(self): + """the current contextual information about the attachment value.""" + return self._attachments + + def measure_int_put(self, measure, value): + """associates the measure of type Int with the given value""" + if value < 0: + # Should be an error in a later release. + logger.warning("Cannot record negative values") + self._measurement_map[measure] = value + + def measure_float_put(self, measure, value): + """associates the measure of type Float with the given value""" + if value < 0: + # Should be an error in a later release. + logger.warning("Cannot record negative values") + self._measurement_map[measure] = value + + def measure_put_attachment(self, key, value): + """Associate the contextual information of an Exemplar to this MeasureMap + Contextual information is represented as key - value string pairs. + If this method is called multiple times with the same key, + only the last value will be kept. + """ + if self._attachments is None: + self._attachments = dict() + + if key is None or not isinstance(key, str): + raise TypeError('attachment key should not be ' + 'empty and should be a string') + if value is None or not isinstance(value, str): + raise TypeError('attachment value should not be ' + 'empty and should be a string') + + self._attachments[key] = value + + def record(self, tags=None): + """records all the measures at the same time with a tag_map. + tag_map could either be explicitly passed to the method, or implicitly + read from current runtime context. + """ + if tags is None: + tags = TagContext.get() + if self._invalid: + logger.warning("Measurement map has included negative value " + "measurements, refusing to record") + return + for measure, value in self.measurement_map.items(): + if value < 0: + self._invalid = True + logger.warning("Dropping values, value to record must be " + "non-negative") + logger.info("Measure '{}' has negative value ({}), refusing " + "to record measurements from {}" + .format(measure.name, value, self)) + return + + self.measure_to_view_map.record( + tags=tags, + measurement_map=self.measurement_map, + timestamp=utils.to_iso_str(), + attachments=self.attachments + ) diff --git a/opencensus/metrics/export/metric_producer.py b/opencensus/metrics/export/metric_producer.py index c8edb4c10..49d6d520b 100644 --- a/opencensus/metrics/export/metric_producer.py +++ b/opencensus/metrics/export/metric_producer.py @@ -13,18 +13,30 @@ # limitations under the License. import threading +from datetime import datetime +from opencensus.metrics.export.metrics_recorder import MetricsRecorder +from opencensus.metrics.export.view_manager import ViewManager class MetricProducer(object): - """Produces a set of metrics for export.""" + """MetricProducer defines a View Manager and a Metrics Recorder in order for the + collection of Metrics + """ + + def __init__(self): + self.metrics_recorder = MetricsRecorder() + self.view_manager = ViewManager() def get_metrics(self): - """Get a set of metrics to be exported. + """Get a Metric for each of the view manager's registered views. - :rtype: set(:class: `opencensus.metrics.export.metric.Metric`) - :return: A set of metrics to be exported. + Convert each registered view's associated `ViewData` into a `Metric` to + be exported, using the current time for metric conversions. + + :rtype: Iterator[:class: `opencensus.metrics.export.metric.Metric`] """ - raise NotImplementedError # pragma: NO COVER + return self.view_manager.measure_to_view_map.get_metrics( + datetime.utcnow()) class MetricProducerManager(object): @@ -79,3 +91,5 @@ def get_all(self): with self.mp_lock: mps_copy = set(self.metric_producers) return mps_copy + +metrics = MetricProducer() \ No newline at end of file diff --git a/opencensus/metrics/export/metric_utils.py b/opencensus/metrics/export/metric_utils.py new file mode 100644 index 000000000..109fcf26f --- /dev/null +++ b/opencensus/metrics/export/metric_utils.py @@ -0,0 +1,132 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Utilities to convert view data models to metrics data models. +""" + +from opencensus.metrics import label_value +from opencensus.metrics.export import metric +from opencensus.metrics.export import metric_descriptor +from opencensus.metrics.export import time_series +from opencensus.metrics.export import aggregation as aggregation_module +from opencensus.metrics.export import measure as measure_module + +# To check that an aggregation's reported type matches its class +AGGREGATION_TYPE_MAP = { + aggregation_module.Type.SUM: + aggregation_module.SumAggregation, + aggregation_module.Type.COUNT: + aggregation_module.CountAggregation, + aggregation_module.Type.DISTRIBUTION: + aggregation_module.DistributionAggregation, + aggregation_module.Type.LASTVALUE: + aggregation_module.LastValueAggregation, +} + + +def get_metric_type(measure, aggregation): + """Get the corresponding metric type for the given aggregation type. + + :type measure: (:class: '~opencensus.metrics.export.measure.BaseMeasure') + :param measure: the measure for which to find a metric type + + :type aggregation: (:class: + '~opencensus.metrics.export.aggregation.BaseAggregation') + :param aggregation: the aggregation for which to find a metric type + """ + if aggregation.aggregation_type == aggregation_module.Type.NONE: + raise ValueError("aggregation type must not be NONE") + assert isinstance(aggregation, + AGGREGATION_TYPE_MAP[aggregation.aggregation_type]) + + if aggregation.aggregation_type == aggregation_module.Type.SUM: + if isinstance(measure, measure_module.MeasureInt): + return metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 + elif isinstance(measure, measure_module.MeasureFloat): + return metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE + else: + raise ValueError + elif aggregation.aggregation_type == aggregation_module.Type.COUNT: + return metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 + elif aggregation.aggregation_type == aggregation_module.Type.DISTRIBUTION: + return metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION + elif aggregation.aggregation_type == aggregation_module.Type.LASTVALUE: + if isinstance(measure, measure_module.MeasureInt): + return metric_descriptor.MetricDescriptorType.GAUGE_INT64 + elif isinstance(measure, measure_module.MeasureFloat): + return metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE + else: + raise ValueError + else: + raise AssertionError # pragma: NO COVER + + +def is_gauge(md_type): + """Whether a given MetricDescriptorType value is a gauge. + + :type md_type: int + :param md_type: A MetricDescriptorType enum value. + """ + if md_type not in metric_descriptor.MetricDescriptorType: + raise ValueError # pragma: NO COVER + + return md_type in { + metric_descriptor.MetricDescriptorType.GAUGE_INT64, + metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE, + metric_descriptor.MetricDescriptorType.GAUGE_DISTRIBUTION + } + + +def get_label_values(tag_values): + """Convert an iterable of TagValues into a list of LabelValues. + + :type tag_values: list(:class: `opencensus.tags.tag_value.TagValue`) + :param tag_values: An iterable of TagValues to convert. + + :rtype: list(:class: `opencensus.metrics.label_value.LabelValue`) + :return: A list of LabelValues, converted from TagValues. + """ + return [label_value.LabelValue(tv) for tv in tag_values] + + +def view_data_to_metric(view_data, timestamp): + """Convert a ViewData to a Metric at time `timestamp`. + + :type view_data: :class: `opencensus.stats.view_data.ViewData` + :param view_data: The ViewData to convert. + + :type timestamp: :class: `datetime.datetime` + :param timestamp: The time to set on the metric's point's aggregation, + usually the current time. + + :rtype: :class: `opencensus.metrics.export.metric.Metric` + :return: A converted Metric. + """ + if not view_data.tag_value_aggregation_data_map: + return None + + md = view_data.view.get_metric_descriptor() + + # TODO: implement gauges + if is_gauge(md.type): + ts_start = None # pragma: NO COVER + else: + ts_start = view_data.start_time + + ts_list = [] + for tag_vals, agg_data in view_data.tag_value_aggregation_data_map.items(): + label_values = get_label_values(tag_vals) + point = agg_data.to_point(timestamp) + ts_list.append(time_series.TimeSeries(label_values, [point], ts_start)) + return metric.Metric(md, ts_list) diff --git a/opencensus/metrics/export/metrics_recorder.py b/opencensus/metrics/export/metrics_recorder.py new file mode 100644 index 000000000..f688bd60a --- /dev/null +++ b/opencensus/metrics/export/metrics_recorder.py @@ -0,0 +1,34 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opencensus.metrics.export.measurement_map import MeasurementMap +from opencensus.metrics.export.measure_to_view_map import MeasureToViewMap +from opencensus.metrics.export import execution_context + + +class MetricsRecorder(object): + """Metrics Recorder provides methods to record metrics against tags + + """ + def __init__(self): + if execution_context.get_measure_to_view_map() == {}: + execution_context.set_measure_to_view_map(MeasureToViewMap()) + + self.measure_to_view_map = execution_context.get_measure_to_view_map() + + def new_measurement_map(self): + """Creates a new MeasurementMap in order to record metrics + :returns a MeasurementMap for recording multiple measurements + """ + return MeasurementMap(self.measure_to_view_map) diff --git a/opencensus/metrics/export/view.py b/opencensus/metrics/export/view.py new file mode 100644 index 000000000..dc439c4f9 --- /dev/null +++ b/opencensus/metrics/export/view.py @@ -0,0 +1,100 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import threading + +from opencensus.metrics import label_key +from opencensus.metrics.export import metric_descriptor +from opencensus.metrics.export import metric_utils + + +class View(object): + """A view defines a specific aggregation and a set of tag keys + + :type name: str + :param name: name of the view + + :type description: str + :param description: description of the view + + :type columns: (:class: '~opencensus.tags.tag_key.TagKey') + :param columns: the columns that the tag keys will aggregate on for this + view + + :type measure: :class: '~opencensus.metrics.export.measure.Measure' + :param measure: the measure to be aggregated by the view + + :type aggregation: :class: '~opencensus.metrics.export.aggregation.BaseAggregation' + :param aggregation: the aggregation the view will support + + """ + + def __init__(self, name, description, columns, measure, aggregation): + self._name = name + self._description = description + self._columns = columns + self._measure = measure + self._aggregation = aggregation + + # Cache the converted MetricDescriptor here to avoid creating it each + # time we convert a ViewData that realizes this View into a Metric. + self._md_cache_lock = threading.Lock() + self._metric_descriptor = None + + @property + def name(self): + """the name of the current view""" + return self._name + + @property + def description(self): + """the description of the current view""" + return self._description + + @property + def columns(self): + """the columns of the current view""" + return self._columns + + @property + def measure(self): + """the measure of the current view""" + return self._measure + + @property + def aggregation(self): + """the aggregation of the current view""" + return self._aggregation + + def get_metric_descriptor(self): + """Get a MetricDescriptor for this view. + + Lazily creates a MetricDescriptor for metrics conversion. + + :rtype: :class: + `opencensus.metrics.export.metric_descriptor.MetricDescriptor` + :return: A converted Metric. + """ # noqa + with self._md_cache_lock: + if self._metric_descriptor is None: + self._metric_descriptor = metric_descriptor.MetricDescriptor( + self.name, + self.description, + self.measure.unit, + metric_utils.get_metric_type(self.measure, + self.aggregation), + # TODO: add label key description + [label_key.LabelKey(tk, "") for tk in self.columns]) + return self._metric_descriptor diff --git a/opencensus/metrics/export/view_data.py b/opencensus/metrics/export/view_data.py new file mode 100644 index 000000000..700a1e2e2 --- /dev/null +++ b/opencensus/metrics/export/view_data.py @@ -0,0 +1,98 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from opencensus.common import utils + + +class ViewData(object): + """View Data is the aggregated data for a particular view + + :type view: + :param view: The view associated with this view data + + :type start_time: datetime + :param start_time: the start time for this view data + + :type end_time: datetime + :param end_time: the end time for this view data + + """ + def __init__(self, + view, + start_time, + end_time): + self._view = view + self._start_time = start_time + self._end_time = end_time + self._tag_value_aggregation_data_map = {} + + @property + def view(self): + """the current view in the view data""" + return self._view + + # TODO: `start_time` and `end_time` are sometimes a `datetime` object but + # should always be a `string`. + @property + def start_time(self): + """the current start time in the view data""" + return self._start_time + + @property + def end_time(self): + """the current end time in the view data""" + return self._end_time + + @property + def tag_value_aggregation_data_map(self): + """the current tag value aggregation map in the view data""" + return self._tag_value_aggregation_data_map + + def start(self): + """sets the start time for the view data""" + self._start_time = utils.to_iso_str() + + def end(self): + """sets the end time for the view data""" + self._end_time = utils.to_iso_str() + + def get_tag_values(self, tags, columns): + """function to get the tag values from tags and columns""" + tag_values = [] + i = 0 + while i < len(columns): + tag_key = columns[i] + if tag_key in tags: + tag_values.append(tags.get(tag_key)) + else: + tag_values.append(None) + i += 1 + return tag_values + + def record(self, context, value, timestamp, attachments=None): + """records the view data against context""" + if context is None: + tags = dict() + else: + tags = context.map + tag_values = self.get_tag_values(tags=tags, + columns=self.view.columns) + tuple_vals = tuple(tag_values) + if tuple_vals not in self.tag_value_aggregation_data_map: + self.tag_value_aggregation_data_map[tuple_vals] = copy.deepcopy( + self.view.aggregation.aggregation_data) + self.tag_value_aggregation_data_map.get(tuple_vals).\ + add_sample(value, timestamp, attachments) diff --git a/opencensus/metrics/export/view_manager.py b/opencensus/metrics/export/view_manager.py new file mode 100644 index 000000000..9f034ac03 --- /dev/null +++ b/opencensus/metrics/export/view_manager.py @@ -0,0 +1,55 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opencensus.common import utils +from opencensus.metrics.export.measure_to_view_map import MeasureToViewMap +from opencensus.metrics.export import execution_context + + +class ViewManager(object): + """View Manager allows the registering of Views for collecting metrics + and receiving metrics data as View Data""" + def __init__(self): + self.time = utils.to_iso_str() + if execution_context.get_measure_to_view_map() == {}: + execution_context.set_measure_to_view_map(MeasureToViewMap()) + + self._measure_view_map = execution_context.get_measure_to_view_map() + + @property + def measure_to_view_map(self): + """the current measure to view map for the View Manager""" + return self._measure_view_map + + def register_view(self, view): + """registers the given view""" + self.measure_to_view_map.register_view(view=view, timestamp=self.time) + + def get_view(self, view_name): + """gets the view given the view name """ + return self.measure_to_view_map.get_view(view_name=view_name, + timestamp=self.time) + + def get_all_exported_views(self): + """returns all of the exported views for the current measure to view + map""" + return self.measure_to_view_map.exported_views + + def register_exporter(self, exporter): + """register the exporter""" + self.measure_to_view_map.exporters.append(exporter) + + def unregister_exporter(self, exporter): + """unregister the exporter""" + self.measure_to_view_map.exporters.remove(exporter) diff --git a/opencensus/stats/stats.py b/opencensus/stats/stats.py index b3c4f322b..ee10303f0 100644 --- a/opencensus/stats/stats.py +++ b/opencensus/stats/stats.py @@ -14,12 +14,11 @@ from datetime import datetime -from opencensus.metrics.export.metric_producer import MetricProducer from opencensus.stats.stats_recorder import StatsRecorder from opencensus.stats.view_manager import ViewManager -class _Stats(MetricProducer): +class _Stats(object): """Stats defines a View Manager and a Stats Recorder in order for the collection of Stats """ From ab059f9f3394cdbf2d087715263b0b3367cf624f Mon Sep 17 00:00:00 2001 From: Leighton Date: Mon, 10 Jun 2019 15:01:59 -0700 Subject: [PATCH 08/21] package name --- contrib/opencensus-ext-azure/examples/metrics/simple.py | 4 ++-- .../azure/{metric_exporter => metrics_exporter}/__init__.py | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename contrib/opencensus-ext-azure/opencensus/ext/azure/{metric_exporter => metrics_exporter}/__init__.py (100%) diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 8df4884df..ee8e60064 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -14,7 +14,7 @@ import time -from opencensus.ext.azure import metric_exporter +from opencensus.ext.azure import metrics_exporter from opencensus.metrics.export import aggregation as aggregation_module from opencensus.metrics.export import measure as measure_module from opencensus.metrics.export import metric_producer as metric_module @@ -31,7 +31,7 @@ def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metric_exporter.new_metrics_exporter(export_interval = 2) + exporter = metrics_exporter.new_metrics_exporter(export_interval=2) view_manager.register_exporter(exporter) view_manager.register_view(PROBLEMS_SOLVED_VIEW) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py similarity index 100% rename from contrib/opencensus-ext-azure/opencensus/ext/azure/metric_exporter/__init__.py rename to contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py From fb2f4cf7bcb25de031a46a07a530be45d1259f38 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 11:29:39 -0700 Subject: [PATCH 09/21] switch to stats --- .../examples/metrics/distribution.py | 20 +- .../examples/metrics/simple.py | 16 +- .../examples/metrics/sum.py | 20 +- .../ext/azure/metrics_exporter/__init__.py | 4 +- opencensus/metrics/export/aggregation.py | 190 -------- opencensus/metrics/export/aggregation_data.py | 415 ------------------ .../metrics/export/bucket_boundaries.py | 41 -- .../metrics/export/execution_context.py | 32 -- opencensus/metrics/export/measure.py | 60 --- .../metrics/export/measure_to_view_map.py | 148 ------- opencensus/metrics/export/measurement_map.py | 119 ----- opencensus/metrics/export/metric_producer.py | 25 +- opencensus/metrics/export/metric_utils.py | 132 ------ opencensus/metrics/export/metrics_recorder.py | 34 -- opencensus/metrics/export/view.py | 100 ----- opencensus/metrics/export/view_data.py | 98 ----- opencensus/metrics/export/view_manager.py | 55 --- opencensus/stats/stats.py | 3 +- 18 files changed, 37 insertions(+), 1475 deletions(-) delete mode 100644 opencensus/metrics/export/aggregation.py delete mode 100644 opencensus/metrics/export/aggregation_data.py delete mode 100644 opencensus/metrics/export/bucket_boundaries.py delete mode 100644 opencensus/metrics/export/execution_context.py delete mode 100644 opencensus/metrics/export/measure.py delete mode 100644 opencensus/metrics/export/measure_to_view_map.py delete mode 100644 opencensus/metrics/export/measurement_map.py delete mode 100644 opencensus/metrics/export/metric_utils.py delete mode 100644 opencensus/metrics/export/metrics_recorder.py delete mode 100644 opencensus/metrics/export/view.py delete mode 100644 opencensus/metrics/export/view_data.py delete mode 100644 opencensus/metrics/export/view_manager.py diff --git a/contrib/opencensus-ext-azure/examples/metrics/distribution.py b/contrib/opencensus-ext-azure/examples/metrics/distribution.py index 256b2e894..6da4c9e2b 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/distribution.py +++ b/contrib/opencensus-ext-azure/examples/metrics/distribution.py @@ -15,16 +15,16 @@ import random import time -from opencensus.ext.azure import metric_exporter -from opencensus.metrics.export import aggregation as aggregation_module -from opencensus.metrics.export import measure as measure_module -from opencensus.metrics.export import metric_producer as metric_module -from opencensus.metrics.export import view as view_module +from opencensus.ext.azure import metrics_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module from opencensus.tags import tag_map as tag_map_module -metrics = metric_module.metrics -view_manager = metrics.view_manager -metrics_recorder = metrics.metrics_recorder +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder # Create the measures # The latency in milliseconds @@ -44,11 +44,11 @@ def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metric_exporter.new_metrics_exporter(export_interval=5) + exporter = metrics_exporter.new_metrics_exporter(export_interval=5) view_manager.register_exporter(exporter) view_manager.register_view(latency_view) - mmap = metrics_recorder.new_measurement_map() + mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() for i in range(100): diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index ee8e60064..750a56138 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -15,15 +15,15 @@ import time from opencensus.ext.azure import metrics_exporter -from opencensus.metrics.export import aggregation as aggregation_module -from opencensus.metrics.export import measure as measure_module -from opencensus.metrics.export import metric_producer as metric_module -from opencensus.metrics.export import view as view_module +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module from opencensus.tags import tag_map as tag_map_module -metrics = metric_module.metrics -view_manager = metrics.view_manager -metrics_recorder = metrics.metrics_recorder +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) @@ -35,7 +35,7 @@ def main(): view_manager.register_exporter(exporter) view_manager.register_view(PROBLEMS_SOLVED_VIEW) - mmap = metrics_recorder.new_measurement_map() + mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index 46c4fcd80..6f31eded2 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -15,16 +15,16 @@ import random import time -from opencensus.ext.azure import metric_exporter -from opencensus.metrics.export import aggregation as aggregation_module -from opencensus.metrics.export import measure as measure_module -from opencensus.metrics.export import metric_producer as metric_module -from opencensus.metrics.export import view as view_module +from opencensus.ext.azure import metrics_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module from opencensus.tags import tag_map as tag_map_module -metrics = metric_module.metrics -view_manager = metrics.view_manager -metrics_recorder = metrics.metrics_recorder +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", "number of chips eaten", "chips") CHIPS_EATEN_VIEW = view_module.View('chips_eaten_view', "number of chips eaten", [], CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) @@ -32,11 +32,11 @@ def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metric_exporter.new_metrics_exporter(export_interval=5) + exporter = metrics_exporter.new_metrics_exporter(export_interval=5) view_manager.register_exporter(exporter) view_manager.register_view(CHIPS_EATEN_VIEW) - mmap = metrics_recorder.new_measurement_map() + mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() for i in range(100): diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index 552370baf..55c7cbbc6 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -16,7 +16,7 @@ from opencensus.ext.azure.common import Options from opencensus.metrics import transport -from opencensus.metrics.export import metric_producer +from opencensus.stats import stats __all__ = ['MetricsExporter'] @@ -44,5 +44,5 @@ def new_metrics_exporter(**options): if not options.instrumentation_key: raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) - transport.get_exporter_thread(metric_producer.metrics, exporter, interval=options.export_interval) + transport.get_exporter_thread(stats.stats, exporter, interval=options.export_interval) return exporter diff --git a/opencensus/metrics/export/aggregation.py b/opencensus/metrics/export/aggregation.py deleted file mode 100644 index 184e81142..000000000 --- a/opencensus/metrics/export/aggregation.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from opencensus.metrics.export import bucket_boundaries -from opencensus.metrics.export import aggregation_data - - -logger = logging.getLogger(__name__) - - -class Type(object): - """ The type of aggregation function used on a View. - - Attributes: - NONE (int): The aggregation type of the view is 'unknown'. - SUM (int): The aggregation type of the view is 'sum'. - COUNT (int): The aggregation type of the view is 'count'. - DISTRIBUTION (int): The aggregation type of the view is 'distribution'. - LASTVALUE (int): The aggregation type of the view is 'lastvalue'. - """ - NONE = 0 - SUM = 1 - COUNT = 2 - DISTRIBUTION = 3 - LASTVALUE = 4 - - -class BaseAggregation(object): - """Aggregation describes how the data collected is aggregated by type of - aggregation and buckets - - :type buckets: list(:class: '~opencensus.stats.bucket_boundaries. - BucketBoundaries') - :param buckets: list of endpoints if the aggregation represents a - distribution - - :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - - """ - def __init__(self, buckets=None, aggregation_type=Type.NONE): - self._aggregation_type = aggregation_type - self._buckets = buckets or [] - - @property - def aggregation_type(self): - """The aggregation type of the current aggregation""" - return self._aggregation_type - - @property - def buckets(self): - """The buckets of the current aggregation""" - return self._buckets - - -class SumAggregation(BaseAggregation): - """Sum Aggregation escribes that data collected and aggregated with this - method will be summed - - :type sum: int or float - :param sum: the sum of the data collected and aggregated - - - :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - - """ - def __init__(self, sum=None, aggregation_type=Type.SUM): - super(SumAggregation, self).__init__(aggregation_type=aggregation_type) - self._sum = aggregation_data.SumAggregationDataFloat( - sum_data=float(sum or 0)) - self.aggregation_data = self._sum - - @property - def sum(self): - """The sum of the current aggregation""" - return self._sum - - -class CountAggregation(BaseAggregation): - """Describes that the data collected and aggregated with this method will - be turned into a count value - - :type count: int - :param count: represents the count of this aggregation - - :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - - """ - def __init__(self, count=0, aggregation_type=Type.COUNT): - super(CountAggregation, self).__init__( - aggregation_type=aggregation_type) - self._count = aggregation_data.CountAggregationData(count) - self.aggregation_data = self._count - - @property - def count(self): - """The count of the current aggregation""" - return self._count - - -class DistributionAggregation(BaseAggregation): - """Distribution Aggregation indicates that the desired aggregation is a - histogram distribution - - :type boundaries: list(:class:'~opencensus.metrics.export.bucket_boundaries. - BucketBoundaries') - :param boundaries: the bucket endpoints - - :type distribution: histogram - :param distribution: histogram of the values of the population - - :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - - """ - - def __init__(self, - boundaries=None, - distribution=None, - aggregation_type=Type.DISTRIBUTION): - if boundaries: - if not all(boundaries[ii] < boundaries[ii + 1] - for ii in range(len(boundaries) - 1)): - raise ValueError("bounds must be sorted in increasing order") - for ii, bb in enumerate(boundaries): - if bb > 0: - break - else: - ii += 1 - if ii: - logger.warning("Dropping %s non-positive bucket boundaries", - ii) - boundaries = boundaries[ii:] - - super(DistributionAggregation, self).__init__( - buckets=boundaries, aggregation_type=aggregation_type) - self._boundaries = bucket_boundaries.BucketBoundaries(boundaries) - self._distribution = distribution or {} - self.aggregation_data = aggregation_data.DistributionAggregationData( - 0, 0, 0, None, boundaries) - - @property - def boundaries(self): - """The boundaries of the current aggregation""" - return self._boundaries - - @property - def distribution(self): - """The distribution of the current aggregation""" - return self._distribution - - -class LastValueAggregation(BaseAggregation): - """Describes that the data collected with this method will - overwrite the last recorded value - - :type value: long - :param value: represents the value of this aggregation - - :type aggregation_type: :class:`~opencensus.metrics.export.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - - """ - def __init__(self, value=0, aggregation_type=Type.LASTVALUE): - super(LastValueAggregation, self).__init__( - aggregation_type=aggregation_type) - self.aggregation_data = aggregation_data.LastValueAggregationData( - value=value) - self._value = value - - @property - def value(self): - """The current recorded value - """ - return self._value diff --git a/opencensus/metrics/export/aggregation_data.py b/opencensus/metrics/export/aggregation_data.py deleted file mode 100644 index 160258df4..000000000 --- a/opencensus/metrics/export/aggregation_data.py +++ /dev/null @@ -1,415 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import logging - -from opencensus.metrics.export import point -from opencensus.metrics.export import value -from opencensus.metrics.export import bucket_boundaries - - -logger = logging.getLogger(__name__) - - -class BaseAggregationData(object): - """Aggregation Data represents an aggregated value from a collection - - :type aggregation_data: aggregated value - :param aggregation_data: represents the aggregated value from a collection - - """ - - def __init__(self, aggregation_data): - self._aggregation_data = aggregation_data - - @property - def aggregation_data(self): - """The current aggregation data""" - return self._aggregation_data - - def to_point(self, timestamp): - """Get a Point conversion of this aggregation. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to report the point as having been recorded. - - :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a Point with with this aggregation's value and appropriate - value type. - """ - raise NotImplementedError # pragma: NO COVER - - -class SumAggregationDataFloat(BaseAggregationData): - """Sum Aggregation Data is the aggregated data for the Sum aggregation - - :type sum_data: float - :param sum_data: represents the aggregated sum - - """ - - def __init__(self, sum_data): - super(SumAggregationDataFloat, self).__init__(sum_data) - self._sum_data = sum_data - - def __repr__(self): - return ("{}({})" - .format( - type(self).__name__, - self.sum_data, - )) - - def add_sample(self, value, timestamp=None, attachments=None): - """Allows the user to add a sample to the Sum Aggregation Data - The value of the sample is then added to the current sum data - """ - self._sum_data += value - - @property - def sum_data(self): - """The current sum data""" - return self._sum_data - - def to_point(self, timestamp): - """Get a Point conversion of this aggregation. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to report the point as having been recorded. - - :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a :class: `opencensus.metrics.export.value.ValueDouble`-valued - Point with value equal to `sum_data`. - """ - return point.Point(value.ValueDouble(self.sum_data), timestamp) - - -class CountAggregationData(BaseAggregationData): - """Count Aggregation Data is the count value of aggregated data - - :type count_data: long - :param count_data: represents the aggregated count - - """ - - def __init__(self, count_data): - super(CountAggregationData, self).__init__(count_data) - self._count_data = count_data - - def __repr__(self): - return ("{}({})" - .format( - type(self).__name__, - self.count_data, - )) - - def add_sample(self, value, timestamp=None, attachments=None): - """Adds a sample to the current Count Aggregation Data and adds 1 to - the count data""" - self._count_data = self._count_data + 1 - - @property - def count_data(self): - """The current count data""" - return self._count_data - - def to_point(self, timestamp): - """Get a Point conversion of this aggregation. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to report the point as having been recorded. - - :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a :class: `opencensus.metrics.export.value.ValueLong`-valued - Point with value equal to `count_data`. - """ - return point.Point(value.ValueLong(self.count_data), timestamp) - - -class DistributionAggregationData(BaseAggregationData): - """Distribution Aggregation Data refers to the distribution stats of - aggregated data - - :type mean_data: float - :param mean_data: the mean value of the distribution - - :type count_data: int - :param count_data: the count value of the distribution - - :type sum_of_sqd_deviations: float - :param sum_of_sqd_deviations: the sum of the sqd deviations from the mean - - :type counts_per_bucket: list(int) - :param counts_per_bucket: the number of occurrences per bucket - - :type exemplars: list(Exemplar) - :param: exemplars: the exemplars associated with histogram buckets. - - :type bounds: list(float) - :param bounds: the histogram distribution of the values - - """ - - def __init__(self, - mean_data, - count_data, - sum_of_sqd_deviations, - counts_per_bucket=None, - bounds=None, - exemplars=None): - if bounds is None and exemplars is not None: - raise ValueError - if exemplars is not None and len(exemplars) != len(bounds) + 1: - raise ValueError - - super(DistributionAggregationData, self).__init__(mean_data) - self._mean_data = mean_data - self._count_data = count_data - self._sum_of_sqd_deviations = sum_of_sqd_deviations - - if bounds is None: - bounds = [] - self._exemplars = None - else: - assert bounds == list(sorted(set(bounds))) - assert all(bb > 0 for bb in bounds) - if exemplars is None: - self._exemplars = {ii: None for ii in range(len(bounds) + 1)} - else: - self._exemplars = {ii: ex for ii, ex in enumerate(exemplars)} - self._bounds = (bucket_boundaries.BucketBoundaries(boundaries=bounds) - .boundaries) - - if counts_per_bucket is None: - counts_per_bucket = [0 for ii in range(len(bounds) + 1)] - else: - assert all(cc >= 0 for cc in counts_per_bucket) - assert len(counts_per_bucket) == len(bounds) + 1 - self._counts_per_bucket = counts_per_bucket - - def __repr__(self): - return ("{}({})" - .format( - type(self).__name__, - self.count_data, - )) - - @property - def mean_data(self): - """The current mean data""" - return self._mean_data - - @property - def count_data(self): - """The current count data""" - return self._count_data - - @property - def sum_of_sqd_deviations(self): - """The current sum of squared deviations from the mean""" - return self._sum_of_sqd_deviations - - @property - def counts_per_bucket(self): - """The current counts per bucket for the distribution""" - return self._counts_per_bucket - - @property - def exemplars(self): - """The current counts per bucket for the distribution""" - return self._exemplars - - @property - def bounds(self): - """The current bounds for the distribution""" - return self._bounds - - @property - def sum(self): - """The sum of the current distribution""" - return self._mean_data * self._count_data - - @property - def variance(self): - """The variance of the current distribution""" - if self._count_data <= 1: - return 0 - return self.sum_of_sqd_deviations / (self._count_data - 1) - - def add_sample(self, value, timestamp, attachments): - """Adding a sample to Distribution Aggregation Data""" - self._count_data += 1 - bucket = self.increment_bucket_count(value) - - if attachments is not None and self.exemplars is not None: - self.exemplars[bucket] = Exemplar(value, timestamp, attachments) - if self.count_data == 1: - self._mean_data = value - return - - old_mean = self._mean_data - self._mean_data = self._mean_data + ( - (value - self._mean_data) / self._count_data) - self._sum_of_sqd_deviations = self._sum_of_sqd_deviations + ( - (value - old_mean) * (value - self._mean_data)) - - def increment_bucket_count(self, value): - """Increment the bucket count based on a given value from the user""" - if len(self._bounds) == 0: - self._counts_per_bucket[0] += 1 - return 0 - - for ii, bb in enumerate(self._bounds): - if value < bb: - self._counts_per_bucket[ii] += 1 - return ii - else: - last_bucket_index = len(self._bounds) - self._counts_per_bucket[last_bucket_index] += 1 - return last_bucket_index - - def to_point(self, timestamp): - """Get a Point conversion of this aggregation. - - This method creates a :class: `opencensus.metrics.export.point.Point` - with a :class: `opencensus.metrics.export.value.ValueDistribution` - value, and creates buckets and exemplars for that distribution from the - appropriate classes in the `metrics` package. If the distribution - doesn't have a histogram (i.e. `bounds` is empty) the converted point's - `buckets` attribute will be null. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to report the point as having been recorded. - - :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a :class: `opencensus.metrics.export.value.ValueDistribution` - -valued Point. - """ - if self.bounds: - bucket_options = value.BucketOptions(value.Explicit(self.bounds)) - buckets = [None] * len(self.counts_per_bucket) - for ii, count in enumerate(self.counts_per_bucket): - stat_ex = self.exemplars.get(ii) if self.exemplars else None - if stat_ex is not None: - metric_ex = value.Exemplar(stat_ex.value, - stat_ex.timestamp, - copy.copy(stat_ex.attachments)) - buckets[ii] = value.Bucket(count, metric_ex) - else: - buckets[ii] = value.Bucket(count) - - else: - bucket_options = value.BucketOptions() - buckets = None - return point.Point( - value.ValueDistribution( - count=self.count_data, - sum_=self.sum, - sum_of_squared_deviation=self.sum_of_sqd_deviations, - bucket_options=bucket_options, - buckets=buckets - ), - timestamp - ) - - -class LastValueAggregationData(BaseAggregationData): - """ - LastValue Aggregation Data is the value of aggregated data - - :type value: long - :param value: represents the current value - - """ - - def __init__(self, value): - super(LastValueAggregationData, self).__init__(value) - self._value = value - - def __repr__(self): - return ("{}({})" - .format( - type(self).__name__, - self.value, - )) - - def add_sample(self, value, timestamp=None, attachments=None): - """Adds a sample to the current - LastValue Aggregation Data and overwrite - the current recorded value""" - self._value = value - - @property - def value(self): - """The current value recorded""" - return self._value - - def to_point(self, timestamp): - """Get a Point conversion of this aggregation. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to report the point as having been recorded. - - :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a :class: `opencensus.metrics.export.value.ValueDouble`-valued - Point. - """ - return point.Point(value.ValueDouble(self.value), timestamp) - - -class Exemplar(object): - """ Exemplar represents an example point that may be used to annotate - aggregated distribution values, associated with a histogram bucket. - - :type value: double - :param value: value of the Exemplar point. - - :type timestamp: time - :param timestamp: the time that this Exemplar's value was recorded. - - :type attachments: dict - :param attachments: the contextual information about the example value. - """ - - def __init__(self, value, timestamp, attachments): - self._value = value - - self._timestamp = timestamp - - if attachments is None: - raise TypeError('attachments should not be empty') - - for key, value in attachments.items(): - if key is None or not isinstance(key, str): - raise TypeError('attachment key should not be ' - 'empty and should be a string') - if value is None or not isinstance(value, str): - raise TypeError('attachment value should not be ' - 'empty and should be a string') - self._attachments = attachments - - @property - def value(self): - """The current value of the Exemplar point""" - return self._value - - @property - def timestamp(self): - """The time that this Exemplar's value was recorded""" - return self._timestamp - - @property - def attachments(self): - """The contextual information about the example value""" - return self._attachments diff --git a/opencensus/metrics/export/bucket_boundaries.py b/opencensus/metrics/export/bucket_boundaries.py deleted file mode 100644 index b03be3f71..000000000 --- a/opencensus/metrics/export/bucket_boundaries.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BucketBoundaries(object): - """The bucket boundaries for a histogram - - :type boundaries: list(float) - :param boundaries: boundaries for the buckets in the underlying histogram - - """ - def __init__(self, boundaries=None): - self._boundaries = list(boundaries or []) - - @property - def boundaries(self): - """the current boundaries""" - return self._boundaries - - def is_valid_boundaries(self, boundaries): - """checks if the boundaries are in ascending order""" - if boundaries is not None: - min_ = boundaries[0] - for value in boundaries: - if value < min_: - return False - else: - min_ = value - return True - return False diff --git a/opencensus/metrics/export/execution_context.py b/opencensus/metrics/export/execution_context.py deleted file mode 100644 index 90c76eb36..000000000 --- a/opencensus/metrics/export/execution_context.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from opencensus.common.runtime_context import RuntimeContext - -_measure_to_view_map_slot = RuntimeContext.register_slot( - 'measure_to_view_map_metrics', - lambda: {}) - - -def get_measure_to_view_map(): - return RuntimeContext.measure_to_view_map_metrics - - -def set_measure_to_view_map(measure_to_view_map_metrics): - RuntimeContext.measure_to_view_map_metrics = measure_to_view_map_metrics - - -def clear(): - """Clear the context, used in test.""" - _measure_to_view_map_slot.clear() diff --git a/opencensus/metrics/export/measure.py b/opencensus/metrics/export/measure.py deleted file mode 100644 index f17179457..000000000 --- a/opencensus/metrics/export/measure.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BaseMeasure(object): - """ A measure is the type of metric that is being recorded with - a name, description, and unit - - :type name: str - :param name: string representing the name of the measure - - :type description: str - :param description: a string representing the description of the measure - - :type unit: str - :param unit: the units in which the measure values are measured - - """ - def __init__(self, name, description, unit=None): - self._name = name - self._description = description - self._unit = unit - - @property - def name(self): - """The name of the current measure""" - return self._name - - @property - def description(self): - """The description of the current measure""" - return self._description - - @property - def unit(self): - """The unit of the current measure""" - return self._unit - - -class MeasureInt(BaseMeasure): - """Creates an Integer Measure""" - def __init__(self, name, description, unit=None): - super(MeasureInt, self).__init__(name, description, unit) - - -class MeasureFloat(BaseMeasure): - """Creates a Float Measure""" - def __init__(self, name, description, unit=None): - super(MeasureFloat, self).__init__(name, description, unit) diff --git a/opencensus/metrics/export/measure_to_view_map.py b/opencensus/metrics/export/measure_to_view_map.py deleted file mode 100644 index e90963492..000000000 --- a/opencensus/metrics/export/measure_to_view_map.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict -import copy -import logging - -from opencensus.metrics.export import metric_utils -from opencensus.metrics.export import view_data as view_data_module - - -class MeasureToViewMap(object): - """Measure To View Map stores a map from names of Measures to - specific View Datas - - """ - - def __init__(self): - # stores the one-to-many mapping from Measures to View Datas - self._measure_to_view_data_list_map = defaultdict(list) - # stores a map from the registered View names to the Views - self._registered_views = {} - # stores a map from the registered Measure names to the Measures - self._registered_measures = {} - # stores the set of the exported views - self._exported_views = set() - # Stores the registered exporters - self._exporters = [] - - @property - def exported_views(self): - """the current exported views""" - return self._exported_views - - @property - def exporters(self): - """registered exporters""" - return self._exporters - - def get_view(self, view_name, timestamp): - """get the View Data from the given View name""" - view = self._registered_views.get(view_name) - if view is None: - return None - - view_data_list = self._measure_to_view_data_list_map.get( - view.measure.name) - - if not view_data_list: - return None - - for view_data in view_data_list: - if view_data.view.name == view_name: - break - else: - return None - - return self.copy_and_finalize_view_data(view_data) - - def filter_exported_views(self, all_views): - """returns the subset of the given view that should be exported""" - views = set(all_views) - return views - - # TODO: deprecate - def register_view(self, view, timestamp): - """registers the view's measure name to View Datas given a view""" - if len(self.exporters) > 0: - try: - for e in self.exporters: - e.on_register_view(view) - except AttributeError: - pass - - self._exported_views = None - existing_view = self._registered_views.get(view.name) - if existing_view is not None: - if existing_view == view: - # ignore the views that are already registered - return - else: - logging.warning( - "A different view with the same name is already registered" - ) # pragma: NO COVER - measure = view.measure - registered_measure = self._registered_measures.get(measure.name) - if registered_measure is not None and registered_measure != measure: - logging.warning( - "A different measure with the same name is already registered") - self._registered_views[view.name] = view - if registered_measure is None: - self._registered_measures[measure.name] = measure - self._measure_to_view_data_list_map[view.measure.name].append( - view_data_module.ViewData(view=view, start_time=timestamp, - end_time=timestamp)) - - def record(self, tags, measurement_map, timestamp, attachments=None): - """records stats with a set of tags""" - assert all(vv >= 0 for vv in measurement_map.values()) - for measure, value in measurement_map.items(): - if measure != self._registered_measures.get(measure.name): - return - view_datas = [] - for measure_name, view_data_list \ - in self._measure_to_view_data_list_map.items(): - if measure_name == measure.name: - view_datas.extend(view_data_list) - for view_data in view_datas: - view_data.record( - context=tags, value=value, timestamp=timestamp, - attachments=attachments) - - def get_metrics(self, timestamp): - """Get a Metric for each registered view. - - Convert each registered view's associated `ViewData` into a `Metric` to - be exported. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The timestamp to use for metric conversions, usually - the current time. - - :rtype: Iterator[:class: `opencensus.metrics.export.metric.Metric`] - """ - for vdl in self._measure_to_view_data_list_map.values(): - for vd in vdl: - metric = metric_utils.view_data_to_metric(vd, timestamp) - if metric is not None: - yield metric - - # TODO(issue #470): remove this method once we export immutable metrics. - def copy_and_finalize_view_data(self, view_data): - view_data_copy = copy.copy(view_data) - tvdam_copy = copy.deepcopy(view_data.tag_value_aggregation_data_map) - view_data_copy._tag_value_aggregation_data_map = tvdam_copy - view_data_copy.end() - return view_data_copy diff --git a/opencensus/metrics/export/measurement_map.py b/opencensus/metrics/export/measurement_map.py deleted file mode 100644 index 1f6e5cfd4..000000000 --- a/opencensus/metrics/export/measurement_map.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from opencensus.common import utils -from opencensus.tags import TagContext - - -logger = logging.getLogger(__name__) - - -class MeasurementMap(object): - """Measurement Map is a map from Measures to measured values - to be recorded at the same time - - :type measure_to_view_map: :class: '~opencensus.metrics.export.measure_to_view_map. - MeasureToViewMap' - :param measure_to_view_map: the measure to view map that will store the - recorded metrics with tags - - :type: attachments: dict - :param attachments: the contextual information about the attachment value. - - """ - def __init__(self, measure_to_view_map, attachments=None): - self._measurement_map = {} - self._measure_to_view_map = measure_to_view_map - self._attachments = attachments - # If the user tries to record a negative value for any measurement, - # refuse to record all measurements from this map. Recording negative - # measurements will become an error in a later release. - self._invalid = False - - @property - def measurement_map(self): - """the current measurement map""" - return self._measurement_map - - @property - def measure_to_view_map(self): - """the current measure to view map for the measurement map""" - return self._measure_to_view_map - - @property - def attachments(self): - """the current contextual information about the attachment value.""" - return self._attachments - - def measure_int_put(self, measure, value): - """associates the measure of type Int with the given value""" - if value < 0: - # Should be an error in a later release. - logger.warning("Cannot record negative values") - self._measurement_map[measure] = value - - def measure_float_put(self, measure, value): - """associates the measure of type Float with the given value""" - if value < 0: - # Should be an error in a later release. - logger.warning("Cannot record negative values") - self._measurement_map[measure] = value - - def measure_put_attachment(self, key, value): - """Associate the contextual information of an Exemplar to this MeasureMap - Contextual information is represented as key - value string pairs. - If this method is called multiple times with the same key, - only the last value will be kept. - """ - if self._attachments is None: - self._attachments = dict() - - if key is None or not isinstance(key, str): - raise TypeError('attachment key should not be ' - 'empty and should be a string') - if value is None or not isinstance(value, str): - raise TypeError('attachment value should not be ' - 'empty and should be a string') - - self._attachments[key] = value - - def record(self, tags=None): - """records all the measures at the same time with a tag_map. - tag_map could either be explicitly passed to the method, or implicitly - read from current runtime context. - """ - if tags is None: - tags = TagContext.get() - if self._invalid: - logger.warning("Measurement map has included negative value " - "measurements, refusing to record") - return - for measure, value in self.measurement_map.items(): - if value < 0: - self._invalid = True - logger.warning("Dropping values, value to record must be " - "non-negative") - logger.info("Measure '{}' has negative value ({}), refusing " - "to record measurements from {}" - .format(measure.name, value, self)) - return - - self.measure_to_view_map.record( - tags=tags, - measurement_map=self.measurement_map, - timestamp=utils.to_iso_str(), - attachments=self.attachments - ) diff --git a/opencensus/metrics/export/metric_producer.py b/opencensus/metrics/export/metric_producer.py index 49d6d520b..eddd0ecb8 100644 --- a/opencensus/metrics/export/metric_producer.py +++ b/opencensus/metrics/export/metric_producer.py @@ -13,30 +13,17 @@ # limitations under the License. import threading -from datetime import datetime - -from opencensus.metrics.export.metrics_recorder import MetricsRecorder -from opencensus.metrics.export.view_manager import ViewManager class MetricProducer(object): - """MetricProducer defines a View Manager and a Metrics Recorder in order for the - collection of Metrics - """ - - def __init__(self): - self.metrics_recorder = MetricsRecorder() - self.view_manager = ViewManager() + """Produces a set of metrics for export.""" def get_metrics(self): - """Get a Metric for each of the view manager's registered views. + """Get a set of metrics to be exported. - Convert each registered view's associated `ViewData` into a `Metric` to - be exported, using the current time for metric conversions. - - :rtype: Iterator[:class: `opencensus.metrics.export.metric.Metric`] + :rtype: set(:class: `opencensus.metrics.export.metric.Metric`) + :return: A set of metrics to be exported. """ - return self.view_manager.measure_to_view_map.get_metrics( - datetime.utcnow()) + raise NotImplementedError # pragma: NO COVER class MetricProducerManager(object): @@ -91,5 +78,3 @@ def get_all(self): with self.mp_lock: mps_copy = set(self.metric_producers) return mps_copy - -metrics = MetricProducer() \ No newline at end of file diff --git a/opencensus/metrics/export/metric_utils.py b/opencensus/metrics/export/metric_utils.py deleted file mode 100644 index 109fcf26f..000000000 --- a/opencensus/metrics/export/metric_utils.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Utilities to convert view data models to metrics data models. -""" - -from opencensus.metrics import label_value -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import time_series -from opencensus.metrics.export import aggregation as aggregation_module -from opencensus.metrics.export import measure as measure_module - -# To check that an aggregation's reported type matches its class -AGGREGATION_TYPE_MAP = { - aggregation_module.Type.SUM: - aggregation_module.SumAggregation, - aggregation_module.Type.COUNT: - aggregation_module.CountAggregation, - aggregation_module.Type.DISTRIBUTION: - aggregation_module.DistributionAggregation, - aggregation_module.Type.LASTVALUE: - aggregation_module.LastValueAggregation, -} - - -def get_metric_type(measure, aggregation): - """Get the corresponding metric type for the given aggregation type. - - :type measure: (:class: '~opencensus.metrics.export.measure.BaseMeasure') - :param measure: the measure for which to find a metric type - - :type aggregation: (:class: - '~opencensus.metrics.export.aggregation.BaseAggregation') - :param aggregation: the aggregation for which to find a metric type - """ - if aggregation.aggregation_type == aggregation_module.Type.NONE: - raise ValueError("aggregation type must not be NONE") - assert isinstance(aggregation, - AGGREGATION_TYPE_MAP[aggregation.aggregation_type]) - - if aggregation.aggregation_type == aggregation_module.Type.SUM: - if isinstance(measure, measure_module.MeasureInt): - return metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 - elif isinstance(measure, measure_module.MeasureFloat): - return metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE - else: - raise ValueError - elif aggregation.aggregation_type == aggregation_module.Type.COUNT: - return metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 - elif aggregation.aggregation_type == aggregation_module.Type.DISTRIBUTION: - return metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION - elif aggregation.aggregation_type == aggregation_module.Type.LASTVALUE: - if isinstance(measure, measure_module.MeasureInt): - return metric_descriptor.MetricDescriptorType.GAUGE_INT64 - elif isinstance(measure, measure_module.MeasureFloat): - return metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE - else: - raise ValueError - else: - raise AssertionError # pragma: NO COVER - - -def is_gauge(md_type): - """Whether a given MetricDescriptorType value is a gauge. - - :type md_type: int - :param md_type: A MetricDescriptorType enum value. - """ - if md_type not in metric_descriptor.MetricDescriptorType: - raise ValueError # pragma: NO COVER - - return md_type in { - metric_descriptor.MetricDescriptorType.GAUGE_INT64, - metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE, - metric_descriptor.MetricDescriptorType.GAUGE_DISTRIBUTION - } - - -def get_label_values(tag_values): - """Convert an iterable of TagValues into a list of LabelValues. - - :type tag_values: list(:class: `opencensus.tags.tag_value.TagValue`) - :param tag_values: An iterable of TagValues to convert. - - :rtype: list(:class: `opencensus.metrics.label_value.LabelValue`) - :return: A list of LabelValues, converted from TagValues. - """ - return [label_value.LabelValue(tv) for tv in tag_values] - - -def view_data_to_metric(view_data, timestamp): - """Convert a ViewData to a Metric at time `timestamp`. - - :type view_data: :class: `opencensus.stats.view_data.ViewData` - :param view_data: The ViewData to convert. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to set on the metric's point's aggregation, - usually the current time. - - :rtype: :class: `opencensus.metrics.export.metric.Metric` - :return: A converted Metric. - """ - if not view_data.tag_value_aggregation_data_map: - return None - - md = view_data.view.get_metric_descriptor() - - # TODO: implement gauges - if is_gauge(md.type): - ts_start = None # pragma: NO COVER - else: - ts_start = view_data.start_time - - ts_list = [] - for tag_vals, agg_data in view_data.tag_value_aggregation_data_map.items(): - label_values = get_label_values(tag_vals) - point = agg_data.to_point(timestamp) - ts_list.append(time_series.TimeSeries(label_values, [point], ts_start)) - return metric.Metric(md, ts_list) diff --git a/opencensus/metrics/export/metrics_recorder.py b/opencensus/metrics/export/metrics_recorder.py deleted file mode 100644 index f688bd60a..000000000 --- a/opencensus/metrics/export/metrics_recorder.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from opencensus.metrics.export.measurement_map import MeasurementMap -from opencensus.metrics.export.measure_to_view_map import MeasureToViewMap -from opencensus.metrics.export import execution_context - - -class MetricsRecorder(object): - """Metrics Recorder provides methods to record metrics against tags - - """ - def __init__(self): - if execution_context.get_measure_to_view_map() == {}: - execution_context.set_measure_to_view_map(MeasureToViewMap()) - - self.measure_to_view_map = execution_context.get_measure_to_view_map() - - def new_measurement_map(self): - """Creates a new MeasurementMap in order to record metrics - :returns a MeasurementMap for recording multiple measurements - """ - return MeasurementMap(self.measure_to_view_map) diff --git a/opencensus/metrics/export/view.py b/opencensus/metrics/export/view.py deleted file mode 100644 index dc439c4f9..000000000 --- a/opencensus/metrics/export/view.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import threading - -from opencensus.metrics import label_key -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import metric_utils - - -class View(object): - """A view defines a specific aggregation and a set of tag keys - - :type name: str - :param name: name of the view - - :type description: str - :param description: description of the view - - :type columns: (:class: '~opencensus.tags.tag_key.TagKey') - :param columns: the columns that the tag keys will aggregate on for this - view - - :type measure: :class: '~opencensus.metrics.export.measure.Measure' - :param measure: the measure to be aggregated by the view - - :type aggregation: :class: '~opencensus.metrics.export.aggregation.BaseAggregation' - :param aggregation: the aggregation the view will support - - """ - - def __init__(self, name, description, columns, measure, aggregation): - self._name = name - self._description = description - self._columns = columns - self._measure = measure - self._aggregation = aggregation - - # Cache the converted MetricDescriptor here to avoid creating it each - # time we convert a ViewData that realizes this View into a Metric. - self._md_cache_lock = threading.Lock() - self._metric_descriptor = None - - @property - def name(self): - """the name of the current view""" - return self._name - - @property - def description(self): - """the description of the current view""" - return self._description - - @property - def columns(self): - """the columns of the current view""" - return self._columns - - @property - def measure(self): - """the measure of the current view""" - return self._measure - - @property - def aggregation(self): - """the aggregation of the current view""" - return self._aggregation - - def get_metric_descriptor(self): - """Get a MetricDescriptor for this view. - - Lazily creates a MetricDescriptor for metrics conversion. - - :rtype: :class: - `opencensus.metrics.export.metric_descriptor.MetricDescriptor` - :return: A converted Metric. - """ # noqa - with self._md_cache_lock: - if self._metric_descriptor is None: - self._metric_descriptor = metric_descriptor.MetricDescriptor( - self.name, - self.description, - self.measure.unit, - metric_utils.get_metric_type(self.measure, - self.aggregation), - # TODO: add label key description - [label_key.LabelKey(tk, "") for tk in self.columns]) - return self._metric_descriptor diff --git a/opencensus/metrics/export/view_data.py b/opencensus/metrics/export/view_data.py deleted file mode 100644 index 700a1e2e2..000000000 --- a/opencensus/metrics/export/view_data.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy - -from opencensus.common import utils - - -class ViewData(object): - """View Data is the aggregated data for a particular view - - :type view: - :param view: The view associated with this view data - - :type start_time: datetime - :param start_time: the start time for this view data - - :type end_time: datetime - :param end_time: the end time for this view data - - """ - def __init__(self, - view, - start_time, - end_time): - self._view = view - self._start_time = start_time - self._end_time = end_time - self._tag_value_aggregation_data_map = {} - - @property - def view(self): - """the current view in the view data""" - return self._view - - # TODO: `start_time` and `end_time` are sometimes a `datetime` object but - # should always be a `string`. - @property - def start_time(self): - """the current start time in the view data""" - return self._start_time - - @property - def end_time(self): - """the current end time in the view data""" - return self._end_time - - @property - def tag_value_aggregation_data_map(self): - """the current tag value aggregation map in the view data""" - return self._tag_value_aggregation_data_map - - def start(self): - """sets the start time for the view data""" - self._start_time = utils.to_iso_str() - - def end(self): - """sets the end time for the view data""" - self._end_time = utils.to_iso_str() - - def get_tag_values(self, tags, columns): - """function to get the tag values from tags and columns""" - tag_values = [] - i = 0 - while i < len(columns): - tag_key = columns[i] - if tag_key in tags: - tag_values.append(tags.get(tag_key)) - else: - tag_values.append(None) - i += 1 - return tag_values - - def record(self, context, value, timestamp, attachments=None): - """records the view data against context""" - if context is None: - tags = dict() - else: - tags = context.map - tag_values = self.get_tag_values(tags=tags, - columns=self.view.columns) - tuple_vals = tuple(tag_values) - if tuple_vals not in self.tag_value_aggregation_data_map: - self.tag_value_aggregation_data_map[tuple_vals] = copy.deepcopy( - self.view.aggregation.aggregation_data) - self.tag_value_aggregation_data_map.get(tuple_vals).\ - add_sample(value, timestamp, attachments) diff --git a/opencensus/metrics/export/view_manager.py b/opencensus/metrics/export/view_manager.py deleted file mode 100644 index 9f034ac03..000000000 --- a/opencensus/metrics/export/view_manager.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from opencensus.common import utils -from opencensus.metrics.export.measure_to_view_map import MeasureToViewMap -from opencensus.metrics.export import execution_context - - -class ViewManager(object): - """View Manager allows the registering of Views for collecting metrics - and receiving metrics data as View Data""" - def __init__(self): - self.time = utils.to_iso_str() - if execution_context.get_measure_to_view_map() == {}: - execution_context.set_measure_to_view_map(MeasureToViewMap()) - - self._measure_view_map = execution_context.get_measure_to_view_map() - - @property - def measure_to_view_map(self): - """the current measure to view map for the View Manager""" - return self._measure_view_map - - def register_view(self, view): - """registers the given view""" - self.measure_to_view_map.register_view(view=view, timestamp=self.time) - - def get_view(self, view_name): - """gets the view given the view name """ - return self.measure_to_view_map.get_view(view_name=view_name, - timestamp=self.time) - - def get_all_exported_views(self): - """returns all of the exported views for the current measure to view - map""" - return self.measure_to_view_map.exported_views - - def register_exporter(self, exporter): - """register the exporter""" - self.measure_to_view_map.exporters.append(exporter) - - def unregister_exporter(self, exporter): - """unregister the exporter""" - self.measure_to_view_map.exporters.remove(exporter) diff --git a/opencensus/stats/stats.py b/opencensus/stats/stats.py index ee10303f0..b3c4f322b 100644 --- a/opencensus/stats/stats.py +++ b/opencensus/stats/stats.py @@ -14,11 +14,12 @@ from datetime import datetime +from opencensus.metrics.export.metric_producer import MetricProducer from opencensus.stats.stats_recorder import StatsRecorder from opencensus.stats.view_manager import ViewManager -class _Stats(object): +class _Stats(MetricProducer): """Stats defines a View Manager and a Stats Recorder in order for the collection of Stats """ From dab802f9a5f86133dfc36e577eb35ac4d5b8c1b7 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 13:55:02 -0700 Subject: [PATCH 10/21] fix docs --- README.rst | 2 +- contrib/opencensus-ext-azure/README.rst | 18 +++++++----------- .../examples/metrics/simple.py | 6 ++++-- opencensus/metrics/export/metric_producer.py | 1 + 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index c003fa2f7..1b9b2c202 100644 --- a/README.rst +++ b/README.rst @@ -243,7 +243,7 @@ Stats Exporter .. _threading: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-threading .. _Zipkin: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-zipkin -Metrics Exporter +Stats Exporter ---------------- - `Azure`_ diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 6a992d063..5c1e1bf77 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -119,9 +119,6 @@ Metric The **OpenCensus Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Monitor`_. -Metrics Exporter Usage -~~~~~~~~~~~~~~~~~~~~~~ - Metrics Exporter Prerequisites ****************************** @@ -135,26 +132,26 @@ Using the Metrics exporter import time - from opencensus.ext.azure import metric_exporter + from opencensus.ext.azure import metrics_exporter from opencensus.stats import aggregation as aggregation_module from opencensus.stats import measure as measure_module from opencensus.stats import stats as stats_module from opencensus.stats import view as view_module from opencensus.tags import tag_map as tag_map_module - - # The stats recorder stats = stats_module.stats view_manager = stats.view_manager stats_recorder = stats.stats_recorder - PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") - PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) + PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", + "number of problems solved", "problems") + PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], + PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metric_exporter.new_metrics_exporter(export_interval=5) + exporter = metrics_exporter.new_metrics_exporter(export_interval=2) view_manager.register_exporter(exporter) view_manager.register_view(PROBLEMS_SOLVED_VIEW) @@ -163,7 +160,7 @@ Using the Metrics exporter mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) mmap.record(tmap) - time.sleep(5) + time.sleep(10) print("Done recording metrics") @@ -177,7 +174,6 @@ References * `Azure Monitor `_ * `Examples `_ -* `Implementation `_ * `OpenCensus Project `_ .. _Azure Monitor: https://docs.microsoft.com/azure/azure-monitor/ diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 750a56138..ae2c6cde0 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -25,8 +25,10 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") -PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) +PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", + "number of problems solved", "problems") +PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], + PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) def main(): # Enable metrics diff --git a/opencensus/metrics/export/metric_producer.py b/opencensus/metrics/export/metric_producer.py index eddd0ecb8..c8edb4c10 100644 --- a/opencensus/metrics/export/metric_producer.py +++ b/opencensus/metrics/export/metric_producer.py @@ -14,6 +14,7 @@ import threading + class MetricProducer(object): """Produces a set of metrics for export.""" From 8f04c070a681228541ca753d4e3232e14ada10ed Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 15:53:19 -0700 Subject: [PATCH 11/21] fix docs, connents --- README.rst | 30 +++++++++---------- contrib/opencensus-ext-azure/README.rst | 6 ++-- .../examples/logs/correlated.py | 2 +- .../examples/logs/error.py | 2 +- .../examples/logs/simple.py | 2 +- .../examples/metrics/simple.py | 2 +- .../examples/metrics/sum.py | 2 +- .../__init__.py | 0 .../ext/azure/metrics_exporter/__init__.py | 7 ++--- 9 files changed, 25 insertions(+), 28 deletions(-) rename contrib/opencensus-ext-azure/opencensus/ext/azure/{log_exporter => logs_exporter}/__init__.py (100%) diff --git a/README.rst b/README.rst index 1b9b2c202..77557e5d2 100644 --- a/README.rst +++ b/README.rst @@ -205,14 +205,15 @@ OpenCensus supports integration with popular web frameworks, client libraries an - `SQLAlchemy`_ - `threading`_ -Trace Exporter +Logs Exporter -------------- - `Azure`_ -- `Jaeger`_ -- `OCAgent`_ -- `Stackdriver`_ -- `Zipkin`_ + +Metrics Exporter +---------------- + +- `Azure`_ Stats Exporter -------------- @@ -221,6 +222,15 @@ Stats Exporter - `Prometheus`_ - `Stackdriver`_ +Trace Exporter +-------------- + +- `Azure`_ +- `Jaeger`_ +- `OCAgent`_ +- `Stackdriver`_ +- `Zipkin`_ + .. _Azure: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-azure .. _Django: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-django .. _Flask: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-flask @@ -243,16 +253,6 @@ Stats Exporter .. _threading: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-threading .. _Zipkin: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-zipkin -Stats Exporter ----------------- - -- `Azure`_ - -Log Exporter --------------- - -- `Azure`_ - ------------ Versioning ------------ diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 5c1e1bf77..8af37a358 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -63,7 +63,7 @@ You can also specify the instrumentation key explicitly in the code. with tracer.span(name='parent'): response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') -Log +Logs ~~~ The **Azure Monitor Log Handler** allows you to export Python logs to `Azure Monitor`_. @@ -77,7 +77,7 @@ This example shows how to send a warning level log to Azure Monitor. import logging - from opencensus.ext.azure.log_exporter import AzureLogHandler + from opencensus.ext.azure.logs_exporter import AzureLogHandler logger = logging.getLogger(__name__) logger.addHandler(AzureLogHandler()) @@ -145,7 +145,7 @@ Using the Metrics exporter PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") - PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], + PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) def main(): diff --git a/contrib/opencensus-ext-azure/examples/logs/correlated.py b/contrib/opencensus-ext-azure/examples/logs/correlated.py index 205a46bc1..ffa713656 100644 --- a/contrib/opencensus-ext-azure/examples/logs/correlated.py +++ b/contrib/opencensus-ext-azure/examples/logs/correlated.py @@ -14,7 +14,7 @@ import logging -from opencensus.ext.azure.log_exporter import AzureLogHandler +from opencensus.ext.azure.logs_exporter import AzureLogHandler from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace import config_integration from opencensus.trace.samplers import ProbabilitySampler diff --git a/contrib/opencensus-ext-azure/examples/logs/error.py b/contrib/opencensus-ext-azure/examples/logs/error.py index 4f801342a..c0b299f84 100644 --- a/contrib/opencensus-ext-azure/examples/logs/error.py +++ b/contrib/opencensus-ext-azure/examples/logs/error.py @@ -14,7 +14,7 @@ import logging -from opencensus.ext.azure.log_exporter import AzureLogHandler +from opencensus.ext.azure.logs_exporter import AzureLogHandler logger = logging.getLogger(__name__) # TODO: you need to specify the instrumentation key in the diff --git a/contrib/opencensus-ext-azure/examples/logs/simple.py b/contrib/opencensus-ext-azure/examples/logs/simple.py index cdda2b688..b8075e4a9 100644 --- a/contrib/opencensus-ext-azure/examples/logs/simple.py +++ b/contrib/opencensus-ext-azure/examples/logs/simple.py @@ -14,7 +14,7 @@ import logging -from opencensus.ext.azure.log_exporter import AzureLogHandler +from opencensus.ext.azure.logs_exporter import AzureLogHandler logger = logging.getLogger(__name__) # TODO: you need to specify the instrumentation key in the diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index ae2c6cde0..843f0d510 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -27,7 +27,7 @@ PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", "number of problems solved", "problems") -PROBLEMS_SOLVED_VIEW = view_module.View('problems_solved_view', "number of problems solved", [], +PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", "number of problems solved", [], PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) def main(): diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index 6f31eded2..5ff2296cd 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -27,7 +27,7 @@ stats_recorder = stats.stats_recorder CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", "number of chips eaten", "chips") -CHIPS_EATEN_VIEW = view_module.View('chips_eaten_view', "number of chips eaten", [], CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) +CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", "number of chips eaten", [], CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) def main(): # Enable metrics diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/logs_exporter/__init__.py similarity index 100% rename from contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py rename to contrib/opencensus-ext-azure/opencensus/ext/azure/logs_exporter/__init__.py diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index 55c7cbbc6..39f26022b 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -18,7 +18,7 @@ from opencensus.metrics import transport from opencensus.stats import stats -__all__ = ['MetricsExporter'] +__all__ = ['MetricsExporter', 'new_metrics_exporter'] class MetricsExporter(object): """Metrics exporter for Microsoft Azure Monitor.""" @@ -29,13 +29,10 @@ def __init__(self, options): self._md_cache = {} self._md_lock = threading.Lock() - @property - def options(self): - return self._options - def export_metrics(self, metrics): metrics = list(metrics) for metric in metrics: + # TODO: implement export to Azure Monitor functionality print(repr(metric)) From a1e940fae6a1e03ffdc7e9cd70f9f32f4a036c92 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 16:09:32 -0700 Subject: [PATCH 12/21] reorder README --- contrib/opencensus-ext-azure/README.rst | 110 ++++++++++++------------ 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 8af37a358..78f5ad5a2 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -16,55 +16,8 @@ Installation Usage ----- -Trace -~~~~~ - -The **Azure Monitor Trace Exporter** allows you to export `OpenCensus`_ traces to `Azure Monitor`_. - -This example shows how to send a span "hello" to Azure Monitor. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. -* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. - -.. code:: python - - from opencensus.ext.azure.trace_exporter import AzureExporter - from opencensus.trace.samplers import ProbabilitySampler - from opencensus.trace.tracer import Tracer - - tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) - - with tracer.span(name='hello'): - print('Hello, World!') - -You can also specify the instrumentation key explicitly in the code. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. -* Install the `requests integration package <../opencensus-ext-requests>`_ using ``pip install opencensus-ext-requests``. -* Put the instrumentation key in the following code. - -.. code:: python - - import requests - - from opencensus.ext.azure.trace_exporter import AzureExporter - from opencensus.trace import config_integration - from opencensus.trace.samplers import ProbabilitySampler - from opencensus.trace.tracer import Tracer - - config_integration.trace_integrations(['requests']) - tracer = Tracer( - exporter=AzureExporter( - # TODO: replace this with your own instrumentation key. - instrumentation_key='00000000-0000-0000-0000-000000000000', - ), - sampler=ProbabilitySampler(1.0), - ) - with tracer.span(name='parent'): - response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') - Logs -~~~ +~~~~ The **Azure Monitor Log Handler** allows you to export Python logs to `Azure Monitor`_. @@ -72,6 +25,7 @@ This example shows how to send a warning level log to Azure Monitor. * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. +* You can also specify the instrumentation key explicitly in the code, which will take priority over a set environment variable. .. code:: python @@ -88,6 +42,7 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Install the `logging integration package <../opencensus-ext-logging>`_ using ``pip install opencensus-ext-logging``. * Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. +* You can also specify the instrumentation key explicitly in the code, which will take priority over a set environment variable. .. code:: python @@ -114,16 +69,14 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr logger.warning('In the span') logger.warning('After the span') -Metric -~~~~~~ +Metrics +~~~~~~~ The **OpenCensus Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Monitor`_. -Metrics Exporter Prerequisites -****************************** - * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. +* You can also specify the instrumentation key explicitly in the code, which will take priority over a set environment variable. Using the Metrics exporter ***************************** @@ -167,7 +120,56 @@ Using the Metrics exporter if __name__ == "__main__": main() - + +Trace +~~~~~ + +The **Azure Monitor Trace Exporter** allows you to export `OpenCensus`_ traces to `Azure Monitor`_. + +This example shows how to send a span "hello" to Azure Monitor. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. +* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. +* You can also specify the instrumentation key explicitly in the code, which will take priority over a set environment variable. + +.. code:: python + + from opencensus.ext.azure.trace_exporter import AzureExporter + from opencensus.trace.samplers import ProbabilitySampler + from opencensus.trace.tracer import Tracer + + tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) + + with tracer.span(name='hello'): + print('Hello, World!') + +You can also specify the instrumentation key explicitly in the code. + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. +* Install the `requests integration package <../opencensus-ext-requests>`_ using ``pip install opencensus-ext-requests``. +* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. +* You can also specify the instrumentation key explicitly in the code, which will take priority over a set environment variable. + +.. code:: python + + import requests + + from opencensus.ext.azure.trace_exporter import AzureExporter + from opencensus.trace import config_integration + from opencensus.trace.samplers import ProbabilitySampler + from opencensus.trace.tracer import Tracer + + config_integration.trace_integrations(['requests']) + tracer = Tracer( + exporter=AzureExporter( + # TODO: replace this with your own instrumentation key. + instrumentation_key='00000000-0000-0000-0000-000000000000', + ), + sampler=ProbabilitySampler(1.0), + ) + with tracer.span(name='parent'): + response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') + References ---------- From d2b6c36d45e4d80c3ff97a6b0b40a815e67fb61b Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 16:40:56 -0700 Subject: [PATCH 13/21] fix log to logs --- contrib/opencensus-ext-azure/README.rst | 2 +- .../examples/metrics/simple.py | 2 +- .../tests/test_azure_log_exporter.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 78f5ad5a2..e12fada5b 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -48,7 +48,7 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr import logging - from opencensus.ext.azure.log_exporter import AzureLogHandler + from opencensus.ext.azure.logs_exporter import AzureLogHandler from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace import config_integration from opencensus.trace.samplers import ProbabilitySampler diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 843f0d510..258c43d4f 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -33,7 +33,7 @@ def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metrics_exporter.new_metrics_exporter(export_interval=2) + exporter = metrics_exporter.new_metrics_exporter(export_interval=2, instrumentation_key='123') view_manager.register_exporter(exporter) view_manager.register_view(PROBLEMS_SOLVED_VIEW) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 2707c9775..1a63e017c 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -18,7 +18,7 @@ import shutil import unittest -from opencensus.ext.azure import log_exporter +from opencensus.ext.azure import logs_exporter TEST_FOLDER = os.path.abspath('.test.logs') @@ -37,7 +37,7 @@ def func(*_args, **_kwargs): return func -class CustomLogHandler(log_exporter.BaseLogHandler): +class CustomLogHandler(logs_exporter.BaseLogHandler): def __init__(self, max_batch_size, callback): self.export_interval = 1 self.max_batch_size = max_batch_size @@ -74,13 +74,13 @@ def test_ctor(self): from opencensus.ext.azure.common import Options instrumentation_key = Options._default.instrumentation_key Options._default.instrumentation_key = None - self.assertRaises(ValueError, lambda: log_exporter.AzureLogHandler()) + self.assertRaises(ValueError, lambda: logs_exporter.AzureLogHandler()) Options._default.instrumentation_key = instrumentation_key @mock.patch('requests.post', return_value=mock.Mock()) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) - handler = log_exporter.AzureLogHandler( + handler = logs_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) @@ -96,7 +96,7 @@ def test_exception(self, requests_mock): @mock.patch('requests.post', return_value=mock.Mock()) def test_export_empty(self, request_mock): - handler = log_exporter.AzureLogHandler( + handler = logs_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) @@ -104,15 +104,15 @@ def test_export_empty(self, request_mock): self.assertEqual(len(os.listdir(handler.storage.path)), 0) handler.close() - @mock.patch('opencensus.ext.azure.log_exporter' + @mock.patch('opencensus.ext.azure.logs_exporter' '.AzureLogHandler.log_record_to_envelope') def test_export_failure(self, log_record_to_envelope_mock): log_record_to_envelope_mock.return_value = ['bar'] - handler = log_exporter.AzureLogHandler( + handler = logs_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) - with mock.patch('opencensus.ext.azure.log_exporter' + with mock.patch('opencensus.ext.azure.logs_exporter' '.AzureLogHandler._transmit') as transmit: transmit.return_value = 10 handler._export(['foo']) @@ -121,7 +121,7 @@ def test_export_failure(self, log_record_to_envelope_mock): handler.close() def test_log_record_to_envelope(self): - handler = log_exporter.AzureLogHandler( + handler = logs_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) From fda620488ee22422fc9079e9c0b661a7b0535202 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 16:51:10 -0700 Subject: [PATCH 14/21] fix lint --- contrib/opencensus-ext-azure/README.rst | 11 ++++++++--- .../examples/metrics/distribution.py | 1 + .../opencensus-ext-azure/examples/metrics/simple.py | 13 +++++++++---- .../opencensus-ext-azure/examples/metrics/sum.py | 12 +++++++++--- .../ext/azure/metrics_exporter/__init__.py | 6 ++++-- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index e12fada5b..f36d66f71 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -97,9 +97,14 @@ Using the Metrics exporter stats_recorder = stats.stats_recorder PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", - "number of problems solved", "problems") - PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", "number of problems solved", [], - PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) + "number of problems solved", + "problems") + PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", + "number of problems solved", + [], + PROBLEMS_SOLVED_MEASURE, + aggregation_module.CountAggregation()) + def main(): # Enable metrics diff --git a/contrib/opencensus-ext-azure/examples/metrics/distribution.py b/contrib/opencensus-ext-azure/examples/metrics/distribution.py index 6da4c9e2b..c020b996a 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/distribution.py +++ b/contrib/opencensus-ext-azure/examples/metrics/distribution.py @@ -41,6 +41,7 @@ aggregation_module.DistributionAggregation( [100.0, 200.0, 400.0, 1000.0, 2000.0, 4000.0])) + def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 258c43d4f..9bd67b8c1 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -26,14 +26,19 @@ stats_recorder = stats.stats_recorder PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", - "number of problems solved", "problems") -PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", "number of problems solved", [], - PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) + "number of problems solved", + "problems") +PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", + "number of problems solved", + [], + PROBLEMS_SOLVED_MEASURE, + aggregation_module.CountAggregation()) + def main(): # Enable metrics # Set the interval in seconds in which you want to send metrics - exporter = metrics_exporter.new_metrics_exporter(export_interval=2, instrumentation_key='123') + exporter = metrics_exporter.new_metrics_exporter(export_interval=2) view_manager.register_exporter(exporter) view_manager.register_view(PROBLEMS_SOLVED_VIEW) diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index 5ff2296cd..2014cac0e 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import random import time from opencensus.ext.azure import metrics_exporter @@ -26,8 +25,15 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", "number of chips eaten", "chips") -CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", "number of chips eaten", [], CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) +CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", + "number of chips eaten", + "chips") +CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", + "number of chips eaten", + [], + CHIPS_EATEN_MEASURE, + aggregation_module.SumAggregation()) + def main(): # Enable metrics diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index 39f26022b..f90938a58 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -20,10 +20,10 @@ __all__ = ['MetricsExporter', 'new_metrics_exporter'] + class MetricsExporter(object): """Metrics exporter for Microsoft Azure Monitor.""" - def __init__(self, options): self._options = options self._md_cache = {} @@ -41,5 +41,7 @@ def new_metrics_exporter(**options): if not options.instrumentation_key: raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) - transport.get_exporter_thread(stats.stats, exporter, interval=options.export_interval) + transport.get_exporter_thread(stats.stats, + exporter, + interval=options.export_interval) return exporter From 5c7adae87408882a3fa722e2661d44047f807784 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 16:59:08 -0700 Subject: [PATCH 15/21] remove spaces --- contrib/opencensus-ext-azure/README.rst | 10 +++++----- .../examples/metrics/simple.py | 10 +++++----- .../opencensus-ext-azure/examples/metrics/sum.py | 14 +++++++------- .../ext/azure/metrics_exporter/__init__.py | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index f36d66f71..6bc869f2b 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -97,12 +97,12 @@ Using the Metrics exporter stats_recorder = stats.stats_recorder PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", - "number of problems solved", + "number of problems solved", "problems") - PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", - "number of problems solved", - [], - PROBLEMS_SOLVED_MEASURE, + PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", + "number of problems solved", + [], + PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 9bd67b8c1..654456e44 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -26,12 +26,12 @@ stats_recorder = stats.stats_recorder PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", - "number of problems solved", + "number of problems solved", "problems") -PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", - "number of problems solved", - [], - PROBLEMS_SOLVED_MEASURE, +PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", + "number of problems solved", + [], + PROBLEMS_SOLVED_MEASURE, aggregation_module.CountAggregation()) diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index 2014cac0e..dce18c8d0 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -25,13 +25,13 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", - "number of chips eaten", - "chips") -CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", - "number of chips eaten", - [], - CHIPS_EATEN_MEASURE, +CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", + "number of chips eaten", + "chips") +CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", + "number of chips eaten", + [], + CHIPS_EATEN_MEASURE, aggregation_module.SumAggregation()) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index f90938a58..fef71d666 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -41,7 +41,7 @@ def new_metrics_exporter(**options): if not options.instrumentation_key: raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) - transport.get_exporter_thread(stats.stats, - exporter, - interval=options.export_interval) + transport.get_exporter_thread(stats.stats, + exporter, + interval=options.export_interval) return exporter From ca6a0dca1d5d2f5815721a1672c8166c0af97a09 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 17:09:27 -0700 Subject: [PATCH 16/21] lessen line length --- .../examples/metrics/sum.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index dce18c8d0..cfc6c0cab 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -25,14 +25,14 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", - "number of chips eaten", - "chips") -CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", - "number of chips eaten", - [], - CHIPS_EATEN_MEASURE, - aggregation_module.SumAggregation()) +CARROTS_MEASURE = measure_module.MeasureFloat("carrots", + "number of carrots", + "carrots") +CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.SumAggregation()) def main(): @@ -41,13 +41,13 @@ def main(): exporter = metrics_exporter.new_metrics_exporter(export_interval=5) view_manager.register_exporter(exporter) - view_manager.register_view(CHIPS_EATEN_VIEW) + view_manager.register_view(CARROTS_VIEW) mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() for i in range(100): print(i) - mmap.measure_int_put(CHIPS_EATEN_MEASURE, 1) + mmap.measure_int_put(CARROTS_MEASURE, 1) mmap.record(tmap) time.sleep(1) From 6e38ff646b7762959edcf7d540c07e18768b5725 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 17:29:55 -0700 Subject: [PATCH 17/21] change examples --- contrib/opencensus-ext-azure/README.rst | 20 +++++++++---------- .../examples/metrics/simple.py | 20 +++++++++---------- .../examples/metrics/sum.py | 20 +++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 6bc869f2b..4e707a9e7 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -96,14 +96,14 @@ Using the Metrics exporter view_manager = stats.view_manager stats_recorder = stats.stats_recorder - PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", - "number of problems solved", - "problems") - PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", - "number of problems solved", - [], - PROBLEMS_SOLVED_MEASURE, - aggregation_module.CountAggregation()) + CARROTS_MEASURE = measure_module.MeasureInt("carrots", + "number of carrots", + "carrots") + CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.CountAggregation()) def main(): @@ -112,11 +112,11 @@ Using the Metrics exporter exporter = metrics_exporter.new_metrics_exporter(export_interval=2) view_manager.register_exporter(exporter) - view_manager.register_view(PROBLEMS_SOLVED_VIEW) + view_manager.register_view(CARROTS_VIEW) mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() - mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) + mmap.measure_int_put(CARROTS_MEASURE, 1000) mmap.record(tmap) time.sleep(10) diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 654456e44..3867de390 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -25,14 +25,14 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -PROBLEMS_SOLVED_MEASURE = measure_module.MeasureInt("problems_solved", - "number of problems solved", - "problems") -PROBLEMS_SOLVED_VIEW = view_module.View("problems_solved_view", - "number of problems solved", - [], - PROBLEMS_SOLVED_MEASURE, - aggregation_module.CountAggregation()) +CARROTS_MEASURE = measure_module.MeasureInt("carrots", + "number of carrots", + "carrots") +CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.CountAggregation()) def main(): @@ -41,11 +41,11 @@ def main(): exporter = metrics_exporter.new_metrics_exporter(export_interval=2) view_manager.register_exporter(exporter) - view_manager.register_view(PROBLEMS_SOLVED_VIEW) + view_manager.register_view(CARROTS_VIEW) mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() - mmap.measure_int_put(PROBLEMS_SOLVED_MEASURE, 1000) + mmap.measure_int_put(CARROTS_MEASURE, 1000) mmap.record(tmap) time.sleep(10) diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index cfc6c0cab..5e6354787 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -25,14 +25,14 @@ view_manager = stats.view_manager stats_recorder = stats.stats_recorder -CARROTS_MEASURE = measure_module.MeasureFloat("carrots", - "number of carrots", - "carrots") -CARROTS_VIEW = view_module.View("carrots_view", - "number of carrots", - [], - CARROTS_MEASURE, - aggregation_module.SumAggregation()) +CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", + "number of chips eaten", + "chips") +CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", + "number of chips eaten", + [], + CHIPS_EATEN_MEASURE, + aggregation_module.SumAggregation()) def main(): @@ -41,13 +41,13 @@ def main(): exporter = metrics_exporter.new_metrics_exporter(export_interval=5) view_manager.register_exporter(exporter) - view_manager.register_view(CARROTS_VIEW) + view_manager.register_view(CHIPS_EATEN_VIEW) mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() for i in range(100): print(i) - mmap.measure_int_put(CARROTS_MEASURE, 1) + mmap.measure_int_put(CHIPS_EATEN_MEASURE, 1) mmap.record(tmap) time.sleep(1) From 41a807f0f048e7c38cca6575dec778f4cfc79103 Mon Sep 17 00:00:00 2001 From: Leighton Date: Wed, 12 Jun 2019 17:36:49 -0700 Subject: [PATCH 18/21] line indents --- contrib/opencensus-ext-azure/examples/metrics/simple.py | 4 ++-- contrib/opencensus-ext-azure/examples/metrics/sum.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py index 3867de390..0bc93bd35 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/simple.py +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -26,8 +26,8 @@ stats_recorder = stats.stats_recorder CARROTS_MEASURE = measure_module.MeasureInt("carrots", - "number of carrots", - "carrots") + "number of carrots", + "carrots") CARROTS_VIEW = view_module.View("carrots_view", "number of carrots", [], diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py index 5e6354787..dce18c8d0 100644 --- a/contrib/opencensus-ext-azure/examples/metrics/sum.py +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -26,8 +26,8 @@ stats_recorder = stats.stats_recorder CHIPS_EATEN_MEASURE = measure_module.MeasureFloat("chips_eaten", - "number of chips eaten", - "chips") + "number of chips eaten", + "chips") CHIPS_EATEN_VIEW = view_module.View("chips_eaten_view", "number of chips eaten", [], From 89ea447cdfa631a4abb9673154be9892d5e67d0c Mon Sep 17 00:00:00 2001 From: Leighton Date: Thu, 13 Jun 2019 14:29:01 -0700 Subject: [PATCH 19/21] move validation --- .../opencensus/ext/azure/metrics_exporter/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index fef71d666..457847c8e 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -25,7 +25,9 @@ class MetricsExporter(object): """Metrics exporter for Microsoft Azure Monitor.""" def __init__(self, options): - self._options = options + self.options = options + if not self.options.instrumentation_key: + raise ValueError('The instrumentation_key is not provided.') self._md_cache = {} self._md_lock = threading.Lock() @@ -38,8 +40,6 @@ def export_metrics(self, metrics): def new_metrics_exporter(**options): options = Options(**options) - if not options.instrumentation_key: - raise ValueError('The instrumentation_key is not provided.') exporter = MetricsExporter(options=options) transport.get_exporter_thread(stats.stats, exporter, From bdd8141103a5c8eb6c3b19012c9bf1198d43124f Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 14 Jun 2019 10:55:49 -0700 Subject: [PATCH 20/21] revert logs_exporter --- contrib/opencensus-ext-azure/README.rst | 4 ++-- .../examples/logs/correlated.py | 2 +- .../examples/logs/error.py | 2 +- .../examples/logs/simple.py | 2 +- .../__init__.py | 0 .../tests/test_azure_log_exporter.py | 18 +++++++++--------- 6 files changed, 14 insertions(+), 14 deletions(-) rename contrib/opencensus-ext-azure/opencensus/ext/azure/{logs_exporter => log_exporter}/__init__.py (100%) diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 4e707a9e7..1d51bd055 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -31,7 +31,7 @@ This example shows how to send a warning level log to Azure Monitor. import logging - from opencensus.ext.azure.logs_exporter import AzureLogHandler + from opencensus.ext.azure.log_exporter import AzureLogHandler logger = logging.getLogger(__name__) logger.addHandler(AzureLogHandler()) @@ -48,7 +48,7 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr import logging - from opencensus.ext.azure.logs_exporter import AzureLogHandler + from opencensus.ext.azure.log_exporter import AzureLogHandler from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace import config_integration from opencensus.trace.samplers import ProbabilitySampler diff --git a/contrib/opencensus-ext-azure/examples/logs/correlated.py b/contrib/opencensus-ext-azure/examples/logs/correlated.py index ffa713656..205a46bc1 100644 --- a/contrib/opencensus-ext-azure/examples/logs/correlated.py +++ b/contrib/opencensus-ext-azure/examples/logs/correlated.py @@ -14,7 +14,7 @@ import logging -from opencensus.ext.azure.logs_exporter import AzureLogHandler +from opencensus.ext.azure.log_exporter import AzureLogHandler from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace import config_integration from opencensus.trace.samplers import ProbabilitySampler diff --git a/contrib/opencensus-ext-azure/examples/logs/error.py b/contrib/opencensus-ext-azure/examples/logs/error.py index c0b299f84..4f801342a 100644 --- a/contrib/opencensus-ext-azure/examples/logs/error.py +++ b/contrib/opencensus-ext-azure/examples/logs/error.py @@ -14,7 +14,7 @@ import logging -from opencensus.ext.azure.logs_exporter import AzureLogHandler +from opencensus.ext.azure.log_exporter import AzureLogHandler logger = logging.getLogger(__name__) # TODO: you need to specify the instrumentation key in the diff --git a/contrib/opencensus-ext-azure/examples/logs/simple.py b/contrib/opencensus-ext-azure/examples/logs/simple.py index b8075e4a9..cdda2b688 100644 --- a/contrib/opencensus-ext-azure/examples/logs/simple.py +++ b/contrib/opencensus-ext-azure/examples/logs/simple.py @@ -14,7 +14,7 @@ import logging -from opencensus.ext.azure.logs_exporter import AzureLogHandler +from opencensus.ext.azure.log_exporter import AzureLogHandler logger = logging.getLogger(__name__) # TODO: you need to specify the instrumentation key in the diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/logs_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py similarity index 100% rename from contrib/opencensus-ext-azure/opencensus/ext/azure/logs_exporter/__init__.py rename to contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 1a63e017c..2707c9775 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -18,7 +18,7 @@ import shutil import unittest -from opencensus.ext.azure import logs_exporter +from opencensus.ext.azure import log_exporter TEST_FOLDER = os.path.abspath('.test.logs') @@ -37,7 +37,7 @@ def func(*_args, **_kwargs): return func -class CustomLogHandler(logs_exporter.BaseLogHandler): +class CustomLogHandler(log_exporter.BaseLogHandler): def __init__(self, max_batch_size, callback): self.export_interval = 1 self.max_batch_size = max_batch_size @@ -74,13 +74,13 @@ def test_ctor(self): from opencensus.ext.azure.common import Options instrumentation_key = Options._default.instrumentation_key Options._default.instrumentation_key = None - self.assertRaises(ValueError, lambda: logs_exporter.AzureLogHandler()) + self.assertRaises(ValueError, lambda: log_exporter.AzureLogHandler()) Options._default.instrumentation_key = instrumentation_key @mock.patch('requests.post', return_value=mock.Mock()) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) - handler = logs_exporter.AzureLogHandler( + handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) @@ -96,7 +96,7 @@ def test_exception(self, requests_mock): @mock.patch('requests.post', return_value=mock.Mock()) def test_export_empty(self, request_mock): - handler = logs_exporter.AzureLogHandler( + handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) @@ -104,15 +104,15 @@ def test_export_empty(self, request_mock): self.assertEqual(len(os.listdir(handler.storage.path)), 0) handler.close() - @mock.patch('opencensus.ext.azure.logs_exporter' + @mock.patch('opencensus.ext.azure.log_exporter' '.AzureLogHandler.log_record_to_envelope') def test_export_failure(self, log_record_to_envelope_mock): log_record_to_envelope_mock.return_value = ['bar'] - handler = logs_exporter.AzureLogHandler( + handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) - with mock.patch('opencensus.ext.azure.logs_exporter' + with mock.patch('opencensus.ext.azure.log_exporter' '.AzureLogHandler._transmit') as transmit: transmit.return_value = 10 handler._export(['foo']) @@ -121,7 +121,7 @@ def test_export_failure(self, log_record_to_envelope_mock): handler.close() def test_log_record_to_envelope(self): - handler = logs_exporter.AzureLogHandler( + handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) From 2c2d45b3bc4d5c31ae7ba95e34c9c8d9c640968c Mon Sep 17 00:00:00 2001 From: Leighton Date: Fri, 14 Jun 2019 10:57:40 -0700 Subject: [PATCH 21/21] README --- README.rst | 4 ++-- contrib/opencensus-ext-azure/README.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 77557e5d2..a420aaeb0 100644 --- a/README.rst +++ b/README.rst @@ -205,8 +205,8 @@ OpenCensus supports integration with popular web frameworks, client libraries an - `SQLAlchemy`_ - `threading`_ -Logs Exporter --------------- +Log Exporter +------------ - `Azure`_ diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 1d51bd055..d963a906e 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -16,8 +16,8 @@ Installation Usage ----- -Logs -~~~~ +Log +~~~ The **Azure Monitor Log Handler** allows you to export Python logs to `Azure Monitor`_.