From 8c4f4b92970b90301f28e7099db738ffea1b13fc Mon Sep 17 00:00:00 2001 From: Mingye Xia Date: Thu, 27 Sep 2018 16:11:05 -0700 Subject: [PATCH] [AIRFLOW-3125] Monitor Task Instances creation rates Montor Task Instances creation rates by Operator type. These stats can provide some visibility on how much workload Airflow is getting. They can be used for resource allocation in the long run (i.e. to determine when we should scale up workers) and debugging in scenarios like the creation rate of certain type of Task Instances spikes. --- airflow/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airflow/models.py b/airflow/models.py index e4a50bc476f98..9d1d64ee73e9e 100755 --- a/airflow/models.py +++ b/airflow/models.py @@ -5282,6 +5282,9 @@ def verify_integrity(self, session=None): continue if task.task_id not in task_ids: + Stats.incr( + "task_instance_created-{}".format(task.__class__.__name__), + 1, 1) ti = TaskInstance(task, self.execution_date) session.add(ti)