diff --git a/airflow/executors/base_executor.py b/airflow/executors/base_executor.py index ad7690b3f6e1a..fb8cee38dfe1e 100644 --- a/airflow/executors/base_executor.py +++ b/airflow/executors/base_executor.py @@ -352,7 +352,7 @@ def trigger_tasks(self, open_slots: int) -> None: # Otherwise, we give up and remove the task from the queue. self.log.error( - "could not queue task %s (still running after %d attempts).", + "Could not queue task %s (still running after %d attempts). Learn more: https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#state-mismatch-between-airflow-database-and-executor", key, attempt.total_tries, ) diff --git a/docs/apache-airflow/troubleshooting.rst b/docs/apache-airflow/troubleshooting.rst index 076aa5501bcac..ff803c528f094 100644 --- a/docs/apache-airflow/troubleshooting.rst +++ b/docs/apache-airflow/troubleshooting.rst @@ -20,6 +20,19 @@ Troubleshooting =============== +Obscure scheduling failures +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +State mismatch between Airflow database and executor +---------------------------------------------------- + +This indicates that when the scheduler queried the Airflow database, it observed that the task instance had one status according to the Airflow metadata database, +but a different status according to the executor. A common example is when the query returned to the scheduler, the task instance was in the ``queued`` status, +but the status according to the executor was ``running``. + +This mismatch must have persisted for multiple attempts. When this happens, Airflow will not attempt to queue the task. It's possible that something has gone wrong +in the executor, and the task may need to be cleared. + Obscure task failures ^^^^^^^^^^^^^^^^^^^^^