Reverse dependency order for delete operations in direct engine

NEXT_CHANGELOG.md

@@ -7,7 +7,8 @@
 ### CLI
 
 ### Bundles
-* Pass SYSTEM_ACCESSTOKEN from env to the Terraform provider ([#4135](https://github.com/databricks/cli/pull/4135))
+* engine/direct: Fix dependency-ordered deletion by persisting depends_on in state ([#4105](https://github.com/databricks/cli/pull/4105))
+* Pass SYSTEM_ACCESSTOKEN from env to the Terraform provider ([#4135](https://github.com/databricks/cli/pull/4135)
 
 ### Dependency updates
 

acceptance/bundle/migrate/basic/out.new_state.json

@@ -53,7 +53,33 @@
      "volume_catalog_name": "mycat",
      "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
     }
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.jobs.test_job",
+     "label": "${resources.jobs.test_job.id}"
+    },
+    {
+     "node": "resources.jobs.test_job",
+     "label": "${resources.jobs.test_job.name}"
+    },
+    {
+     "node": "resources.jobs.test_job",
+     "label": "${resources.jobs.test_job.timeout_seconds}"
+    },
+    {
+     "node": "resources.volumes.test_volume",
+     "label": "${resources.volumes.test_volume.catalog_name}"
+    },
+    {
+     "node": "resources.volumes.test_volume",
+     "label": "${resources.volumes.test_volume.id}"
+    },
+    {
+     "node": "resources.volumes.test_volume",
+     "label": "${resources.volumes.test_volume.storage_location}"
+    }
+   ]
   },
   "resources.volumes.test_volume": {
    "__id__": "mycat.myschema.myvol",

acceptance/bundle/migrate/default-python/out.state_after_migration.json

@@ -101,7 +101,13 @@
       "unit": "DAYS"
      }
     }
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.pipelines.my_default_python_etl",
+     "label": "${resources.pipelines.my_default_python_etl.id}"
+    }
+   ]
   },
   "resources.pipelines.my_default_python_etl": {
    "__id__": "[UUID]",

acceptance/bundle/migrate/grants/out.new_state.json

@@ -11,7 +11,13 @@
     "comment": "mycomment",
     "name": "mymodel",
     "schema_name": "schema_grants"
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.schemas.my_schema",
+     "label": "${resources.schemas.my_schema.name}"
+    }
+   ]
   },
   "resources.registered_models.my_registered_model.grants": {
    "__id__": "function/main.schema_grants.mymodel",
@@ -26,7 +32,13 @@
       ]
      }
     ]
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.registered_models.my_registered_model",
+     "label": "${resources.registered_models.my_registered_model.id}"
+    }
+   ]
   },
   "resources.schemas.my_schema": {
    "__id__": "main.schema_grants",
@@ -49,7 +61,13 @@
       ]
      }
     ]
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.schemas.my_schema",
+     "label": "${resources.schemas.my_schema.id}"
+    }
+   ]
   },
   "resources.volumes.my_volume": {
    "__id__": "main.schema_grants.volume_name",
@@ -58,7 +76,13 @@
     "name": "volume_name",
     "schema_name": "schema_grants",
     "volume_type": "MANAGED"
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.schemas.my_schema",
+     "label": "${resources.schemas.my_schema.name}"
+    }
+   ]
   },
   "resources.volumes.my_volume.grants": {
    "__id__": "volume/main.schema_grants.volume_name",
@@ -74,7 +98,13 @@
       ]
      }
     ]
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.volumes.my_volume",
+     "label": "${resources.volumes.my_volume.id}"
+    }
+   ]
   }
  }
 }

acceptance/bundle/migrate/permissions/out.new_state.json

@@ -42,7 +42,13 @@
       "user_name": "[USERNAME]"
      }
     ]
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.jobs.test_job",
+     "label": "${resources.jobs.test_job.id}"
+    }
+   ]
   },
   "resources.pipelines.test_pipeline": {
    "__id__": "[UUID]",
@@ -77,7 +83,13 @@
       "user_name": "[USERNAME]"
      }
     ]
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.pipelines.test_pipeline",
+     "label": "${resources.pipelines.test_pipeline.id}"
+    }
+   ]
   }
  }
 }

acceptance/bundle/migrate/runas/out.new_state.json

@@ -43,7 +43,13 @@
       "user_name": "[USERNAME]"
      }
     ]
-   }
+   },
+   "depends_on": [
+    {
+     "node": "resources.pipelines.foo",
+     "label": "${resources.pipelines.foo.id}"
+    }
+   ]
   }
  }
 }

acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json

@@ -28,6 +28,12 @@
       }
     },
     "resources.jobs.foo": {
+      "depends_on": [
+        {
+          "node": "resources.jobs.bar",
+          "label": "${resources.jobs.bar.id}"
+        }
+      ],
       "action": "delete",
       "remote_state": {
         "created_time": [UNIX_TIME_MILLIS][1],

acceptance/bundle/resource_deps/job_id/output.txt

@@ -65,19 +65,19 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> print_requests.py --sort //jobs
+>>> print_requests.py //jobs
 {
   "method": "POST",
   "path": "/api/2.2/jobs/delete",
   "body": {
-    "job_id": [BAR_ID]
+    "job_id": [FOO_ID]
   }
 }
 {
   "method": "POST",
   "path": "/api/2.2/jobs/delete",
   "body": {
-    "job_id": [FOO_ID]
+    "job_id": [BAR_ID]
   }
 }
 

acceptance/bundle/resource_deps/job_id/script

@@ -13,8 +13,7 @@ echo "$bar_id:BAR_ID" >> ACC_REPLS
 cp empty.yml databricks.yml
 trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
-# TODO sorting requests should not be needed one we persist depends_on in state
-trace print_requests.py --sort //jobs
+trace print_requests.py //jobs
 
 trace $CLI bundle destroy --auto-approve
 trace print_requests.py //jobs

acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml

@@ -0,0 +1,78 @@
+bundle:
+  name: test-bundle
+
+# Complex graph combining: chain, diamond, independent, multi-deps
+#
+#   independent1    independent2
+#
+#   chain_top -> chain_mid -> chain_bottom
+#
+#   diamond_top -> diamond_left  -> diamond_bottom
+#                  diamond_right ->
+#
+#   multi_child -> multi_parent1
+#               -> multi_parent2
+
+resources:
+  jobs:
+    # Independent nodes (no dependencies)
+    independent1:
+      name: job independent1
+    independent2:
+      name: job independent2
+
+    # Chain: chain_top -> chain_mid -> chain_bottom
+    chain_bottom:
+      name: job chain_bottom
+    chain_mid:
+      name: job chain_mid
+      tasks:
+        - task_key: t1
+          run_job_task:
+            job_id: ${resources.jobs.chain_bottom.id}
+    chain_top:
+      name: job chain_top
+      tasks:
+        - task_key: t1
+          run_job_task:
+            job_id: ${resources.jobs.chain_mid.id}
+
+    # Diamond: diamond_top -> diamond_left/right -> diamond_bottom
+    diamond_bottom:
+      name: job diamond_bottom
+    diamond_left:
+      name: job diamond_left
+      tasks:
+        - task_key: t1
+          run_job_task:
+            job_id: ${resources.jobs.diamond_bottom.id}
+    diamond_right:
+      name: job diamond_right
+      tasks:
+        - task_key: t1
+          run_job_task:
+            job_id: ${resources.jobs.diamond_bottom.id}
+    diamond_top:
+      name: job diamond_top
+      tasks:
+        - task_key: t1
+          run_job_task:
+            job_id: ${resources.jobs.diamond_left.id}
+        - task_key: t2
+          run_job_task:
+            job_id: ${resources.jobs.diamond_right.id}
+
+    # Multi-deps: multi_child -> multi_parent1, multi_parent2
+    multi_parent1:
+      name: job multi_parent1
+    multi_parent2:
+      name: job multi_parent2
+    multi_child:
+      name: job multi_child
+      tasks:
+        - task_key: t1
+          run_job_task:
+            job_id: ${resources.jobs.multi_parent1.id}
+        - task_key: t2
+          run_job_task:
+            job_id: ${resources.jobs.multi_parent2.id}

acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt

@@ -0,0 +1,14 @@
+delete jobs.chain_bottom
+delete jobs.chain_mid
+delete jobs.chain_top
+delete jobs.diamond_bottom
+delete jobs.diamond_left
+delete jobs.diamond_right
+delete jobs.diamond_top
+delete jobs.independent1
+delete jobs.independent2
+delete jobs.multi_child
+delete jobs.multi_parent1
+delete jobs.multi_parent2
+
+Plan: 0 to add, 0 to change, 12 to delete, 0 unchanged

acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt

@@ -0,0 +1,14 @@
+delete jobs.chain_bottom
+delete jobs.chain_mid
+delete jobs.chain_top
+delete jobs.diamond_bottom
+delete jobs.diamond_left
+delete jobs.diamond_right
+delete jobs.diamond_top
+delete jobs.independent1
+delete jobs.independent2
+delete jobs.multi_child
+delete jobs.multi_parent1
+delete jobs.multi_parent2
+
+Plan: 0 to add, 0 to change, 12 to delete, 0 unchanged

acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt

@@ -0,0 +1 @@
+Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged

acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt

@@ -0,0 +1 @@
+Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged