diff --git a/acceptance/bin/print_requests.py b/acceptance/bin/print_requests.py index 0e8a74c0759..dbc7d5a6d90 100755 --- a/acceptance/bin/print_requests.py +++ b/acceptance/bin/print_requests.py @@ -178,6 +178,17 @@ def filter_requests(requests, path_filters, include_get, should_sort, unique=Fal return filtered_requests +def del_path(body, field): + """Delete field from body. A dotted field descends into nested objects, e.g. + deployment.version_id removes only that key from the deployment block.""" + *parents, leaf = field.split(".") + for name in parents: + body = body.get(name) + if not isinstance(body, dict): + return + body.pop(leaf, None) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("path_filters", nargs="*", help="Path substring filters") @@ -238,7 +249,7 @@ def main(): body = req.get("body") if isinstance(body, dict): for field in del_body_fields: - body.pop(field, None) + del_path(body, field) for field in del_fields: req.pop(field, None) if args.verbose: diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/ai_runtime_task/test.toml b/acceptance/bundle/ai_runtime_task/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/ai_runtime_task/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/apps/app_yaml/out.test.toml b/acceptance/bundle/apps/app_yaml/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/app_yaml/out.test.toml +++ b/acceptance/bundle/apps/app_yaml/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml +++ b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/compute_size/out.test.toml b/acceptance/bundle/apps/compute_size/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/apps/compute_size/out.test.toml +++ b/acceptance/bundle/apps/compute_size/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/apps/delete_deleting/out.test.toml +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/apps/git_source/out.test.toml b/acceptance/bundle/apps/git_source/out.test.toml index 8f6c4a03c57..dfb2766ed88 100644 --- a/acceptance/bundle/apps/git_source/out.test.toml +++ b/acceptance/bundle/apps/git_source/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/job_permissions/out.test.toml b/acceptance/bundle/apps/job_permissions/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/apps/job_permissions/out.test.toml +++ b/acceptance/bundle/apps/job_permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/job_permissions_warning/out.test.toml b/acceptance/bundle/apps/job_permissions_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/job_permissions_warning/out.test.toml +++ b/acceptance/bundle/apps/job_permissions_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/value_from_warning/out.test.toml b/acceptance/bundle/apps/value_from_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/value_from_warning/out.test.toml +++ b/acceptance/bundle/apps/value_from_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml +++ b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/build_and_files/out.test.toml b/acceptance/bundle/artifacts/build_and_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/build_and_files/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml +++ b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_in_files/out.test.toml b/acceptance/bundle/artifacts/globs_in_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_in_files/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_invalid/out.test.toml b/acceptance/bundle/artifacts/globs_invalid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_invalid/out.test.toml +++ b/acceptance/bundle/artifacts/globs_invalid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/issue_3109/out.test.toml b/acceptance/bundle/artifacts/issue_3109/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/issue_3109/out.test.toml +++ b/acceptance/bundle/artifacts/issue_3109/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml +++ b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/bash/out.test.toml b/acceptance/bundle/artifacts/shell/bash/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/bash/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/basic/out.test.toml b/acceptance/bundle/artifacts/shell/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/basic/out.test.toml +++ b/acceptance/bundle/artifacts/shell/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/cmd/out.test.toml b/acceptance/bundle/artifacts/shell/cmd/out.test.toml index 8471d88c7f3..af56bea47fb 100644 --- a/acceptance/bundle/artifacts/shell/cmd/out.test.toml +++ b/acceptance/bundle/artifacts/shell/cmd/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false GOOS.darwin = false GOOS.linux = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/default/out.test.toml b/acceptance/bundle/artifacts/shell/default/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/default/out.test.toml +++ b/acceptance/bundle/artifacts/shell/default/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/invalid/out.test.toml b/acceptance/bundle/artifacts/shell/invalid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/invalid/out.test.toml +++ b/acceptance/bundle/artifacts/shell/invalid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/sh/out.test.toml b/acceptance/bundle/artifacts/shell/sh/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/sh/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index 61bf8345e7b..a8051bc8ca6 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -29,3 +29,59 @@ Response.Body = ''' "spark_version": "13.3.x-scala2.12" } ''' + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_change_version/out.test.toml b/acceptance/bundle/artifacts/whl_change_version/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_change_version/out.test.toml +++ b/acceptance/bundle/artifacts/whl_change_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_explicit/out.test.toml b/acceptance/bundle/artifacts/whl_explicit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_explicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_explicit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit/out.test.toml b/acceptance/bundle/artifacts/whl_implicit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_multiple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_multiple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml +++ b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/deploy/out.test.toml b/acceptance/bundle/benchmarks/deploy/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/deploy/out.test.toml +++ b/acceptance/bundle/benchmarks/deploy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/plan/out.test.toml b/acceptance/bundle/benchmarks/plan/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/plan/out.test.toml +++ b/acceptance/bundle/benchmarks/plan/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/validate/out.test.toml b/acceptance/bundle/benchmarks/validate/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/validate/out.test.toml +++ b/acceptance/bundle/benchmarks/validate/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/id/out.test.toml b/acceptance/bundle/bundle_tag/id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/id/out.test.toml +++ b/acceptance/bundle/bundle_tag/id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/test.toml b/acceptance/bundle/bundle_tag/test.toml index 8540f9500e6..ea76209cc5a 100644 --- a/acceptance/bundle/bundle_tag/test.toml +++ b/acceptance/bundle/bundle_tag/test.toml @@ -1 +1,57 @@ Badness = "configs with id and url should be rejected" + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/bundle_tag/url/out.test.toml b/acceptance/bundle/bundle_tag/url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/url/out.test.toml +++ b/acceptance/bundle/bundle_tag/url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/url_ref/out.test.toml b/acceptance/bundle/bundle_tag/url_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.test.toml +++ b/acceptance/bundle/bundle_tag/url_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml +++ b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml +++ b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml index 4c2be3166c4..dc7c5353574 100644 --- a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml +++ b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml +++ b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml +++ b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/output_json/out.test.toml b/acceptance/bundle/config-remote-sync/output_json/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/output_json/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_json/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml +++ b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/target_override/out.test.toml b/acceptance/bundle/config-remote-sync/target_override/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/target_override/out.test.toml +++ b/acceptance/bundle/config-remote-sync/target_override/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml +++ b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml +++ b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/debug/list-targets/out.test.toml b/acceptance/bundle/debug/list-targets/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/debug/list-targets/out.test.toml +++ b/acceptance/bundle/debug/list-targets/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/debug/out.test.toml b/acceptance/bundle/debug/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/debug/out.test.toml +++ b/acceptance/bundle/debug/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/empty-bundle/out.test.toml b/acceptance/bundle/deploy/empty-bundle/out.test.toml index 72e8a7a4dfe..3054de89706 100644 --- a/acceptance/bundle/deploy/empty-bundle/out.test.toml +++ b/acceptance/bundle/deploy/empty-bundle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/experimental-python/out.test.toml b/acceptance/bundle/deploy/experimental-python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/experimental-python/out.test.toml +++ b/acceptance/bundle/deploy/experimental-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/force-lock-config/out.test.toml b/acceptance/bundle/deploy/force-lock-config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/force-lock-config/out.test.toml +++ b/acceptance/bundle/deploy/force-lock-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml +++ b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable/out.test.toml b/acceptance/bundle/deploy/immutable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable/out.test.toml +++ b/acceptance/bundle/deploy/immutable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/mlops-stacks/out.test.toml b/acceptance/bundle/deploy/mlops-stacks/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deploy/mlops-stacks/out.test.toml +++ b/acceptance/bundle/deploy/mlops-stacks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml +++ b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/python-notebook/out.test.toml b/acceptance/bundle/deploy/python-notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/python-notebook/out.test.toml +++ b/acceptance/bundle/deploy/python-notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index 71970b719d4..310be221793 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index f7c4cf648a9..6f6238f01b3 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index 65156e0457c..25ad1a52fcf 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml new file mode 100644 index 00000000000..d5628ffe5fe --- /dev/null +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -0,0 +1,6 @@ +# These tests apply a saved plan, which does not carry the deployment stamp: on a first +# deploy there is no deployment to resolve when `bundle plan` runs, so the plan it writes +# leaves the field unset and applying it plans an update the next time. Same reason as +# EnvMatrixExclude.dms_no_readplan in acceptance/bundle/test.toml, which only covers the +# tests that take the saved-plan path through the READPLAN matrix variable. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml +++ b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/spark-jar-task/out.test.toml b/acceptance/bundle/deploy/spark-jar-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deploy/spark-jar-task/out.test.toml +++ b/acceptance/bundle/deploy/spark-jar-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/test.toml b/acceptance/bundle/deploy/test.toml index 84e8a4a1990..10c2541173a 100644 --- a/acceptance/bundle/deploy/test.toml +++ b/acceptance/bundle/deploy/test.toml @@ -3,3 +3,59 @@ Ignore = [ '.databricks', '__pycache__', ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml index 1d895a16c96..426690291a0 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml +++ b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false EnvMatrix.COMMAND = ["plan", "deploy --force-lock", "summary"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml index 9448f875df7..84742e9cd0a 100644 --- a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false EnvMatrix.COMMAND = ["deploy", "plan", "summary"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml +++ b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml +++ b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index fe4076cdf9b..add7ee1060c 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index f61486ff080..ea5b0803e06 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresCluster = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index bcadf671a38..587894f57b0 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index bcadf671a38..587894f57b0 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index e849ec85ace..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index e849ec85ace..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index e849ec85ace..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/test.toml b/acceptance/bundle/deployment/bind/test.toml new file mode 100644 index 00000000000..10bef2f1ccb --- /dev/null +++ b/acceptance/bundle/deployment/bind/test.toml @@ -0,0 +1,2 @@ +# Bind operations are not yet supported by the Deployment Metadata Service (DMS) +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index fe4076cdf9b..add7ee1060c 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index 48203e833cd..a250199143a 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index e849ec85ace..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml index c7c6f58ed6e..32ecf0fa454 100644 --- a/acceptance/bundle/deployment/test.toml +++ b/acceptance/bundle/deployment/test.toml @@ -1 +1,57 @@ Cloud = true + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index e849ec85ace..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index bbc7fcfd1bd..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/test.toml b/acceptance/bundle/deployment/unbind/test.toml new file mode 100644 index 00000000000..2be1dcf74aa --- /dev/null +++ b/acceptance/bundle/deployment/unbind/test.toml @@ -0,0 +1,2 @@ +# Unbind operations are not yet supported by the Deployment Metadata Service (DMS) +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/destroy/all-resources/out.test.toml b/acceptance/bundle/destroy/all-resources/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/destroy/all-resources/out.test.toml +++ b/acceptance/bundle/destroy/all-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml +++ b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml +++ b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/test.toml b/acceptance/bundle/destroy/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/destroy/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/dms/depends-on/databricks.yml b/acceptance/bundle/dms/depends-on/databricks.yml new file mode 100644 index 00000000000..f97e3a38809 --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: dms-depends-on + +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..952dcb00a86 --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,52 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.child" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.child", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.parent" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.parent", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs --oneline +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..be1b9d622f8 --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort + +title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" +rm -rf .databricks +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs --oneline diff --git a/acceptance/bundle/dms/existing-state/databricks.yml b/acceptance/bundle/dms/existing-state/databricks.yml new file mode 100644 index 00000000000..cfd64979342 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-existing-state + +experimental: + record_deployment_history: false + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/existing-state/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..6233f158974 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,66 @@ + +=== Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --oneline + +=== Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out + + +=== No deployment was created in DMS +>>> print_requests.py //api/2.0/bundle --oneline + +=== Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out + + +>>> print_requests.py //api/2.0/bundle --oneline + +=== Destroy clears the tracked resources, so recording can be enabled afterwards +>>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default + +Deleting files... +Destroy complete! + +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..9e448161259 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,22 @@ +title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --oneline + +title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace musterr $CLI bundle deploy + +title "No deployment was created in DMS" +trace print_requests.py //api/2.0/bundle --oneline + +title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" +rm -rf .databricks +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --oneline + +title "Destroy clears the tracked resources, so recording can be enabled afterwards" +trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" +trace $CLI bundle destroy --auto-approve +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml b/acceptance/bundle/dms/multiple-resources/databricks.yml new file mode 100644 index 00000000000..30f2f433b81 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: dms-multiple-resources + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..b593998dbd1 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,25 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --del-body state --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..e9b9339607e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,7 @@ +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/no-drift/databricks.yml b/acceptance/bundle/dms/no-drift/databricks.yml new file mode 100644 index 00000000000..ca2ed6bd96d --- /dev/null +++ b/acceptance/bundle/dms/no-drift/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: dms-no-drift + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + name: bar + catalog: main + schema: default + serverless: true diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt new file mode 100644 index 00000000000..47ffe89cb73 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -0,0 +1,56 @@ + +=== Deploy, then plan without touching anything: the deployment stamp must not show as drift +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== A second deploy is a no-op too: no update request for either resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "name": "bar", + "schema": "default", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script new file mode 100644 index 00000000000..c67c2c28a74 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/script @@ -0,0 +1,12 @@ +title "Deploy, then plan without touching anything: the deployment stamp must not show as drift" +trace $CLI bundle deploy + +# Only the deploy phase stamps deployment.deployment_id (it is not known until the +# version is claimed), so plan sees it in the state and in the workspace but not in the +# local config. Without an ignore_local_changes rule that absence plans an update on a +# job and a pipeline nobody edited. +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +title "A second deploy is a no-op too: no update request for either resource" +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort diff --git a/acceptance/bundle/dms/no-resources/databricks.yml b/acceptance/bundle/dms/no-resources/databricks.yml new file mode 100644 index 00000000000..78fad3a292e --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: dms-no-resources + +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..8801d4179b1 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,87 @@ + +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-no-resources", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" +} + +=== Redeploy: the deployment is resolved from that node, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-no-resources", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..f1b9ad5fa60 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,8 @@ +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --get +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' + +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..b665d545d12 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: the service side is not ready for users yet +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..10a0d995d87 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: the service side is not ready for users yet" +trace musterr $CLI bundle validate + +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..4617ff88f20 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,6 @@ +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml b/acceptance/bundle/dms/operation-upload-fails/databricks.yml new file mode 100644 index 00000000000..1edbd7add88 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: dms-operation-upload-fails + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..26c748d9e79 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,6 @@ +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..ce222ac9e87 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,12 @@ +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# The service rejects every recorded operation. Deploy must stop rather than +# create every remaining resource: a completed version makes DMS the source of +# truth for resource state, so resources it has no record of would be created a +# second time by the next deploy. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/dms/partial-update/databricks.yml b/acceptance/bundle/dms/partial-update/databricks.yml new file mode 100644 index 00000000000..fc61c680fb1 --- /dev/null +++ b/acceptance/bundle/dms/partial-update/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: dms-partial-update + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_partial_update_schema + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/partial-update/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/partial-update/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt new file mode 100644 index 00000000000..f6984c72258 --- /dev/null +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -0,0 +1,165 @@ + +=== Deploy: the state write records the resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-partial-update", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "main.dms_partial_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Recreate writes state twice - the entry is dropped, then the new resource is saved +>>> update_file.py databricks.yml catalog_name: main catalog_name: other + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-partial-update", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "other.dms_partial_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"other\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: the delete is recorded with the id and no state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "target_name": "default", + "display_name": "dms-partial-update", + "previous_version_id": "2", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "other.dms_partial_update_schema", + "resource_key": "schemas.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script new file mode 100644 index 00000000000..9310ded83aa --- /dev/null +++ b/acceptance/bundle/dms/partial-update/script @@ -0,0 +1,15 @@ +title "Deploy: the state write records the resource" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle + +title "Recreate writes state twice - the entry is dropped, then the new resource is saved" +# Only the save is recorded. The service keeps one operation per resource per version +# and rejects a succeeded recreate that carries no state, so the intermediate drop +# cannot be its own event; the save that follows reports the recreate instead. +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" +trace $CLI bundle deploy --auto-approve +trace print_requests.py //api/2.0/bundle + +title "Destroy: the delete is recorded with the id and no state" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/dms/provenance/databricks.yml b/acceptance/bundle/dms/provenance/databricks.yml new file mode 100644 index 00000000000..df3b2d033c4 --- /dev/null +++ b/acceptance/bundle/dms/provenance/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: dms-provenance + +experimental: + record_deployment_history: true + +targets: + dev: + default: true + mode: development + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt new file mode 100644 index 00000000000..c567a12d66c --- /dev/null +++ b/acceptance/bundle/dms/provenance/output.txt @@ -0,0 +1,69 @@ + +=== Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "dev", + "display_name": "dms-provenance", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version +>>> [CLI] api get /api/2.0/bundle/deployments/[NUMID] +{ + "target_name": "dev", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script new file mode 100644 index 00000000000..eb36baf6067 --- /dev/null +++ b/acceptance/bundle/dms/provenance/script @@ -0,0 +1,15 @@ +title "Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode" +git-repo-init +git remote add origin https://github.com/databricks/bundle-examples.git +trace $CLI bundle deploy +# The commit SHA changes every run, so assert it is a 40-char hex string and drop it. +add_repl.py "$(git rev-parse HEAD)" COMMIT +trace print_requests.py //versions --sort + +title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' + +# The deploy uploads files in a nondeterministic order; only the requests above are +# asserted. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/test.toml b/acceptance/bundle/dms/provenance/test.toml new file mode 100644 index 00000000000..0a47bfb1b91 --- /dev/null +++ b/acceptance/bundle/dms/provenance/test.toml @@ -0,0 +1,4 @@ +# git-repo-init creates a repo in the test directory so the deploy has git provenance. +Ignore = [ + '.git', +] diff --git a/acceptance/bundle/dms/record-failure/databricks.yml b/acceptance/bundle/dms/record-failure/databricks.yml new file mode 100644 index 00000000000..8e8573fa70f --- /dev/null +++ b/acceptance/bundle/dms/record-failure/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record-failure + +experimental: + record_deployment_history: true + +resources: + jobs: + doomed: + name: doomed diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt new file mode 100644 index 00000000000..cb4073087d7 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -0,0 +1,79 @@ + +=== A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files... +Deploying resources... +Error: cannot create resources.jobs.doomed: cluster spec is invalid (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: cluster spec is invalid + + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-record-failure", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.doomed" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "error_message": "cluster spec is invalid", + "resource_key": "jobs.doomed", + "status": "OPERATION_STATUS_FAILED" + } +} + +=== The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed +>>> [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "name": "deployments/[NUMID]/resources/jobs.doomed", + "resource_key": "jobs.doomed", + "resource_type": "" + } + ] +} + +=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script new file mode 100644 index 00000000000..66af9601402 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/script @@ -0,0 +1,16 @@ +title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml new file mode 100644 index 00000000000..bf8710439b1 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -0,0 +1,6 @@ +# The job cannot be created, so applying it fails and the operation is recorded as +# failed rather than omitted. +[[Server]] +Pattern = "POST /api/2.2/jobs/create" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "cluster spec is invalid"}''' diff --git a/acceptance/bundle/dms/record/databricks.yml b/acceptance/bundle/dms/record/databricks.yml new file mode 100644 index 00000000000..b20e6274310 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..a707e5e46d7 --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,153 @@ + +=== Deploy: the server assigns the deployment ID, and a version + create operation are recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-record", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" +} + +>>> jq has("deployment_id") .databricks/bundle/default/resources.json +false + +=== Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-record", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: a destroy version and delete operation are recorded, then the deployment is deleted +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "target_name": "default", + "display_name": "dms-record", + "previous_version_id": "2", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..895a9033a39 --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,20 @@ +title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle + +title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path +# into C:/Program Files/Git/Workspace/... before the CLI sees it. Set per command +# rather than in test.toml: trace exports it in a subshell, so it cannot reach the +# python helpers, whose PATH does need converting. +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json + +title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" +rm -rf .databricks +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle + +title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml new file mode 100644 index 00000000000..8f79a2c0381 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-redeploy-after-destroy + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt new file mode 100644 index 00000000000..12ce1a298c4 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -0,0 +1,80 @@ + +=== Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default + +Deleting files... +Destroy complete! + +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. + +=== Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" +} + +>>> print_requests.py //api/2.0/bundle --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-redeploy-after-destroy", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script new file mode 100644 index 00000000000..8dabf189f49 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -0,0 +1,11 @@ +title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve +print_requests.py //api/2.0/bundle --get > /dev/null + +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" + +title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" +trace $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' +trace print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/summary/databricks.yml b/acceptance/bundle/dms/summary/databricks.yml new file mode 100644 index 00000000000..c0698376223 --- /dev/null +++ b/acceptance/bundle/dms/summary/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-summary + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt new file mode 100644 index 00000000000..c792a60fcfa --- /dev/null +++ b/acceptance/bundle/dms/summary/output.txt @@ -0,0 +1,39 @@ + +=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + +=== Redeploying advances the version the summary reports +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + +=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary/default + +Deleting files... +Destroy complete! + +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script new file mode 100644 index 00000000000..6f44e7c2880 --- /dev/null +++ b/acceptance/bundle/dms/summary/script @@ -0,0 +1,15 @@ +title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "Redeploying advances the version the summary reports" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' + +# This test asserts the summary output, not the requests behind it; the file uploads +# recorded here are ordered nondeterministically. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..e491f968d76 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,27 @@ +Local = true +Cloud = false + +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# These tests enable recording through experimental.record_deployment_history, so the +# DATABRICKS_BUNDLE_DMS variant the rest of the suite adds would just duplicate them. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + +RecordRequests = true + +Ignore = [ + '.databricks', +] + +# experimental.record_deployment_history is rejected outright (see +# validate.ValidateRecordDeploymentHistory). These tests exercise the feature itself, +# so they force allow it the same way DMS development does. bundle/dms/not-supported +# covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" + +# The parent lets the rest of the suite record a bundle whose state already tracks +# resources, since most of those tests seed a state fixture. bundle/dms/existing-state +# asserts that refusal, so it has to stay on here. +Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "" diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..e87ed786023 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,34 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "FILE" +} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> print_requests.py //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..39566343e98 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,7 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..f0a8407e21d --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,7 @@ +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/empty_string_dropped/out.test.toml b/acceptance/bundle/empty_string_dropped/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/empty_string_dropped/out.test.toml +++ b/acceptance/bundle/empty_string_dropped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index 973e436617c..f856c966e20 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -10,11 +10,12 @@ # fix in the initialize phase would drop them, and this golden would show that. $CLI bundle validate -o json -t direct | jq .resources > out.validate.json -# Exclude non-create traffic: workspace file ops and telemetry (nondeterministic). +# Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the +# deployment history calls the DMS run adds. trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t tf -print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.terraform.json +print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.terraform.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy -t direct -print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.direct.json +print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.direct.json $TESTDIR/empty_sent.py diff --git a/acceptance/bundle/empty_string_dropped/test.toml b/acceptance/bundle/empty_string_dropped/test.toml index 51e7bc13e23..ebc74f5b64f 100644 --- a/acceptance/bundle/empty_string_dropped/test.toml +++ b/acceptance/bundle/empty_string_dropped/test.toml @@ -10,3 +10,59 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [ ".databricks", ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/empty_string_variable/out.test.toml b/acceptance/bundle/empty_string_variable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/empty_string_variable/out.test.toml +++ b/acceptance/bundle/empty_string_variable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/environments/dependencies/out.test.toml b/acceptance/bundle/environments/dependencies/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/environments/dependencies/out.test.toml +++ b/acceptance/bundle/environments/dependencies/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/environments/test.toml b/acceptance/bundle/environments/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/environments/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml +++ b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/alert/out.test.toml b/acceptance/bundle/generate/alert/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/alert/out.test.toml +++ b/acceptance/bundle/generate/alert/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml +++ b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/app_subfolders/out.test.toml b/acceptance/bundle/generate/app_subfolders/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/app_subfolders/out.test.toml +++ b/acceptance/bundle/generate/app_subfolders/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/auto-bind/out.test.toml b/acceptance/bundle/generate/auto-bind/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/generate/auto-bind/out.test.toml +++ b/acceptance/bundle/generate/auto-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard/out.test.toml b/acceptance/bundle/generate/dashboard/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard/out.test.toml +++ b/acceptance/bundle/generate/dashboard/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/designer_job/out.test.toml b/acceptance/bundle/generate/designer_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/designer_job/out.test.toml +++ b/acceptance/bundle/generate/designer_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/genie_space/out.test.toml +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/generate/git_job/out.test.toml b/acceptance/bundle/generate/git_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/git_job/out.test.toml +++ b/acceptance/bundle/generate/git_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/include_warning/out.test.toml b/acceptance/bundle/generate/include_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/include_warning/out.test.toml +++ b/acceptance/bundle/generate/include_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/ipynb_job/out.test.toml b/acceptance/bundle/generate/ipynb_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/ipynb_job/out.test.toml +++ b/acceptance/bundle/generate/ipynb_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml +++ b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml +++ b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline/out.test.toml b/acceptance/bundle/generate/pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline/out.test.toml +++ b/acceptance/bundle/generate/pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/python_job/out.test.toml b/acceptance/bundle/generate/python_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/python_job/out.test.toml +++ b/acceptance/bundle/generate/python_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/spark_python_task_job/out.test.toml b/acceptance/bundle/generate/spark_python_task_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/spark_python_task_job/out.test.toml +++ b/acceptance/bundle/generate/spark_python_task_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/git-permerror/out.test.toml b/acceptance/bundle/git-permerror/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/git-permerror/out.test.toml +++ b/acceptance/bundle/git-permerror/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deploy/out.test.toml b/acceptance/bundle/help/bundle-deploy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deploy/out.test.toml +++ b/acceptance/bundle/help/bundle-deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deployment/out.test.toml b/acceptance/bundle/help/bundle-deployment/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deployment/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-destroy/out.test.toml b/acceptance/bundle/help/bundle-destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-destroy/out.test.toml +++ b/acceptance/bundle/help/bundle-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-job/out.test.toml b/acceptance/bundle/help/bundle-generate-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-job/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate/out.test.toml b/acceptance/bundle/help/bundle-generate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate/out.test.toml +++ b/acceptance/bundle/help/bundle-generate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-init/out.test.toml b/acceptance/bundle/help/bundle-init/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-init/out.test.toml +++ b/acceptance/bundle/help/bundle-init/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-open/out.test.toml b/acceptance/bundle/help/bundle-open/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-open/out.test.toml +++ b/acceptance/bundle/help/bundle-open/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-run/out.test.toml b/acceptance/bundle/help/bundle-run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-run/out.test.toml +++ b/acceptance/bundle/help/bundle-run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-schema/out.test.toml b/acceptance/bundle/help/bundle-schema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-schema/out.test.toml +++ b/acceptance/bundle/help/bundle-schema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-summary/out.test.toml b/acceptance/bundle/help/bundle-summary/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-summary/out.test.toml +++ b/acceptance/bundle/help/bundle-summary/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-sync/out.test.toml b/acceptance/bundle/help/bundle-sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-sync/out.test.toml +++ b/acceptance/bundle/help/bundle-sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-validate/out.test.toml b/acceptance/bundle/help/bundle-validate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-validate/out.test.toml +++ b/acceptance/bundle/help/bundle-validate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle/out.test.toml b/acceptance/bundle/help/bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle/out.test.toml +++ b/acceptance/bundle/help/bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/glob_in_root_path/out.test.toml b/acceptance/bundle/includes/glob_in_root_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/glob_in_root_path/out.test.toml +++ b/acceptance/bundle/includes/glob_in_root_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/include_outside_root/out.test.toml b/acceptance/bundle/includes/include_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/include_outside_root/out.test.toml +++ b/acceptance/bundle/includes/include_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml +++ b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/yml_outside_root/out.test.toml b/acceptance/bundle/includes/yml_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/yml_outside_root/out.test.toml +++ b/acceptance/bundle/includes/yml_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/base/out.test.toml b/acceptance/bundle/integration_whl/base/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/base/out.test.toml +++ b/acceptance/bundle/integration_whl/base/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/custom_params/out.test.toml b/acceptance/bundle/integration_whl/custom_params/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/custom_params/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml index 19068d43e0a..5d56f06c3a2 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DATA_SECURITY_MODE = ["USER_ISOLATION", "SINGLE_USER"] diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless/out.test.toml b/acceptance/bundle/integration_whl/serverless/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper/out.test.toml b/acceptance/bundle/integration_whl/wrapper/out.test.toml index 44a1a2186a1..6342b9a4af7 100644 --- a/acceptance/bundle/integration_whl/wrapper/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml index 44a1a2186a1..6342b9a4af7 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index c294b244621..663a49fa779 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index c6fba9c43fb..5daa377779c 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -1,3 +1,9 @@ +# The seed deploy runs an old CLI that knows nothing about the deployment metadata +# service, so the resources it creates are recorded nowhere. Reading state from the +# service then finds none and plans a create on top of them. Adopting resources a +# pre-DMS CLI deployed is a migration story of its own, not something this test covers. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + # $resources references to permissions and grants are not supported on v0.293.0 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 8560caa0ee5..cf188cbc54b 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index ff8a66c196e..ddcf203eb9a 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -1 +1,7 @@ EnvMatrix.READPLAN = ["", "1"] + +# A 1000-task job serializes to ~110 KB, over the 64 KB per-operation state limit the +# deployment metadata service accepts. Recording skips the resource with a warning, so it +# is absent from the state the service reports and the next plan wants to create it again. +# Raising the limit or splitting the state is a service-side decision. +EnvMatrixExclude.dms_state_too_large = ["DATABRICKS_BUNDLE_DMS=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 1d0d883f6d5..d20dc220ff5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -125,3 +125,59 @@ Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"col [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/libraries/maven/out.test.toml b/acceptance/bundle/libraries/maven/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/maven/out.test.toml +++ b/acceptance/bundle/libraries/maven/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml +++ b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/libraries/pypi/out.test.toml b/acceptance/bundle/libraries/pypi/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/pypi/out.test.toml +++ b/acceptance/bundle/libraries/pypi/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml index b37ee45aed6..931153dacc9 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.test.toml +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/lifecycle/started/out.test.toml +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/local_state_staleness/out.test.toml b/acceptance/bundle/local_state_staleness/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/local_state_staleness/out.test.toml +++ b/acceptance/bundle/local_state_staleness/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 71970b719d4..310be221793 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 8c4484f983a..964175b8b01 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -7,3 +7,9 @@ Ignore = [".databricks"] # matrix to ["direct"] so CI's engine filter includes these tests without # also running the same script twice per engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# These tests deploy on terraform first and then migrate to direct. Recording is a +# direct-engine feature, so the resources the terraform half creates are recorded nowhere +# and the migration reads state the service does not have. Migrating a deployment onto the +# service is a story of its own; these tests are not it. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/multi_profile/auto_select/out.test.toml b/acceptance/bundle/multi_profile/auto_select/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/auto_select/out.test.toml +++ b/acceptance/bundle/multi_profile/auto_select/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml +++ b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml +++ b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml +++ b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/open/out.test.toml b/acceptance/bundle/open/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/open/out.test.toml +++ b/acceptance/bundle/open/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/clusters/out.test.toml b/acceptance/bundle/override/clusters/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/clusters/out.test.toml +++ b/acceptance/bundle/override/clusters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_cluster/out.test.toml b/acceptance/bundle/override/job_cluster/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_cluster/out.test.toml +++ b/acceptance/bundle/override/job_cluster/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_cluster_var/out.test.toml b/acceptance/bundle/override/job_cluster_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_cluster_var/out.test.toml +++ b/acceptance/bundle/override/job_cluster_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_tasks/out.test.toml b/acceptance/bundle/override/job_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_tasks/out.test.toml +++ b/acceptance/bundle/override/job_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/merge-string-map/out.test.toml b/acceptance/bundle/override/merge-string-map/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/merge-string-map/out.test.toml +++ b/acceptance/bundle/override/merge-string-map/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/pipeline_cluster/out.test.toml b/acceptance/bundle/override/pipeline_cluster/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/pipeline_cluster/out.test.toml +++ b/acceptance/bundle/override/pipeline_cluster/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/designer_notebook/out.test.toml b/acceptance/bundle/paths/designer_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/designer_notebook/out.test.toml +++ b/acceptance/bundle/paths/designer_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/fallback/out.test.toml b/acceptance/bundle/paths/fallback/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/fallback/out.test.toml +++ b/acceptance/bundle/paths/fallback/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/git_source_jobs/out.test.toml b/acceptance/bundle/paths/git_source_jobs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/git_source_jobs/out.test.toml +++ b/acceptance/bundle/paths/git_source_jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/nominal/out.test.toml b/acceptance/bundle/paths/nominal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/nominal/out.test.toml +++ b/acceptance/bundle/paths/nominal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml +++ b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_globs/out.test.toml b/acceptance/bundle/paths/pipeline_globs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/pipeline_globs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml +++ b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/relative_path_translation/out.test.toml b/acceptance/bundle/paths/relative_path_translation/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/relative_path_translation/out.test.toml +++ b/acceptance/bundle/paths/relative_path_translation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/plan/no_upload/out.test.toml b/acceptance/bundle/plan/no_upload/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/plan/no_upload/out.test.toml +++ b/acceptance/bundle/plan/no_upload/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml +++ b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility/out.test.toml b/acceptance/bundle/python/experimental-compatibility/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/grants-aliases/out.test.toml b/acceptance/bundle/python/grants-aliases/out.test.toml index 98d084e3bb9..c806f1e3811 100644 --- a/acceptance/bundle/python/grants-aliases/out.test.toml +++ b/acceptance/bundle/python/grants-aliases/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/mutator-ordering/out.test.toml b/acceptance/bundle/python/mutator-ordering/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/mutator-ordering/out.test.toml +++ b/acceptance/bundle/python/mutator-ordering/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml index 4c48a83f25b..256d0941b25 100644 --- a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml +++ b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/pipelines-support/out.test.toml b/acceptance/bundle/python/pipelines-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/pipelines-support/out.test.toml +++ b/acceptance/bundle/python/pipelines-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/propagates-auth-env/out.test.toml b/acceptance/bundle/python/propagates-auth-env/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/propagates-auth-env/out.test.toml +++ b/acceptance/bundle/python/propagates-auth-env/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resolve-variable/out.test.toml b/acceptance/bundle/python/resolve-variable/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/resolve-variable/out.test.toml +++ b/acceptance/bundle/python/resolve-variable/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resource-loading/out.test.toml b/acceptance/bundle/python/resource-loading/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/resource-loading/out.test.toml +++ b/acceptance/bundle/python/resource-loading/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/restricted-execution/out.test.toml b/acceptance/bundle/python/restricted-execution/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/restricted-execution/out.test.toml +++ b/acceptance/bundle/python/restricted-execution/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/schemas-support/out.test.toml b/acceptance/bundle/python/schemas-support/out.test.toml index 98d084e3bb9..c806f1e3811 100644 --- a/acceptance/bundle/python/schemas-support/out.test.toml +++ b/acceptance/bundle/python/schemas-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/unicode-support/out.test.toml b/acceptance/bundle/python/unicode-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/unicode-support/out.test.toml +++ b/acceptance/bundle/python/unicode-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/volumes-support/out.test.toml b/acceptance/bundle/python/volumes-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/volumes-support/out.test.toml +++ b/acceptance/bundle/python/volumes-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/quality_monitor/out.test.toml b/acceptance/bundle/quality_monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/quality_monitor/out.test.toml +++ b/acceptance/bundle/quality_monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/refschema/out.test.toml b/acceptance/bundle/refschema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/refschema/out.test.toml +++ b/acceptance/bundle/refschema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml +++ b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/create_error/out.test.toml b/acceptance/bundle/resource_deps/create_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/create_error/out.test.toml +++ b/acceptance/bundle/resource_deps/create_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/id_chain/out.test.toml b/acceptance/bundle/resource_deps/id_chain/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.test.toml +++ b/acceptance/bundle/resource_deps/id_chain/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/id_star/out.test.toml b/acceptance/bundle/resource_deps/id_star/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/id_star/out.test.toml +++ b/acceptance/bundle/resource_deps/id_star/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id/out.test.toml b/acceptance/bundle/resource_deps/job_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_tasks/out.test.toml b/acceptance/bundle/resource_deps/job_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.test.toml +++ b/acceptance/bundle/resource_deps/job_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update/out.test.toml b/acceptance/bundle/resource_deps/jobs_update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/jobs_update/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_self/out.test.toml b/acceptance/bundle/resource_deps/loop_self/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/loop_self/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_self/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml +++ b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/permission_ref/out.test.toml b/acceptance/bundle/resource_deps/permission_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/permission_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..2f48d384b3f 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -14,7 +14,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -29,7 +29,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -98,7 +98,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort ^//import-file/ +>>> print_requests.py --sort ^//import-file/ ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index d38692366b3..2ce8c3e3c5e 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py '^//import-file/' +trace print_requests.py '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py '^//import-file/' +trace print_requests.py '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --sort '^//import-file/' +trace print_requests.py --sort '^//import-file/' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml index 8c738f635ac..4ba1c38c46b 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var/out.test.toml b/acceptance/bundle/resource_deps/resources_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index dc29b70c320..405ffae696a 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -4,3 +4,59 @@ Ignore = [ ".databricks", ".gitignore", ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml +++ b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/alerts/basic/out.test.toml b/acceptance/bundle/resources/alerts/basic/out.test.toml index c45d8e76a8e..eaf2e7a7966 100644 --- a/acceptance/bundle/resources/alerts/basic/out.test.toml +++ b/acceptance/bundle/resources/alerts/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file/out.test.toml b/acceptance/bundle/resources/alerts/with_file/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/alerts/with_file/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-drift/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml +++ b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/default_description/out.test.toml b/acceptance/bundle/resources/apps/default_description/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/default_description/out.test.toml +++ b/acceptance/bundle/resources/apps/default_description/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/immutable/out.test.toml b/acceptance/bundle/resources/apps/immutable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/immutable/out.test.toml +++ b/acceptance/bundle/resources/apps/immutable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/inline_config/out.test.toml b/acceptance/bundle/resources/apps/inline_config/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/apps/inline_config/out.test.toml +++ b/acceptance/bundle/resources/apps/inline_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index c836adff587..79a1ff2e93b 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json > LOG.planjson +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt index 4562a6b9b22..910f5722e38 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //deployments +>>> print_requests.py //deployments ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script index 710dec5a10a..15a9197c3c5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/script +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -15,7 +15,7 @@ rm -f out.requests.txt title "Re-deploy with description change: code deployed again" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION trace errcode $CLI bundle deploy -trace print_requests.py //deployments +trace print_requests.py //deployments ^//api/2.0/bundle rm -f out.requests.txt title "Stop app externally while config says started=true: plan detects drift" diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/apps/resource-refs/out.test.toml b/acceptance/bundle/resources/apps/resource-refs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/resource-refs/out.test.toml +++ b/acceptance/bundle/resources/apps/resource-refs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/update/out.test.toml b/acceptance/bundle/resources/apps/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/update/out.test.toml +++ b/acceptance/bundle/resources/apps/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/basic/out.test.toml b/acceptance/bundle/resources/catalogs/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/catalogs/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml +++ b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml +++ b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml index 5a821e39edc..813b3a187d4 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudEnvs.aws = false CloudEnvs.azure = false CloudEnvs.gcp = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 1370846fe4b..16928c71889 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index f2d80d05de0..06cdfc7caaf 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" diff --git a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index 475b179caed..ebb5db02455 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-name/out.test.toml b/acceptance/bundle/resources/dashboards/change-name/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-name/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml index c2ac722e76a..ec9bbdf7292 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/destroy/out.test.toml b/acceptance/bundle/resources/dashboards/destroy/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/destroy/out.test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml index 1976bc173ca..65ad5749140 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml @@ -2,5 +2,6 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml index b5ce19512e3..2e27b38ff97 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/simple/out.test.toml b/acceptance/bundle/resources/dashboards/simple/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index 8631b921342..e6267ead0a8 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml index 1f9d1fc1b75..281190c64eb 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml @@ -4,4 +4,5 @@ CloudSlow = true RequiresUnityCatalog = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/recreate/out.test.toml b/acceptance/bundle/resources/database_instances/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/database_instances/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_instances/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml index 12ab4ea7f78..d093a69af64 100644 --- a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml +++ b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/experiments/basic/out.test.toml b/acceptance/bundle/resources/experiments/basic/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/experiments/basic/out.test.toml +++ b/acceptance/bundle/resources/experiments/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/external_locations/out.test.toml b/acceptance/bundle/resources/external_locations/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/external_locations/out.test.toml +++ b/acceptance/bundle/resources/external_locations/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.test.toml b/acceptance/bundle/resources/grants/catalogs/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.test.toml +++ b/acceptance/bundle/resources/grants/catalogs/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/registered_models/out.test.toml b/acceptance/bundle/resources/grants/registered_models/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.test.toml +++ b/acceptance/bundle/resources/grants/registered_models/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/volumes/out.test.toml b/acceptance/bundle/resources/grants/volumes/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/volumes/out.test.toml +++ b/acceptance/bundle/resources/grants/volumes/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/independent/out.test.toml b/acceptance/bundle/resources/independent/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/independent/out.test.toml +++ b/acceptance/bundle/resources/independent/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/instance_pools/out.test.toml b/acceptance/bundle/resources/instance_pools/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/instance_pools/out.test.toml +++ b/acceptance/bundle/resources/instance_pools/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/basic/out.test.toml b/acceptance/bundle/resources/job_runs/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/basic/out.test.toml +++ b/acceptance/bundle/resources/job_runs/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml +++ b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml +++ b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/alert-task/out.test.toml b/acceptance/bundle/resources/jobs/alert-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/jobs/alert-task/out.test.toml +++ b/acceptance/bundle/resources/jobs/alert-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml index 71970b719d4..310be221793 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.test.toml +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/big_id/output.txt b/acceptance/bundle/resources/jobs/big_id/output.txt index 82ec469ca83..c539d037f79 100644 --- a/acceptance/bundle/resources/jobs/big_id/output.txt +++ b/acceptance/bundle/resources/jobs/big_id/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", @@ -70,7 +70,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index 0b803cb1b8d..6f0e1215c6a 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,8 +1,8 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index 6dd4efd85d3..467d0855fa4 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,6 +3,12 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [[Repls]] Old = '9223372036854775807' New = '[MAX_INT_64]' diff --git a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml +++ b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/create-error/out.test.toml b/acceptance/bundle/resources/jobs/create-error/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/jobs/create-error/out.test.toml +++ b/acceptance/bundle/resources/jobs/create-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.test.toml b/acceptance/bundle/resources/jobs/delete_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index 022d90a82b7..c3ce3c66802 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/delete_task/out.test.toml b/acceptance/bundle/resources/jobs/delete_task/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index d4bbeb7e7ef..cca4fd1ac9c 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,2 +1,8 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/out.test.toml b/acceptance/bundle/resources/jobs/num_workers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/num_workers/out.test.toml +++ b/acceptance/bundle/resources/jobs/num_workers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index 16e4d600cd2..702558444e9 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -21,7 +21,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script index 8e430e43063..674a820061c 100644 --- a/acceptance/bundle/resources/jobs/num_workers/script +++ b/acceptance/bundle/resources/jobs/num_workers/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace $CLI bundle plan rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index d7593e7b8f7..37a37b0059f 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index d4bbeb7e7ef..cca4fd1ac9c 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,2 +1,8 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt index c97c93273cb..45a761af80e 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt @@ -21,4 +21,4 @@ Updating deployment state... Deployment complete! === No delete API calls for resources that are already gone remotely ->>> print_requests.py //jobs/delete //pipelines/ +>>> print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script index 2b032406a00..45098c8a690 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script @@ -18,4 +18,4 @@ trace $CLI bundle deploy trace $CLI bundle summary &> out.summary.$DATABRICKS_BUNDLE_ENGINE.txt title "No delete API calls for resources that are already gone remotely" -trace print_requests.py //jobs/delete //pipelines/ +trace print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index a067a141773..d1f2a5cf7d5 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -23,4 +23,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 44a0eb849fe..b528539bf92 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,9 +13,9 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml +++ b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/task-source/out.test.toml b/acceptance/bundle/resources/jobs/task-source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/task-source/out.test.toml +++ b/acceptance/bundle/resources/jobs/task-source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/task-source/output.txt b/acceptance/bundle/resources/jobs/task-source/output.txt index 1815edbc0eb..2409cb86a2a 100644 --- a/acceptance/bundle/resources/jobs/task-source/output.txt +++ b/acceptance/bundle/resources/jobs/task-source/output.txt @@ -9,9 +9,9 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body | del(.deployment.deployment_id, .deployment.version_id) out.requests.txt ->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body | del(.deployment.deployment_id, .deployment.version_id) out.requests.txt >>> [CLI] bundle plan update jobs.git_job @@ -24,6 +24,6 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id) out.requests.txt ->>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id) out.requests.txt diff --git a/acceptance/bundle/resources/jobs/task-source/script b/acceptance/bundle/resources/jobs/task-source/script index 838aac092c3..e8908986c50 100644 --- a/acceptance/bundle/resources/jobs/task-source/script +++ b/acceptance/bundle/resources/jobs/task-source/script @@ -2,8 +2,8 @@ trace $CLI bundle deploy # For terraform we expect the task source to be explicitly set always # For direct we do not expect it to be set unless explicitly specified in the bundle config -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body' out.requests.txt | jq --sort-keys > out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys > out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body | del(.deployment.deployment_id, .deployment.version_id)' out.requests.txt | jq --sort-keys > out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body | del(.deployment.deployment_id, .deployment.version_id)' out.requests.txt | jq --sort-keys > out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt # Removing the git_source block and deploying again @@ -13,7 +13,7 @@ trace $CLI bundle deploy # In direct mode update should not contain source unless explicitly specified in the bundle config # In terraform mode update should always contain source field -trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body' out.requests.txt | jq --sort-keys >> out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt -trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys >> out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id)' out.requests.txt | jq --sort-keys >> out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id)' out.requests.txt | jq --sort-keys >> out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml +++ b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/jobs/update/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/jobs/update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/update/output.txt b/acceptance/bundle/resources/jobs/update/output.txt index eea83d272d5..f6a44af17c6 100644 --- a/acceptance/bundle/resources/jobs/update/output.txt +++ b/acceptance/bundle/resources/jobs/update/output.txt @@ -8,7 +8,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -19,7 +19,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -33,7 +33,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 270efcaea6d..c15b425741c 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -2,14 +2,14 @@ echo "*" > .gitignore trace $CLI bundle plan $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) -trace print_requests.py //jobs > out.create.requests.json +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS @@ -17,7 +17,7 @@ trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) -trace print_requests.py //jobs | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json trace $CLI bundle plan @@ -30,7 +30,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index ff8a66c196e..a8a9a1e90b9 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1 +1,7 @@ EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml +++ b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index aca783207f3..ec195ebc965 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -10,7 +10,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -26,7 +26,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 9e84f677e73..d822881fe20 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,17 +1,17 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs > out.create.requests.txt +trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace $CLI bundle plan @@ -19,12 +19,12 @@ title "Fetch job ID and verify remote state" ppid=`read_id.py foo` -trace $CLI jobs get $ppid | jq 'del(.settings.run_as)' +trace $CLI jobs get $ppid | jq 'del(.settings.run_as, .settings.deployment.deployment_id, .settings.deployment.version_id)' rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index e8c0f8a022a..2185aedcef6 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -16,7 +16,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index ebbcb3409b3..7d9103aca52 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes | del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml index 12ab4ea7f78..d093a69af64 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/basic/out.test.toml b/acceptance/bundle/resources/models/basic/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/models/basic/out.test.toml +++ b/acceptance/bundle/resources/models/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/empty-name/out.test.toml b/acceptance/bundle/resources/models/empty-name/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/models/empty-name/out.test.toml +++ b/acceptance/bundle/resources/models/empty-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml index 7845c49f70c..973ce7c68cf 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/factcheck/out.test.toml b/acceptance/bundle/resources/permissions/factcheck/out.test.toml index 581c975b773..64851222e54 100644 --- a/acceptance/bundle/resources/permissions/factcheck/out.test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index ae805caebbc..22a34cb117e 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml index 887e4650a78..0b68a00d7ba 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 96eaf8a4c75..3cfdd63e806 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -2,10 +2,10 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs &> out.deploy.requests.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json # check plan to ensure there is not drift trace $CLI bundle plan -o json -t green > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index 9b877617211..aa99ae397ac 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index 9b877617211..aa99ae397ac 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script index 4c406d91cfa..6d9f9c0453f 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs &> out.deploy.requests.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 9ac6f2cd41a..eb4446a728d 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/out.test.toml b/acceptance/bundle/resources/permissions/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/resources/permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 1a07dad637e..67a775cfb49 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 8cea9634565..7f517420608 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipelines +>>> print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script index d121e073a3d..9055e4ba02b 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script @@ -16,4 +16,4 @@ export PIPELINE_ID # Deploy the bundle that has a pipeline with the same name: trace $CLI bundle deploy -trace print_requests.py //pipelines +trace print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml +++ b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml +++ b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate/out.test.toml b/acceptance/bundle/resources/pipelines/recreate/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 5b9c37402c7..ccbc1936b57 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/out.test.toml b/acceptance/bundle/resources/pipelines/update/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index 255dea2e00d..f48e025aba2 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -4,7 +4,7 @@ touch bar.py trace $CLI bundle deploy print_requests() { - print_requests.py //pipelines + print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id read_state.py pipelines my id name } diff --git a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml +++ b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index 110f841fa05..587c52b624e 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml new file mode 100644 index 00000000000..31e3a00a75e --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml @@ -0,0 +1,3 @@ +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the plan after the unbind still sees the database as tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_databases/update/out.test.toml b/acceptance/bundle/resources/postgres_databases/update/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/update/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index c5b8e7c8a71..67797faa7b2 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -3,4 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index c5b8e7c8a71..50ada34cac8 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -3,4 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml index 3e475e74819..18183d0f8f0 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml @@ -5,3 +5,7 @@ Cloud = false # Deploy error wording differs between engines; the conflict itself is engine-agnostic. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the role staged above still looks tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index 110f841fa05..587c52b624e 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml new file mode 100644 index 00000000000..2123f2f1544 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml @@ -0,0 +1,3 @@ +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the plan after the unbind still sees the role as tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/update/out.test.toml b/acceptance/bundle/resources/postgres_roles/update/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/update/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 08f6c53ae17..fba01abcffb 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 6caf49a7f4e..99e80f5f70a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -26,5 +26,5 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index d67ee41975f..c3e638e9907 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index 72c57e08401..a14879d79f1 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index 879222aa8a4..9ee37b2f03d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 891aece1c11..6cf47e95c59 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -26,7 +26,7 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/create/out.test.toml b/acceptance/bundle/resources/quality_monitors/create/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index 8037d5ec9cc..5390fd078e6 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 78c7853b264..22aaeec97aa 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml +++ b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/registered_models/basic/out.test.toml b/acceptance/bundle/resources/registered_models/basic/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/registered_models/basic/out.test.toml +++ b/acceptance/bundle/resources/registered_models/basic/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/out.test.toml b/acceptance/bundle/resources/schemas/recreate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/schemas/recreate/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/update/out.test.toml b/acceptance/bundle/resources/schemas/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/schemas/update/out.test.toml +++ b/acceptance/bundle/resources/schemas/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 2d469e4abce..e21f54fcf1f 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index b12e98775a3..c7bcc9bf0a5 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt trace $CLI bundle deploy -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml index 6fc644d5164..aa7060abb02 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml index 6b858c4df47..5ab6742ca97 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/secrets/basic/out.test.toml +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/resources/secrets/direct-only/out.test.toml +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/update-value/out.test.toml +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml index 5bbfaf5e65a..fe1401b93c2 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml index 91e30e807cf..82f3670f861 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml index d0abd00ab97..755a233e350 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml index d0abd00ab97..755a233e350 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/out.test.toml b/acceptance/bundle/resources/sql_warehouses/out.test.toml index 355ae0775bc..b1754ac936e 100644 --- a/acceptance/bundle/resources/sql_warehouses/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml index e991fce9180..dbcf84075ed 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 159efe02696..3d68723dbe7 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1 +1,57 @@ RecordRequests = true + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/change-comment/out.test.toml b/acceptance/bundle/resources/volumes/change-comment/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-comment/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-comment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-name/out.test.toml b/acceptance/bundle/resources/volumes/change-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index ba4d63c4032..a897616ae12 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/recreate/out.test.toml b/acceptance/bundle/resources/volumes/recreate/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/volumes/recreate/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 2485737fdf4..b4ca369abe4 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' diff --git a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml index 8c738f635ac..4ba1c38c46b 100644 --- a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/root/env-not-a-directory/out.test.toml b/acceptance/bundle/root/env-not-a-directory/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/env-not-a-directory/out.test.toml +++ b/acceptance/bundle/root/env-not-a-directory/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/env-not-found/out.test.toml b/acceptance/bundle/root/env-not-found/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/env-not-found/out.test.toml +++ b/acceptance/bundle/root/env-not-found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/not-found/out.test.toml b/acceptance/bundle/root/not-found/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/not-found/out.test.toml +++ b/acceptance/bundle/root/not-found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/real-empty-dir/out.test.toml b/acceptance/bundle/root/real-empty-dir/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/real-empty-dir/out.test.toml +++ b/acceptance/bundle/root/real-empty-dir/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run/app-with-job/out.test.toml b/acceptance/bundle/run/app-with-job/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/run/app-with-job/out.test.toml +++ b/acceptance/bundle/run/app-with-job/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/basic/out.test.toml b/acceptance/bundle/run/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/basic/out.test.toml +++ b/acceptance/bundle/run/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/diagnostics/out.test.toml b/acceptance/bundle/run/diagnostics/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/diagnostics/out.test.toml +++ b/acceptance/bundle/run/diagnostics/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/basic/out.test.toml b/acceptance/bundle/run/inline-script/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/basic/out.test.toml +++ b/acceptance/bundle/run/inline-script/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/cwd/out.test.toml b/acceptance/bundle/run/inline-script/cwd/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/cwd/out.test.toml +++ b/acceptance/bundle/run/inline-script/cwd/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-auth/out.test.toml b/acceptance/bundle/run/inline-script/no-auth/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-auth/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-auth/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-separator/out.test.toml b/acceptance/bundle/run/inline-script/no-separator/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-separator/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-separator/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/jobs/partial_run/out.test.toml b/acceptance/bundle/run/jobs/partial_run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/jobs/partial_run/out.test.toml +++ b/acceptance/bundle/run/jobs/partial_run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/no-state/out.test.toml b/acceptance/bundle/run/no-state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/no-state/out.test.toml +++ b/acceptance/bundle/run/no-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/refresh-flags/out.test.toml b/acceptance/bundle/run/refresh-flags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/refresh-flags/out.test.toml +++ b/acceptance/bundle/run/refresh-flags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/basic/out.test.toml b/acceptance/bundle/run/scripts/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/basic/out.test.toml +++ b/acceptance/bundle/run/scripts/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/cwd/out.test.toml b/acceptance/bundle/run/scripts/cwd/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/cwd/out.test.toml +++ b/acceptance/bundle/run/scripts/cwd/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml +++ b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-precedence/out.test.toml b/acceptance/bundle/run/scripts/env-precedence/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-precedence/out.test.toml +++ b/acceptance/bundle/run/scripts/env-precedence/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-section/out.test.toml b/acceptance/bundle/run/scripts/env-section/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-section/out.test.toml +++ b/acceptance/bundle/run/scripts/env-section/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/exit_code/out.test.toml b/acceptance/bundle/run/scripts/exit_code/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/exit_code/out.test.toml +++ b/acceptance/bundle/run/scripts/exit_code/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/io/out.test.toml b/acceptance/bundle/run/scripts/io/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/io/out.test.toml +++ b/acceptance/bundle/run/scripts/io/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no-auth/out.test.toml b/acceptance/bundle/run/scripts/no-auth/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no-auth/out.test.toml +++ b/acceptance/bundle/run/scripts/no-auth/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml +++ b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no_content/out.test.toml b/acceptance/bundle/run/scripts/no_content/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no_content/out.test.toml +++ b/acceptance/bundle/run/scripts/no_content/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/shell/math/out.test.toml b/acceptance/bundle/run/scripts/shell/math/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/shell/math/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/math/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/state-wiped/out.test.toml b/acceptance/bundle/run/state-wiped/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/state-wiped/out.test.toml +++ b/acceptance/bundle/run/state-wiped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/dashboard_embed/out.test.toml b/acceptance/bundle/run_as/dashboard_embed/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/dashboard_embed/out.test.toml +++ b/acceptance/bundle/run_as/dashboard_embed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_override/out.test.toml b/acceptance/bundle/run_as/empty_override/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_override/out.test.toml +++ b/acceptance/bundle/run_as/empty_override/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as/out.test.toml b/acceptance/bundle/run_as/empty_run_as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_run_as/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_sp/out.test.toml b/acceptance/bundle/run_as/empty_sp/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_sp/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user/out.test.toml b/acceptance/bundle/run_as/empty_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_user/out.test.toml +++ b/acceptance/bundle/run_as/empty_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml +++ b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/job_default/out.test.toml b/acceptance/bundle/run_as/job_default/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/run_as/job_default/out.test.toml +++ b/acceptance/bundle/run_as/job_default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run_as/model_serving_different/out.test.toml b/acceptance/bundle/run_as/model_serving_different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/model_serving_different/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/model_serving_matching/out.test.toml b/acceptance/bundle/run_as/model_serving_matching/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/model_serving_matching/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_matching/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/out.test.toml b/acceptance/bundle/run_as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/out.test.toml +++ b/acceptance/bundle/run_as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml +++ b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/test.toml b/acceptance/bundle/run_as/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/run_as/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml +++ b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/out.test.toml b/acceptance/bundle/scripts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/out.test.toml +++ b/acceptance/bundle/scripts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/restricted-execution/out.test.toml b/acceptance/bundle/scripts/restricted-execution/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/restricted-execution/out.test.toml +++ b/acceptance/bundle/scripts/restricted-execution/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/ambiguous/out.test.toml b/acceptance/bundle/select/ambiguous/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/select/ambiguous/out.test.toml +++ b/acceptance/bundle/select/ambiguous/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index 8b995e4d177..9c22c36f16e 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index 55ed5ee6619..c96f9a9f6c9 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/missing/out.test.toml b/acceptance/bundle/select/missing/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/select/missing/out.test.toml +++ b/acceptance/bundle/select/missing/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/rejected/out.test.toml b/acceptance/bundle/select/rejected/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/select/rejected/out.test.toml +++ b/acceptance/bundle/select/rejected/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/select/test.toml b/acceptance/bundle/select/test.toml index 85ce448afd3..792257f226f 100644 --- a/acceptance/bundle/select/test.toml +++ b/acceptance/bundle/select/test.toml @@ -1,3 +1,59 @@ Local = true Cloud = false Ignore = [".databricks", ".gitignore"] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/state/bad_env/out.test.toml b/acceptance/bundle/state/bad_env/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/bad_env/out.test.toml +++ b/acceptance/bundle/state/bad_env/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/basic/out.test.toml b/acceptance/bundle/state/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/basic/out.test.toml +++ b/acceptance/bundle/state/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/engine_default/out.test.toml b/acceptance/bundle/state/engine_default/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/engine_default/out.test.toml +++ b/acceptance/bundle/state/engine_default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/permission_level_migration/out.test.toml b/acceptance/bundle/state/permission_level_migration/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/permission_level_migration/out.test.toml +++ b/acceptance/bundle/state/permission_level_migration/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/state_present/out.test.toml b/acceptance/bundle/state/state_present/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/state_present/out.test.toml +++ b/acceptance/bundle/state/state_present/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/test.toml b/acceptance/bundle/state/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/state/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml +++ b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/summary/modified_status/out.test.toml b/acceptance/bundle/summary/modified_status/out.test.toml index 7f4e2c0ca80..262e580a832 100644 --- a/acceptance/bundle/summary/modified_status/out.test.toml +++ b/acceptance/bundle/summary/modified_status/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"] diff --git a/acceptance/bundle/summary/test.toml b/acceptance/bundle/summary/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/summary/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/sync/dryrun/out.test.toml b/acceptance/bundle/sync/dryrun/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/sync/dryrun/out.test.toml +++ b/acceptance/bundle/sync/dryrun/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/sync/out.test.toml b/acceptance/bundle/sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/sync/out.test.toml +++ b/acceptance/bundle/sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/syncroot/dotdot-git/out.test.toml b/acceptance/bundle/syncroot/dotdot-git/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/syncroot/dotdot-git/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-git/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 4e136c6838f..ef2b279f225 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index f7c4cf648a9..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index f7c4cf648a9..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index f7c4cf648a9..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 14453c07f92..92804cc8ed9 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -1,6 +1,10 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] +# Telemetry reports the byte size of the serialized state, which the deployment stamp +# legitimately grows. The number is the assertion here, so there is nothing to normalize. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helper_username/out.test.toml b/acceptance/bundle/templates-machinery/helper_username/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helper_username/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_username/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml +++ b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/number-precision/out.test.toml b/acceptance/bundle/templates-machinery/number-precision/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/number-precision/out.test.toml +++ b/acceptance/bundle/templates-machinery/number-precision/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/supported-url/out.test.toml b/acceptance/bundle/templates-machinery/supported-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/supported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/supported-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/dbt-sql/out.test.toml b/acceptance/bundle/templates/dbt-sql/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/dbt-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/python/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/skip/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/sql/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/azure-government/out.test.toml b/acceptance/bundle/templates/default-python/azure-government/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/azure-government/out.test.toml +++ b/acceptance/bundle/templates/default-python/azure-government/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/classic/out.test.toml b/acceptance/bundle/templates/default-python/classic/out.test.toml index 2f44fc0b7cc..8a113e1dfd4 100644 --- a/acceptance/bundle/templates/default-python/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index 9f9b4934ffe..2b02b68bfbb 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index 9f9b4934ffe..2b02b68bfbb 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml index 50677b5f636..e5eb55f33db 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.UV_PYTHON = [ "3.9", diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/no-uc/out.test.toml +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml index be193812ec2..40a9dbfa26b 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless/out.test.toml b/acceptance/bundle/templates/default-python/serverless/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-scala/out.test.toml b/acceptance/bundle/templates/default-scala/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-scala/out.test.toml +++ b/acceptance/bundle/templates/default-scala/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-sql/out.test.toml b/acceptance/bundle/templates/default-sql/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-sql/out.test.toml +++ b/acceptance/bundle/templates/default-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/nested-output/out.test.toml b/acceptance/bundle/templates/nested-output/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/nested-output/out.test.toml +++ b/acceptance/bundle/templates/nested-output/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml index 88bd948e0a9..8f126618fbc 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml index 88bd948e0a9..8f126618fbc 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml +++ b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-python/out.test.toml b/acceptance/bundle/templates/telemetry/default-python/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index 977f3725d01..5d7fc7bfa7a 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -1,5 +1,11 @@ # Local-only: At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. +# A template test materializes a whole project and deploys it, taking tens of seconds each, +# and some diff against a sibling test's output directory. Running all of that a second time +# for deployment history recording costs minutes and adds no coverage the rest of the suite +# does not already give, so these opt out. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [[Server]] Pattern = "POST /telemetry-ext" Response.Body = ''' diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index fca8259a3b5..32ccd18e452 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -1,10 +1,44 @@ # This allows recording per-deployment output files, e.g. $CLI bundle deploy > out.$DATABRICKS_BUNDLE_ENGINE.txt EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" +# Runs the whole bundle suite a second time with deployment history recording on, so the +# deployment metadata service (DMS) is exercised by every test rather than only the +# handful under bundle/dms. Empty is the default pair of engine runs. +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] + +# DMS is only supported by the direct engine, and only against the local testserver: +# the service runs in dev and staging, so a cloud run has nothing to record to. +EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] +EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=true"] + +# A saved plan does not carry the deployment stamp. On a first deploy there is no +# deployment to resolve when `bundle plan` runs, so the plan it writes leaves the field +# unset; `deploy --plan` then creates the resources without it and the next plan reports +# drift. Stamping at plan time would mean `bundle plan` creating the deployment record, +# which is a design decision, so the saved-plan path is left out of the DMS run for now. +EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_DMS=true", "READPLAN=1"] + +# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory) and +# refuses a bundle whose state already tracks resources - which most tests here seed. +# Both are forced on: these tests assert what a deploy does, so the resource duplication +# the refusal guards against cannot bite them. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" +Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "1" + +# The DMS run asserts the same golden files as the engine runs. +EnvRepl.DATABRICKS_BUNDLE_DMS = false + +Ignore = ["databricks.yml"] + # The lowest Python version we support. Alternative to "uv run --python 3.10" Env.UV_PYTHON = "3.10" -Ignore = ["databricks.yml"] +# The link a recorded deploy prints, dropped so a test asserts the same output whether or +# not recording is on. The URL itself is covered by workspaceurls.TestDeploymentURL, and +# the calls behind it by bundle/dms/record. +[[Repls]] +Old = '(?m)^Deployment history: .*\n' +New = '' # User-agent: [[Repls]] diff --git a/acceptance/bundle/trampoline/warning_message/out.test.toml b/acceptance/bundle/trampoline/warning_message/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/undefined_resources/out.test.toml b/acceptance/bundle/undefined_resources/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/undefined_resources/out.test.toml +++ b/acceptance/bundle/undefined_resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/upload/internal_server_error/out.test.toml b/acceptance/bundle/upload/internal_server_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/upload/internal_server_error/out.test.toml +++ b/acceptance/bundle/upload/internal_server_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/upload/timeout/out.test.toml b/acceptance/bundle/upload/timeout/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/upload/timeout/out.test.toml +++ b/acceptance/bundle/upload/timeout/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/out.test.toml b/acceptance/bundle/user_agent/out.test.toml index be193812ec2..40a9dbfa26b 100644 --- a/acceptance/bundle/user_agent/out.test.toml +++ b/acceptance/bundle/user_agent/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/simple/out.test.toml b/acceptance/bundle/user_agent/simple/out.test.toml index f784a183258..48cc8d7c1c4 100644 --- a/acceptance/bundle/user_agent/simple/out.test.toml +++ b/acceptance/bundle/user_agent/simple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/test.toml b/acceptance/bundle/user_agent/test.toml index a9f876b6b77..59b0f354509 100644 --- a/acceptance/bundle/user_agent/test.toml +++ b/acceptance/bundle/user_agent/test.toml @@ -3,5 +3,10 @@ RecordRequests = true Local = true IncludeRequestHeaders = ["User-Agent"] +# This test asserts the User-Agent on every single request the CLI makes, so recording's +# extra calls belong in the golden rather than being filtered out - but they are the same +# header the existing requests already cover, so the DMS run only adds entries to maintain. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/anchor_containers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/anchor_containers/out.test.toml +++ b/acceptance/bundle/validate/anchor_containers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/validate/dashboard_defaults/out.test.toml b/acceptance/bundle/validate/dashboard_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_defaults/out.test.toml +++ b/acceptance/bundle/validate/dashboard_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_name/out.test.toml b/acceptance/bundle/validate/dashboard_required_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_required_name/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml +++ b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/null/out.test.toml b/acceptance/bundle/validate/empty_resources/null/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/null/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/null/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_tasks/out.test.toml b/acceptance/bundle/validate/empty_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_tasks/out.test.toml +++ b/acceptance/bundle/validate/empty_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/engine-config-valid/out.test.toml b/acceptance/bundle/validate/engine-config-valid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/engine-config-valid/out.test.toml +++ b/acceptance/bundle/validate/engine-config-valid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum/out.test.toml b/acceptance/bundle/validate/enum/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/enum/out.test.toml +++ b/acceptance/bundle/validate/enum/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum_resource_refs/out.test.toml b/acceptance/bundle/validate/enum_resource_refs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/enum_resource_refs/out.test.toml +++ b/acceptance/bundle/validate/enum_resource_refs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/grants_required_principal/out.test.toml b/acceptance/bundle/validate/grants_required_principal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/grants_required_principal/out.test.toml +++ b/acceptance/bundle/validate/grants_required_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml +++ b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/include_locations/out.test.toml b/acceptance/bundle/validate/include_locations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/include_locations/out.test.toml +++ b/acceptance/bundle/validate/include_locations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-target/out.test.toml b/acceptance/bundle/validate/invalid-engine-target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/invalid-engine-target/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/job-references/out.test.toml b/acceptance/bundle/validate/job-references/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/job-references/out.test.toml +++ b/acceptance/bundle/validate/job-references/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml +++ b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_conversion/out.test.toml b/acceptance/bundle/validate/model_serving_conversion/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/model_serving_conversion/out.test.toml +++ b/acceptance/bundle/validate/model_serving_conversion/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/missing_name/out.test.toml b/acceptance/bundle/validate/models/missing_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/models/missing_name/out.test.toml +++ b/acceptance/bundle/validate/models/missing_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/user_id/out.test.toml b/acceptance/bundle/validate/models/user_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/models/user_id/out.test.toml +++ b/acceptance/bundle/validate/models/user_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml +++ b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/permissions/out.test.toml b/acceptance/bundle/validate/permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/permissions/out.test.toml +++ b/acceptance/bundle/validate/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/permissions_overlap/out.test.toml b/acceptance/bundle/validate/permissions_overlap/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/permissions_overlap/out.test.toml +++ b/acceptance/bundle/validate/permissions_overlap/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml +++ b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix/out.test.toml b/acceptance/bundle/validate/presets_name_prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_name_prefix/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/presets_tags/out.test.toml b/acceptance/bundle/validate/presets_tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_tags/out.test.toml +++ b/acceptance/bundle/validate/presets_tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/required/out.test.toml b/acceptance/bundle/validate/required/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/required/out.test.toml +++ b/acceptance/bundle/validate/required/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml +++ b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml +++ b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/strict/out.test.toml b/acceptance/bundle/validate/strict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/strict/out.test.toml +++ b/acceptance/bundle/validate/strict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sync_patterns/out.test.toml b/acceptance/bundle/validate/sync_patterns/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/sync_patterns/out.test.toml +++ b/acceptance/bundle/validate/sync_patterns/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml +++ b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/volume_defaults/out.test.toml b/acceptance/bundle/validate/volume_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/volume_defaults/out.test.toml +++ b/acceptance/bundle/validate/volume_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/arg-repeat/out.test.toml b/acceptance/bundle/variables/arg-repeat/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/arg-repeat/out.test.toml +++ b/acceptance/bundle/variables/arg-repeat/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cross-ref/out.test.toml b/acceptance/bundle/variables/complex-cross-ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cross-ref/out.test.toml +++ b/acceptance/bundle/variables/complex-cross-ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle-self/out.test.toml b/acceptance/bundle/variables/complex-cycle-self/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cycle-self/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle-self/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle/out.test.toml b/acceptance/bundle/variables/complex-cycle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cycle/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-simple/out.test.toml b/acceptance/bundle/variables/complex-simple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-simple/out.test.toml +++ b/acceptance/bundle/variables/complex-simple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive/out.test.toml b/acceptance/bundle/variables/complex-transitive/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml +++ b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-within-complex/out.test.toml b/acceptance/bundle/variables/complex-within-complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-within-complex/out.test.toml +++ b/acceptance/bundle/variables/complex-within-complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex/out.test.toml b/acceptance/bundle/variables/complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex/out.test.toml +++ b/acceptance/bundle/variables/complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex_multiple_files/out.test.toml b/acceptance/bundle/variables/complex_multiple_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex_multiple_files/out.test.toml +++ b/acceptance/bundle/variables/complex_multiple_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/cycle/out.test.toml b/acceptance/bundle/variables/cycle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/cycle/out.test.toml +++ b/acceptance/bundle/variables/cycle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/double_underscore/out.test.toml b/acceptance/bundle/variables/double_underscore/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/double_underscore/out.test.toml +++ b/acceptance/bundle/variables/double_underscore/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/empty/out.test.toml b/acceptance/bundle/variables/empty/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/empty/out.test.toml +++ b/acceptance/bundle/variables/empty/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/env_overrides/out.test.toml b/acceptance/bundle/variables/env_overrides/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/env_overrides/out.test.toml +++ b/acceptance/bundle/variables/env_overrides/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/file-defaults/out.test.toml b/acceptance/bundle/variables/file-defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/file-defaults/out.test.toml +++ b/acceptance/bundle/variables/file-defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/git-branch/out.test.toml b/acceptance/bundle/variables/git-branch/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/git-branch/out.test.toml +++ b/acceptance/bundle/variables/git-branch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/host/out.test.toml b/acceptance/bundle/variables/host/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/host/out.test.toml +++ b/acceptance/bundle/variables/host/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/int/out.test.toml b/acceptance/bundle/variables/int/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/int/out.test.toml +++ b/acceptance/bundle/variables/int/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_2436/out.test.toml b/acceptance/bundle/variables/issue_2436/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/issue_2436/out.test.toml +++ b/acceptance/bundle/variables/issue_2436/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/lookup/out.test.toml b/acceptance/bundle/variables/lookup/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/variables/lookup/out.test.toml +++ b/acceptance/bundle/variables/lookup/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml +++ b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-builtin/out.test.toml b/acceptance/bundle/variables/resolve-builtin/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-builtin/out.test.toml +++ b/acceptance/bundle/variables/resolve-builtin/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-empty/out.test.toml b/acceptance/bundle/variables/resolve-empty/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-empty/out.test.toml +++ b/acceptance/bundle/variables/resolve-empty/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml +++ b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml +++ b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml +++ b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/unicode_reference/out.test.toml b/acceptance/bundle/variables/unicode_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/unicode_reference/out.test.toml +++ b/acceptance/bundle/variables/unicode_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/vanilla/out.test.toml b/acceptance/bundle/variables/vanilla/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/vanilla/out.test.toml +++ b/acceptance/bundle/variables/vanilla/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/var_in_var/out.test.toml b/acceptance/bundle/variables/var_in_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/var_in_var/out.test.toml +++ b/acceptance/bundle/variables/var_in_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml +++ b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml +++ b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/without_definition/out.test.toml b/acceptance/bundle/variables/without_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/without_definition/out.test.toml +++ b/acceptance/bundle/variables/without_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_file/out.test.toml b/acceptance/bundle/volume_path/invalid_file/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_file/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_file/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_resource/out.test.toml b/acceptance/bundle/volume_path/invalid_resource/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_resource/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_resource/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_root/out.test.toml b/acceptance/bundle/volume_path/invalid_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_root/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_state/out.test.toml b/acceptance/bundle/volume_path/invalid_state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_state/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/valid/out.test.toml b/acceptance/bundle/volume_path/valid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/valid/out.test.toml +++ b/acceptance/bundle/volume_path/valid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b7efb4456f9..b59d1b1da02 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -7,4 +7,20 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` + + // History reports what the deployment metadata service has recorded for this + // bundle. Output only: it is read from the service for 'bundle summary' and is + // unset when the bundle does not record deployment history. + History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` +} + +// DeploymentHistory identifies the bundle's deployment in the deployment +// metadata service. +type DeploymentHistory struct { + // DeploymentID is the ID the service assigned to this bundle's deployment. + DeploymentID string `json:"deployment_id,omitempty"` + + // LatestVersionID is the most recent version recorded for the deployment. It is + // unset when the deployment exists but has no version yet. + LatestVersionID string `json:"latest_version_id,omitempty"` } diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..c3f1465d880 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,10 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Only supported for a bundle with no deployed resources yet: DMS becomes the + // source of truth for resource state, and resources tracked in an existing + // state file cannot be handed over to it yet. See dstate.DeploymentState.Open. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go new file mode 100644 index 00000000000..b90800e4e70 --- /dev/null +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -0,0 +1,63 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +type initializeDeploymentHistory struct{} + +// InitializeDeploymentHistory populates bundle.deployment.history with the +// deployment recorded by the deployment metadata service, for the output of the +// 'bundle summary' command. +// +// NOTE: this makes extra API calls, so like InitializeURLs it should only be used +// when the fields are needed. It is a no-op unless the bundle records deployment +// history. +func InitializeDeploymentHistory() bundle.Mutator { + return &initializeDeploymentHistory{} +} + +func (m *initializeDeploymentHistory) Name() string { + return "InitializeDeploymentHistory" +} + +func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + if !env.RecordsDeploymentHistory(ctx, configured) { + return nil + } + + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + return diag.FromErr(err) + } + if deploymentID == "" { + // Nothing recorded yet: the bundle has not been deployed, or its deployment + // was destroyed. + return nil + } + + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by + // design the service has a deployment for every such node, so this get does not + // have a not-found case. last_version_id is empty until the first version. + dep, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + deploymentID, + }) + if err != nil { + return diag.FromErr(err) + } + + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ + DeploymentID: deploymentID, + LatestVersionID: dep.LastVersionId, + } + return nil +} diff --git a/bundle/config/mutator/initialize_deployment_history_test.go b/bundle/config/mutator/initialize_deployment_history_test.go new file mode 100644 index 00000000000..a0478819bfd --- /dev/null +++ b/bundle/config/mutator/initialize_deployment_history_test.go @@ -0,0 +1,36 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInitializeDeploymentHistoryIsNoOpWithoutRecording(t *testing.T) { + // Without recording there is no deployment to report, and the mutator must not + // make the API calls that would find one. + cases := []struct { + name string + experimental *config.Experimental + }{ + {"experimental unset", nil}, + {"recording disabled", &config.Experimental{RecordDeploymentHistory: false}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: tc.experimental, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, InitializeDeploymentHistory()) + require.NoError(t, diags.Error()) + assert.Nil(t, b.Config.Bundle.Deployment.History) + }) + } +} diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..4f0137fae0f --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,47 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but the service side is not +// ready for users: the deployment metadata service is only deployed to dev and staging, +// and reading state back needs the workspace APIs to expose the deployment's tree node, +// which is still behind a flag. Enabling this today also makes DMS the source of truth +// for resource state, so a bundle that turns it on cannot be turned back. +// +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.ForceAllowRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..1bb172766f2 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + forceAllow string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 5cc6349331d..e9f6d00b321 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -159,7 +159,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go new file mode 100644 index 00000000000..46e0ab7ed41 --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -0,0 +1,69 @@ +package metadata + +import ( + "context" + "strconv" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type annotateDeployment struct { + deploymentID string +} + +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a +// resource in the workspace points back at the deployment that produced it (which is +// how lineage resolves a job to its bundle). +// +// It runs before the plan is computed, since a resource whose deployment is unset +// locally but set in the workspace would otherwise show as drift. +func AnnotateDeployment(deploymentID string) bundle.Mutator { + return &annotateDeployment{deploymentID: deploymentID} +} + +func (m *annotateDeployment) Name() string { + return "metadata.AnnotateDeployment" +} + +func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + } + + return nil +} + +type annotateDeploymentVersion struct { + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. It +// is separate from AnnotateDeployment because the version only exists once +// CreateVersion has claimed one, which happens during deploy. +func AnnotateDeploymentVersion(version int64) bundle.Mutator { + return &annotateDeploymentVersion{version: version} +} + +func (m *annotateDeploymentVersion) Name() string { + return "metadata.AnnotateDeploymentVersion" +} + +func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + versionID := strconv.FormatInt(m.version, 10) + + for _, job := range b.Config.Resources.Jobs { + job.Deployment.VersionId = versionID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.VersionId = versionID + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go new file mode 100644 index 00000000000..297ba95f8ee --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -0,0 +1,50 @@ +package metadata + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotateDeploymentVersion(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my-job": { + JobSettings: jobs.JobSettings{ + Deployment: &jobs.JobDeployment{Kind: jobs.JobDeploymentKindBundle}, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Deployment: &pipelines.PipelineDeployment{Kind: pipelines.DeploymentKindBundle}, + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeployment("dep-123"), AnnotateDeploymentVersion(7)) + require.NoError(t, diags.Error()) + + job := b.Config.Resources.Jobs["my-job"].Deployment + assert.Equal(t, "dep-123", job.DeploymentId) + assert.Equal(t, "7", job.VersionId) + // The kind set by AnnotateJobs is preserved. + assert.Equal(t, jobs.JobDeploymentKindBundle, job.Kind) + + pipeline := b.Config.Resources.Pipelines["my-pipeline"].Deployment + assert.Equal(t, "dep-123", pipeline.DeploymentId) + assert.Equal(t, "7", pipeline.VersionId) + assert.Equal(t, pipelines.DeploymentKindBundle, pipeline.Kind) +} diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index cbb0a2d45ff..d615b5c3566 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -75,7 +75,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.Create) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -116,7 +116,11 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // Drop the state entry so a subsequent failure of Create or WaitAfterDelete // leaves no malformed (empty-ID) entry behind. The next plan will see "no // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a recreate, not a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate rather + // than deliberately removed. + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Recreate) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -158,12 +162,16 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a delete, not the update that caused it: the resource is no longer + // tracked, and DMS drops it from the deployment only for a delete. Recording an + // update would leave it listed with no state, which the next plan cannot read. + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Update) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -208,7 +216,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.UpdateWithID) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -250,7 +258,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } } - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -291,7 +299,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Resize) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..4de9c8d736a 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,14 +86,14 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, deployplan.Create) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,13 +145,13 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn, deployplan.Create) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,13 +215,13 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } // Delete the main resource - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, deployplan.Delete) if err != nil { return err } @@ -235,7 +235,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(key) + err = b.StateDB.DeleteState(ctx, key, deployplan.Delete) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c4178c4e601..76710211c05 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -34,6 +34,19 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // Operations are recorded with DMS from background workers so a resource's + // deploy is not held up by the CreateOperation round trip. The queue is + // drained below, once every apply worker has finished recording. + // + // The state DB records through it, so every state write becomes an operation and + // DMS mirrors the WAL. + opQueue := newOperationQueue(ctx, b.OpRec) + if opQueue != nil { + // Assigned only when non-nil: a nil *operationQueue in an interface is not a + // nil interface, so the state DB's nil check would not see it. + b.StateDB.SetOperationSink(opQueue) + } + g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -64,6 +77,12 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop resource CRUD once uploading DMS state has failed. + if err := opQueue.firstErr(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) @@ -77,14 +96,19 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { + // Read the ID before the delete removes it from state; recording a failure + // below needs it to say which resource the operation refers to. + deletedID := b.StateDB.GetResourceID(resourceKey) if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, action) } else { err = d.Destroy(ctx, &b.StateDB) } if err != nil { + _, priorState := priorRecord(&b.StateDB, resourceKey) + opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -111,8 +135,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a resource that + // writes state more than once (a recreate) reports each step. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { + // Both are empty for a create that never got an ID, which is what the + // service expects for a failed create. + priorID, priorState := priorRecord(&b.StateDB, resourceKey) + opQueue.recordFailure(ctx, resourceKey, action, priorID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -139,6 +170,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the queued operations before returning: the caller completes the + // DMS version right after, and a version must not be completed with uploads + // still in flight. + if err := opQueue.close(); err != nil { + logdiag.LogError(ctx, err) + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..a96e864c761 --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,99 @@ +package dstate + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// RecordedState is what the CLI serializes into the DMS Operation.State field. It +// wraps the config rather than being it, so depends_on survives the round trip: DMS +// has no field for dependency edges, and they cannot be recomputed once references +// are resolved to literals. Nesting them in the config would collide with resource +// fields of the same name (e.g. jobs.Task.depends_on). +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + +// OperationSink records one resource operation with the deployment metadata service. +// SaveState and DeleteState call it for every state write, so what DMS holds mirrors +// the WAL - including the intermediate writes of a recreate. +// +// It does not return an error: the upload happens on a background worker, and the +// deploy learns about a failure when the queue is drained. +type OperationSink interface { + RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) +} + +// readDMSState replaces the file-derived resource state with the state recorded in +// DMS. Recording is only enabled for net-new deployments, so once a deployment +// exists DMS owns its resource set outright - an empty set means a successful deploy +// of nothing, not missing data. The caller holds db.mu. +func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) + if err != nil { + return err + } + + db.Data.State = resources + db.stateIDs = make(map[string]string, len(resources)) + for key, entry := range resources { + db.stateIDs[key] = entry.ID + } + return nil +} + +// fetchDeploymentResources lists every resource recorded for the deployment in +// DMS and maps them into state entries keyed by the fully-qualified resource key. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { + it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: "deployments/" + deploymentID, + }) + + out := make(map[string]ResourceEntry) + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + + // DMS reports resource keys without the "resources." prefix (e.g. + // "jobs.foo"), but the state DB keys are fully qualified + // ("resources.jobs.foo"), so prepend it here. + key := "resources." + res.ResourceKey + + var recorded RecordedState + if res.State != nil { + // State is a string field, so it arrives as a quoted JSON string (see the + // write side in direct.operationRecorder.upload). Unquote it, then parse + // the envelope it holds. + var envelope string + if err := json.Unmarshal(*res.State, &envelope); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } + if err := json.Unmarshal([]byte(envelope), &recorded); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } + } + + // A resource with no id was never created: the deploy that recorded it failed + // before the API assigned one (a failed create is recorded with the error and + // nothing else). Leaving it out keeps it untracked, so the next deploy creates + // it and a destroy skips it - an entry with an empty id would instead look + // tracked and fail the delete with "missing in state". + if res.ResourceId == "" { + continue + } + + out[key] = ResourceEntry{ + ID: res.ResourceId, + State: recorded.State, + DependsOn: recorded.DependsOn, + } + } + return out, nil +} diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..c475942f829 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,89 @@ +package dstate + +import ( + "context" + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/listing" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeResourceLister serves a fixed set of resources from ListResources. It +// embeds the SDK interface so it satisfies it while only overriding the one +// method the read path uses. +type fakeResourceLister struct { + bundledeployments.BundleDeploymentsInterface + resources []bundledeployments.Resource +} + +func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeployments.ListResourcesRequest) listing.Iterator[bundledeployments.Resource] { + return listing.NewIterator( + &req, + func(ctx context.Context, r bundledeployments.ListResourcesRequest) (*bundledeployments.ListResourcesResponse, error) { + return &bundledeployments.ListResourcesResponse{Resources: f.resources}, nil + }, + func(resp *bundledeployments.ListResourcesResponse) []bundledeployments.Resource { + return resp.Resources + }, + func(resp *bundledeployments.ListResourcesResponse) *bundledeployments.ListResourcesRequest { + return nil + }, + ) +} + +func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { + // DMS types state as a string, so the envelope arrives as a quoted JSON string. + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + quoted, err := json.Marshal(envelope) + require.NoError(t, err) + recorded := json.RawMessage(quoted) + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, + {ResourceKey: "pipelines.bar", ResourceId: "456"}, + }} + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") + require.NoError(t, err) + + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, + "resources.pipelines.bar": {ID: "456"}, + }, got) +} + +func TestFetchDeploymentResourcesSkipsResourceWithoutID(t *testing.T) { + // A failed create is recorded with its error and nothing else, so the resource has + // no id. Keeping it would make the resource look tracked while referring to nothing, + // and a later destroy fails with "missing in state" instead of skipping it. + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.created", ResourceId: "123"}, + {ResourceKey: "jobs.failed"}, + }} + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") + require.NoError(t, err) + + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.created": {ID: "123"}, + }, got) +} + +func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { + recorded := json.RawMessage(`not json`) + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, + }} + + _, err := fetchDeploymentResources(t.Context(), f, "dep-1") + assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 7e92aa7c1d2..91d0aae453b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -70,6 +71,18 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // sink records each state write with DMS. Nil unless the bundle records + // deployment history, in which case SetOperationSink installs it. + sink OperationSink +} + +// SetOperationSink makes every subsequent state write also record an operation with +// DMS. It is set after the version is created, which is why it is not an Open option. +func (db *DeploymentState) SetOperationSink(sink OperationSink) { + db.mu.Lock() + defer db.mu.Unlock() + db.sink = sink } type Header struct { @@ -118,7 +131,10 @@ func NewDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// SaveState records the resource's state after action was applied to it. action is +// what the deployment metadata service reports for the write; it is ignored when the +// bundle does not record deployment history. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, action deployplan.ActionType) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -139,13 +155,27 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) - if err == nil { - db.stateIDs[key] = newID + if err != nil { + return err } - return err + db.stateIDs[key] = newID + + // Recorded after the WAL write, so DMS never reports a state the deploy failed to + // persist locally. + if db.sink != nil { + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + db.sink.RecordOperation(ctx, key, action, newID, recorded) + } + + return nil } -func (db *DeploymentState) DeleteState(key string) error { +// DeleteState drops the resource's state entry. action distinguishes a real delete +// from the intermediate drop a recreate performs, both of which are recorded. +func (db *DeploymentState) DeleteState(ctx context.Context, key string, action deployplan.ActionType) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -154,11 +184,28 @@ func (db *DeploymentState) DeleteState(key string) error { return nil } + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] + err := appendJSONLine(db.walFile, WALEntry{Key: key}) - if err == nil { - delete(db.stateIDs, key) + if err != nil { + return err } - return err + delete(db.stateIDs, key) + + // State is nil: the resource no longer exists. + // + // A recreate is the exception. It drops the entry and then saves the new + // resource, but the service keeps one operation per resource per version whose + // action_type is fixed at creation, and it rejects a succeeded recreate that + // carries no state ("it leaves a resource that exists"). So the intermediate drop + // cannot be recorded as its own event; the save that follows reports the recreate, + // and if that save never happens the failure path reports it instead. + if db.sink != nil && action != deployplan.Recreate { + db.sink.RecordOperation(ctx, key, action, deletedID, nil) + } + + return nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { @@ -213,7 +260,33 @@ type ( WithWrite bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// DMSSource tells Open to read resource state from the deployment metadata +// service instead of the state file. Callers pass it only when the bundle set +// experimental.record_deployment_history; a nil *DMSSource keeps Open file-only. +type DMSSource struct { + Client bundledeployments.BundleDeploymentsInterface + + // DeploymentID is resolved from the deployment's workspace node (see + // dms.ResolveDeploymentID), and empty before the first recorded deploy. + DeploymentID string + + // AllowExistingResources records a bundle whose state file already tracks + // resources, instead of refusing it. Those resources are not handed over to DMS: + // the first recorded deploy reports only what it touches, so the ones it does not + // touch are absent from DMS and a later deploy plans them as creates. + // + // It exists for the CLI's own acceptance tests, which run the whole bundle suite + // with recording on. Most of those tests seed a state fixture, and they assert the + // output of a single deploy rather than reading state back, so the duplication the + // refusal prevents cannot bite them. + AllowExistingResources bool +} + +// Open reads the deployment state from disk (and recovers the WAL when +// withRecovery is set). With a non-nil dmsSource, resources come from DMS rather +// than the file. Lineage and serial always come from the file, since that is +// what the write path increments. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -261,6 +334,35 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } + if dmsSource != nil { + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. + // + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 && !dmsSource.AllowExistingResources { + // The remedy is ordered deliberately: this error also blocks destroy, so the + // setting has to come out first or there is no way to tear the bundle down. + return fmt.Errorf(`cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out`, path) + } + if dmsSource.DeploymentID != "" { + if err := db.readDMSState(ctx, dmsSource); err != nil { + return err + } + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 11589944472..9f0374c0ec4 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -1,11 +1,14 @@ package dstate import ( + "context" "encoding/json" + "fmt" "os" "path/filepath" "testing" + "github.com/databricks/cli/bundle/deployplan" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,18 +19,80 @@ func mustFinalize(t *testing.T, db *DeploymentState) { require.NoError(t, err) } +// fakeSink captures what the state writes report to DMS. +type fakeSink struct { + ops []string +} + +func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { + f.ops = append(f.ops, fmt.Sprintf("%s %s id=%s state=%s", action, resourceKey, resourceID, string(state))) +} + +func TestStateWritesRecordOperations(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) + + // A recreate: the old entry is dropped, then the new resource is saved. + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Recreate)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) + mustFinalize(t, &db) + + // The recreate's intermediate drop is not reported: the service keeps one + // operation per resource per version and rejects a succeeded recreate carrying no + // state, so the save that follows is what reports it. + assert.Equal(t, []string{ + `create jobs.my_job id=123 state={"state":{"key":"old"}}`, + `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, + }, sink.ops) +} + +func TestDeleteStateRecordsRealDelete(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) + + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + mustFinalize(t, &db) + + // A real delete reports the id it had and no state: the resource is gone. + assert.Equal(t, []string{ + `create jobs.my_job id=123 state={"state":{}}`, + `delete jobs.my_job id=123 state=`, + }, sink.ops) +} + +func TestStateWritesRecordNothingWithoutSink(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // No sink: recording is off, and the writes still succeed. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + mustFinalize(t, &db) +} + func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, deployplan.Create)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -37,7 +102,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -93,10 +158,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -107,12 +172,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -128,7 +193,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -171,17 +236,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db2.DeleteState("jobs.my_job")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -193,7 +258,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -205,12 +270,12 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go new file mode 100644 index 00000000000..009089cdb85 --- /dev/null +++ b/bundle/direct/opclient.go @@ -0,0 +1,80 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The CLI calls the operations API directly rather than through the generated +// client because the SDK cannot read the response: it types sequence_id as an +// int64, while the service sends it as a JSON string (proto3 encodes 64-bit ints +// that way), so unmarshalling a CreateOperation response fails with +// "invalid character '1' after top-level value". The write itself succeeds - the +// status is 200 - so only the response parse is affected. + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update of this operation. + // Typed as a string because that is what the service sends; see above. + SequenceId string `json:"sequence_id,omitempty"` +} + +// updateOperationRequest carries the fields a later write for the same resource +// changes. action_type and resource_key are omitted: the service fixes them when +// the operation is created and ignores them here. +type updateOperationRequest struct { + State *json.RawMessage `json:"state,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ResourceId string `json:"resource_id,omitempty"` + Status bundledeployments.OperationStatus `json:"status,omitempty"` + SequenceId string `json:"sequence_id,omitempty"` +} + +// operationClient records operations under a deployment version. +type operationClient interface { + CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) + UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) +} + +// apiOperationClient talks to the operations API through the workspace client. +type apiOperationClient struct { + client *client.DatabricksClient +} + +// newAPIOperationClient returns an operationClient that posts to the DMS API. +func newAPIOperationClient(c *client.DatabricksClient) operationClient { + return &apiOperationClient{client: c} +} + +func (a *apiOperationClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/%s/operations", parent) + err := a.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"resource_key": resourceKey}, + op, &result) + if err != nil { + return operationResponse{}, err + } + return result, nil +} + +func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/%s/operations/%s", parent, resourceKey) + err := a.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"update_mask": strings.Join(updatableFields, ",")}, + body, &result) + if err != nil { + return operationResponse{}, err + } + return result, nil +} diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go new file mode 100644 index 00000000000..ea796abf84d --- /dev/null +++ b/bundle/direct/opqueue.go @@ -0,0 +1,222 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" +) + +const ( + // operationQueueSize bounds how many recorded operations wait for upload. Deep + // enough that an apply worker practically never blocks on a free slot. + operationQueueSize = 10 + + // operationUploadWorkers is how many uploads run at a time. + operationUploadWorkers = 8 +) + +// operationQueue uploads recorded operations from background workers, so a deploy +// never waits on the CreateOperation round trip. Two rules shape it: +// +// - One resource, one upload at a time. DMS keeps a single state per resource, so +// overlapping uploads could land out of order and leave the older state. +// - Newest operation wins. Each carries the resource's full state, so a queued +// operation superseded by a newer one is dropped ("coalesced"). +// +// close reports the first upload failure, which fails the deploy: DMS becomes the +// source of truth (see dstate.readDMSState), so a missing record would make the +// next deploy create a resource that already exists. +type operationQueue struct { + uploader operationUploader + + // queue carries resource keys, not operations: a worker looks the operation up + // when it picks the key up, which is what makes coalescing work. + queue chan string + wg sync.WaitGroup + + // mu guards the fields below. + mu sync.Mutex + + // pending holds the operations waiting per resource key, oldest first. Every one + // is uploaded: a resource can write state more than once in a deploy (a recreate + // drops it, then saves the new resource), and each write is its own event, so + // dropping the older one would hide a step. No key means nothing is waiting. + pending map[string][]recordedOperation + + // queuedOrUploading means "some worker will get to this key". Recording such a + // key writes to pending only, so two workers never upload one resource at once. + queuedOrUploading map[string]bool + + err error + closed bool +} + +// newOperationQueue starts the upload workers, returning nil when uploader is nil +// (recording off; every method is a no-op on a nil queue). ctx must outlive close. +func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { + if uploader == nil { + return nil + } + + q := &operationQueue{ + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string][]recordedOperation), + queuedOrUploading: make(map[string]bool), + } + + q.wg.Add(operationUploadWorkers) + for range operationUploadWorkers { + go q.work(ctx) + } + + return q +} + +// RecordOperation implements dstate.OperationSink: every state write becomes an +// operation, so DMS mirrors the WAL. state is already the serialized envelope, and +// nil for a delete. +// +// An earlier upload failure does not stop this: every write is still recorded, best +// effort, so DMS ends up as close to reality as it can get. +func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { + if q == nil { + return + } + + op, err := newStateOperation(action, resourceID, state) + if err != nil { + // The deploy already persisted this write locally, so failing it here would + // report an error about history for a resource that deployed fine. + log.Warnf(ctx, "Not recording operation for %s: %s", resourceKey, err) + return + } + + q.enqueue(ctx, resourceKey, op) +} + +// recordFailure records that applying a resource failed, so the deployment history +// explains the failure instead of omitting the resource. It returns nothing: the +// deploy is already failing, and a second error would mask the one the user needs. +func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { + if q == nil { + return + } + + op, err := newFailedOperation(action, resourceID, priorState, cause) + if err != nil { + log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) + return + } + + q.enqueue(ctx, resourceKey, op) +} + +// enqueue appends op to the operations waiting for resourceKey and makes sure a +// worker will pick it up. +func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { + q.mu.Lock() + q.pending[resourceKey] = append(q.pending[resourceKey], op) + alreadyHandled := q.queuedOrUploading[resourceKey] + q.queuedOrUploading[resourceKey] = true + q.mu.Unlock() + + // A worker will re-read pending before it finishes, so it picks up the operation + // appended above. Queueing again would let a second worker upload the same key. + if alreadyHandled { + return + } + + q.queue <- resourceKey +} + +// close drains the queue and returns the first upload error. Every record caller +// must have returned first (record on a closed queue panics); calling close twice +// is safe, so it can be deferred and still checked at a specific point. +// +// It takes no lock: it runs after every apply worker returned, and wg.Wait orders +// the workers' writes to err before it is read here. +func (q *operationQueue) close() error { + if q == nil { + return nil + } + + if !q.closed { + q.closed = true + close(q.queue) + q.wg.Wait() + } + + return q.err +} + +func (q *operationQueue) work(ctx context.Context) { + defer q.wg.Done() + + for resourceKey := range q.queue { + // Drain this key here instead of re-queueing it: a worker sending to the + // channel it consumes from deadlocks once the queue is full. + for { + op, ok := q.take(resourceKey) + if !ok { + break + } + + // Keep going after a failure, so one bad upload does not drop the records + // for every resource behind it. + if err := q.uploader.upload(ctx, resourceKey, op); err != nil { + q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) + } + } + } +} + +// take claims the oldest operation waiting for resourceKey, reporting false and +// clearing the queuedOrUploading mark when nothing is left, which lets record queue +// it again. Both happen under one lock, so a key can never be left for no worker to +// pick up. +func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + ops := q.pending[resourceKey] + if len(ops) == 0 { + delete(q.pending, resourceKey) + delete(q.queuedOrUploading, resourceKey) + return recordedOperation{}, false + } + + // Oldest first, so the service sees the writes in the order they happened. The + // mark stays until the branch above clears it, so anything recorded during this + // upload is still picked up and no second worker takes the key meanwhile. + q.pending[resourceKey] = ops[1:] + return ops[0], true +} + +// setErr keeps the first upload error; later ones are dropped because one failure +// is enough to fail the deploy. +func (q *operationQueue) setErr(err error) { + q.mu.Lock() + defer q.mu.Unlock() + + if q.err == nil { + q.err = err + } +} + +// firstErr returns the first upload error, or nil if every upload so far +// succeeded. A nil queue (recording disabled) never errors. +func (q *operationQueue) firstErr() error { + if q == nil { + return nil + } + + q.mu.Lock() + defer q.mu.Unlock() + + return q.err +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go new file mode 100644 index 00000000000..1048db35d9c --- /dev/null +++ b/bundle/direct/opqueue_test.go @@ -0,0 +1,378 @@ +package direct + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeUploader records the uploads it receives and optionally blocks until +// release is closed, so a test can hold operations in the queue and observe +// coalescing. +type fakeUploader struct { + block chan struct{} + started chan string + // done receives the resource key after the upload returns, for tests that need + // an upload to have completed rather than merely started. + done chan string + err error + + mu sync.Mutex + uploads []string + actions map[string]bundledeployments.OperationActionType + resourceIDs map[string]string +} + +func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + if f.started != nil { + f.started <- resourceKey + } + if f.block != nil { + <-f.block + } + + f.mu.Lock() + f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + if f.actions == nil { + f.actions = map[string]bundledeployments.OperationActionType{} + f.resourceIDs = map[string]string{} + } + f.actions[resourceKey] = op.action + f.resourceIDs[resourceKey] = op.resourceID + f.mu.Unlock() + + // Sent outside the lock: a test that stops reading this channel would otherwise + // hold f.mu and deadlock every other worker. + if f.done != nil { + f.done <- resourceKey + } + return f.err +} + +func (f *fakeUploader) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.uploads...) +} + +func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { + f.mu.Lock() + defer f.mu.Unlock() + return f.actions[resourceKey] +} + +func (f *fakeUploader) resourceIDFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.resourceIDs[resourceKey] +} + +// envelope builds the serialized RecordedState the state DB hands the queue. +func envelope(t *testing.T, name string) json.RawMessage { + t.Helper() + raw, err := json.Marshal(dstate.RecordedState{State: json.RawMessage(`{"name":"` + name + `"}`)}) + require.NoError(t, err) + return raw +} + +func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { + t.Helper() + q.RecordOperation(t.Context(), resourceKey, deployplan.Update, "id-1", envelope(t, name)) +} + +func TestOperationQueueUploadsEachOperation(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + for i := range 20 { + recordState(t, q, "resources.jobs.job"+strconv.Itoa(i), "n") + } + require.NoError(t, q.close()) + + assert.Len(t, f.recorded(), 20) +} + +func TestOperationQueueUploadsEveryWriteForSameResource(t *testing.T) { + // Hold the first upload so the writes behind it queue up. Each one is its own + // event, so all three are uploaded, oldest first - a resource can legitimately + // write state several times in one deploy (see Recreate). + // + // started is buffered for all three: every write now uploads, and a worker + // blocking on an unread send would deadlock the drain below. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 3)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + // Wait until a worker owns the key, so the operations below are queued behind + // an in-flight upload rather than racing it. + assert.Equal(t, "resources.jobs.foo", <-f.started) + + recordState(t, q, "resources.jobs.foo", "v2") + recordState(t, q, "resources.jobs.foo", "v3") + + close(f.block) + require.NoError(t, q.close()) + + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, + `resources.jobs.foo={"state":{"name":"v3"}}`, + }, f.recorded()) +} + +func TestOperationQueueUploadsQueuedWritesWhileWorkersAreBusy(t *testing.T) { + // Every worker is parked mid-upload, so the writes below sit in pending rather + // than being picked up. Both still go out, in order, once a worker frees up. + // + // started is buffered for the two foo writes as well: nothing reads it after the + // loop below, and a worker blocking on the send would deadlock the drain. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.hold", "v1") + assert.Equal(t, "resources.jobs.hold", <-f.started) + + // Occupy the remaining workers so nothing drains the key under test. + for i := range operationUploadWorkers - 1 { + recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + + // A resource whose ID is only known after it was created: the first write has no + // ID, the second fills it in. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "created")) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "updated")) + + close(f.block) + require.NoError(t, q.close()) + + var uploadsForFoo []string + for _, u := range f.recorded() { + if strings.HasPrefix(u, "resources.jobs.foo=") { + uploadsForFoo = append(uploadsForFoo, u) + } + } + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"created"}}`, + `resources.jobs.foo={"state":{"name":"updated"}}`, + }, uploadsForFoo) + + // The ID recorded last is the one the create learned. + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) + assert.Equal(t, + bundledeployments.OperationActionTypeOperationActionTypeCreate, + f.actionFor("resources.jobs.foo")) +} + +func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { + // Record while the key's own upload is in flight: the key is off the queue but + // still marked, so record does not queue it again. The worker that holds the key + // has to come back for it, or the operation would be silently dropped. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "v1")) + assert.Equal(t, "resources.jobs.foo", <-f.started) + + // The worker has taken the key off the queue and is uploading v1 right now. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v2")) + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up + // after v1 rather than replacing it. The service ends up with the newest state. + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, + }, f.recorded()) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + +func TestOperationQueueReturnsUploadError(t *testing.T) { + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + err := q.close() + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + assert.Contains(t, err.Error(), "resources.jobs.foo") +} + +func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { + // A failed upload must not stop the ones behind it: every applied resource is + // recorded best effort, so DMS ends up as close to reality as it can get. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Wait for the failing upload to finish, so the error is stored before the next + // record rather than racing it. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v1")) + assert.Equal(t, "resources.jobs.foo", <-f.done) + + // The next resource is still accepted, even though the first upload failed. + q.RecordOperation(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "v1")) + + // Both were attempted, and close still reports the failure so the deploy fails. + require.ErrorIs(t, q.close(), uploadErr) + assert.ElementsMatch(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.bar={"state":{"name":"v1"}}`, + }, f.recorded()) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + +func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { + // A failure does not discard work already recorded: the records DMS ends up with + // have to match the resources that were applied. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Every worker is parked mid-upload, so these stay queued. + for i := range operationUploadWorkers { + q.RecordOperation(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", envelope(t, "v1")) + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + q.RecordOperation(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", envelope(t, "v1")) + + close(f.block) + require.ErrorIs(t, q.close(), uploadErr) + + // The queued operation was uploaded rather than dropped on the way out. + assert.Contains(t, f.recorded(), `resources.jobs.queued={"state":{"name":"v1"}}`) + assert.Len(t, f.recorded(), operationUploadWorkers+1) +} + +func TestOperationQueueRecordDropsUnsupportedAction(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + // The state write already succeeded, so an operation that cannot be described is + // dropped with a warning rather than failing the deploy. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + +func TestOperationQueueRecordDropsOversizedState(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + +func TestOperationQueueCloseIsIdempotent(t *testing.T) { + f := &fakeUploader{err: errors.New("boom")} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + require.Error(t, q.close()) + // A second close reports the same error instead of panicking on the already + // closed channel, so callers can both defer close and check it explicitly. + require.Error(t, q.close()) +} + +// serialUploader fails if two uploads for the same resource key ever overlap. +type serialUploader struct { + mu sync.Mutex + live map[string]bool + last map[string]string + uneven bool +} + +func (s *serialUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + s.mu.Lock() + if s.live[resourceKey] { + s.uneven = true + } + s.live[resourceKey] = true + s.mu.Unlock() + + s.mu.Lock() + defer s.mu.Unlock() + s.live[resourceKey] = false + s.last[resourceKey] = string(op.state) + return nil +} + +func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { + // Two workers must never upload the same resource at the same time. DMS stores + // one state per resource, so concurrent uploads can finish out of order and + // leave the older state as the final one. + // + // Lots of goroutines record a small set of keys, so the same key is recorded + // repeatedly while its earlier upload may still be running. serialUploader flags + // any overlap it sees. + // + // Whether a bug shows up depends on how the scheduler interleaves things, so one + // pass proves little - repeat it to get many chances at a bad ordering. + const ( + iterations = 200 + workers = 10 + perWorker = 5 + distinctKeyMod = 12 + ) + + for range iterations { + ctx := t.Context() + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(ctx, u) + + // The envelopes are built up front: json.Marshal is fine on many goroutines, + // but the helper takes *testing.T, which is not. + states := make([]json.RawMessage, workers) + for w := range workers { + states[w] = envelope(t, strconv.Itoa(w)) + } + + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + q.RecordOperation(ctx, key, deployplan.Update, "id-1", states[w]) + } + }) + } + wg.Wait() + require.NoError(t, q.close()) + + require.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + require.Len(t, u.last, distinctKeyMod) + require.Empty(t, q.pending) + require.Empty(t, q.queuedOrUploading) + } +} + +func TestNilOperationQueueIsNoOp(t *testing.T) { + // Recording is disabled: newOperationQueue returns nil and every method is a + // no-op, so Apply does not have to branch. + q := newOperationQueue(t.Context(), nil) + require.Nil(t, q) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil) + require.NoError(t, q.close()) +} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go new file mode 100644 index 00000000000..26afccd8f2f --- /dev/null +++ b/bundle/direct/oprecorder.go @@ -0,0 +1,243 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// maxOperationStateSize is the largest serialized state DMS accepts per +// operation. Uploading more is rejected server-side, so fail early with a +// message that names the resource. +const maxOperationStateSize = 64 * 1024 + +// maxOperationErrorMessageSize is the largest error message DMS accepts per +// operation. A longer message is truncated rather than rejected, so a failing +// resource is still recorded with its error instead of the recording itself +// failing and masking the error we are trying to report. +const maxOperationErrorMessageSize = 16 * 1024 + +// recordedOperation is an applied resource operation, serialized and waiting to be +// uploaded to the deployment metadata service (DMS). +// +// The payload is built on the apply worker rather than in the uploader so the +// queue does not hold on to the live resource struct, and so a malformed state +// fails the resource that produced it instead of the drain at the end of apply. +type recordedOperation struct { + action bundledeployments.OperationActionType + resourceID string + status bundledeployments.OperationStatus + + // errorMessage is why the operation failed. It is set only when status is + // failed, which the service enforces. + errorMessage string + + // state is the serialized local config after the operation. It is nil for a + // delete, where the resource no longer exists, and for a failure, where the + // resource was not written. + state json.RawMessage +} + +// newStateOperation describes a state write for upload. state is the serialized +// RecordedState envelope the state DB just persisted, and nil for a delete, where +// the resource is gone. It errors when the state exceeds maxOperationStateSize. +func newStateOperation(action deployplan.ActionType, resourceID string, state json.RawMessage) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + if len(state) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) + } + + return recordedOperation{ + action: actionType, + resourceID: resourceID, + status: bundledeployments.OperationStatusOperationStatusSucceeded, + state: state, + }, nil +} + +// newFailedOperation records an operation that did not apply, so the deployment +// history says why a resource failed rather than just omitting it. +// +// priorState is the resource's state from before the deploy, carried through +// unchanged: an action other than a create acts on a resource that still exists, and +// the service rejects such an operation without state because dropping it would +// leave DMS unable to describe a resource it still owns. It is nil for a create, +// where there is no prior state and no resource to describe - which is also why the +// resourceID may be empty for CREATE and RECREATE. +func newFailedOperation(action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + message := cause.Error() + if len(message) > maxOperationErrorMessageSize { + message = message[:maxOperationErrorMessageSize] + } + + return recordedOperation{ + action: actionType, + resourceID: resourceID, + status: bundledeployments.OperationStatusOperationStatusFailed, + errorMessage: message, + state: priorState, + }, nil +} + +// priorRecord returns the resource's id and state from before this deploy, in the +// same envelope form the success path uploads, or empty values when the resource has +// no prior record (a create). A failed operation reports these unchanged: the resource +// is whatever it was before the attempt. +// +// Both come from the same pre-deploy entry because the service requires an id +// alongside state: state describes a resource that exists, so it needs the id to say +// which one. Reading the id from live state instead would return "" for a failed +// recreate, whose delete step already dropped it, and the mismatch is rejected. +func priorRecord(db *dstate.DeploymentState, resourceKey string) (string, json.RawMessage) { + entry, ok := db.GetResourceEntry(resourceKey) + if !ok || len(entry.State) == 0 { + return "", nil + } + + raw, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return "", nil + } + return entry.ID, raw +} + +// operationUploader records an applied resource operation with DMS. Uploads run +// on the operationQueue workers, off the apply path. +type operationUploader interface { + upload(ctx context.Context, resourceKey string, op recordedOperation) error +} + +// operationRecorder uploads operations via the DMS operations API. +type operationRecorder struct { + ops operationClient + // parent is the version the operations are recorded under, formatted as + // "deployments/{deployment_id}/versions/{version_id}". + parent string + + // mu guards sequenceIDs. + mu sync.Mutex + + // sequenceIDs holds the last sequence_id the service returned per resource key, + // which is how a resource already recorded in this version is recognised. The + // service names operations "operations/{resource_key}", so it keeps one per + // resource per version: the second write for a resource has to update that + // operation, and echo this value as the concurrency precondition. + sequenceIDs map[string]string +} + +// NewOperationRecorder returns an operationUploader backed by the DMS operations +// API. deploymentID and version identify the deployment version assigned by DMS +// that the operations are recorded under. +func NewOperationRecorder(apiClient *client.DatabricksClient, deploymentID string, version int64) operationUploader { + return newOperationRecorder(newAPIOperationClient(apiClient), deploymentID, version) +} + +// newOperationRecorder is the internal constructor, so tests can supply their own +// operationClient. +func newOperationRecorder(ops operationClient, deploymentID string, version int64) operationUploader { + return &operationRecorder{ + ops: ops, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + sequenceIDs: make(map[string]string), + } +} + +// updatableFields are the operation fields a later write for the same resource can +// change. resource_id is included because a recreate learns a new one. +var updatableFields = []string{"state", "error_message", "resource_id", "status"} + +func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state + // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on + // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). + dmsKey := strings.TrimPrefix(resourceKey, "resources.") + + operation := bundledeployments.Operation{ + ActionType: op.action, + ResourceId: op.resourceID, + ResourceKey: dmsKey, + Status: op.status, + ErrorMessage: op.errorMessage, + } + if op.state != nil { + // DMS types state as a string, so the JSON goes on the wire as a quoted + // string rather than an embedded object. The SDK field is a json.RawMessage, + // so quote the payload here; sending the object directly is rejected with + // "Invalid value: {...} for expected type: STRING". + quoted, err := json.Marshal(string(op.state)) + if err != nil { + return fmt.Errorf("serializing state: %w", err) + } + raw := json.RawMessage(quoted) + operation.State = &raw + } + + r.mu.Lock() + sequenceID, recorded := r.sequenceIDs[dmsKey] + r.mu.Unlock() + + var result operationResponse + var err error + if recorded { + // Only the masked fields and sequence_id are read on an update; action_type + // stays as the operation was created, so sending it would just be misleading. + result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, updateOperationRequest{ + State: operation.State, + ErrorMessage: operation.ErrorMessage, + ResourceId: operation.ResourceId, + Status: operation.Status, + SequenceId: sequenceID, + }) + } else { + result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) + } + if err != nil { + return err + } + + // Remember the sequence the service assigned, so the next write for this + // resource updates rather than re-creates. + r.mu.Lock() + r.sequenceIDs[dmsKey] = result.SequenceId + r.mu.Unlock() + + return nil +} + +// deployActionToSDK maps a deployplan action to its DMS operation action type. +// Only actions that mutate a resource are recordable; Skip and Undefined never +// reach a recorder and are rejected rather than silently coerced. +func deployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go new file mode 100644 index 00000000000..d80caa09341 --- /dev/null +++ b/bundle/direct/oprecorder_test.go @@ -0,0 +1,185 @@ +package direct + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeOpCall is one recorded call to the operations API. +type fakeOpCall struct { + method string + parent string + resourceKey string + op bundledeployments.Operation + update updateOperationRequest +} + +type fakeOpClient struct { + mu sync.Mutex + calls []fakeOpCall + // sequence is what the service reports back; a string, as the service sends it. + sequence string +} + +func (f *fakeOpClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, fakeOpCall{method: "create", parent: parent, resourceKey: resourceKey, op: op}) + return operationResponse{SequenceId: f.sequence}, nil +} + +func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body}) + return operationResponse{SequenceId: f.sequence}, nil +} + +// uploadOne records a single operation through the given uploader, mirroring what +// an operationQueue worker does. +func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { + t.Helper() + op, err := newStateOperation(action, resourceID, state) + require.NoError(t, err) + require.NoError(t, u.upload(t.Context(), resourceKey, op)) +} + +func TestOperationRecorderStripsResourcePrefix(t *testing.T) { + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) + + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", envelope(t, "foo")) + + require.Len(t, f.calls, 1) + c := f.calls[0] + // The wire key drops the CLI-internal "resources." prefix, both in the query + // param and the operation body. + assert.Equal(t, "create", c.method) + assert.Equal(t, "jobs.foo", c.resourceKey) + assert.Equal(t, "jobs.foo", c.op.ResourceKey) + assert.Equal(t, "deployments/dep-1/versions/2", c.parent) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, c.op.ActionType) + assert.Equal(t, "job-123", c.op.ResourceId) + require.NotNil(t, c.op.State) +} + +func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { + // One operation per resource per version: the second write has to update the + // first, echoing the sequence_id the service returned as its precondition. + f := &fakeOpClient{sequence: "7"} + r := newOperationRecorder(f, "dep-1", 2) + + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "", nil) + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "job-456", envelope(t, "new")) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) + + assert.Equal(t, "update", f.calls[1].method) + assert.Equal(t, "jobs.foo", f.calls[1].resourceKey) + assert.Equal(t, "7", f.calls[1].update.SequenceId) + assert.Equal(t, "job-456", f.calls[1].update.ResourceId) + require.NotNil(t, f.calls[1].update.State) +} + +func TestOperationRecorderTracksSequencePerResource(t *testing.T) { + // A different resource has its own operation, so its first write creates. + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) + + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "foo")) + uploadOne(t, r, "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "bar")) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) + assert.Equal(t, "create", f.calls[1].method) +} + +func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { + // The state DB serializes the envelope (see dstate.SaveState); the operation + // carries it through untouched, sensitive fields and all. + state := json.RawMessage(`{"state":{"name":"foo","token":"super-secret"}}`) + + op, err := newStateOperation(deployplan.Create, "job-123", state) + require.NoError(t, err) + + assert.JSONEq(t, string(state), string(op.state)) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, op.status) +} + +func TestNewStateOperationRejectsUnsupportedAction(t *testing.T) { + _, err := newStateOperation(deployplan.Skip, "job-123", nil) + assert.Error(t, err) +} + +func TestNewStateOperationRejectsOversizedState(t *testing.T) { + big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) + + _, err := newStateOperation(deployplan.Create, "job-123", big) + assert.ErrorContains(t, err, "exceeds the 65536 byte limit") +} + +func TestNewFailedOperationRecordsError(t *testing.T) { + op, err := newFailedOperation(deployplan.Create, "", nil, errors.New("cluster spec is invalid")) + require.NoError(t, err) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) + assert.Equal(t, "cluster spec is invalid", op.errorMessage) + // The resource was never written, so there is no state to serve back for it. + assert.Nil(t, op.state) +} + +func TestNewFailedOperationRecordsPriorStateWithID(t *testing.T) { + // A failed recreate has already deleted the resource, so the id must come from + // the pre-deploy record alongside the state: the service rejects state without + // an id, since state describes a resource that exists. + op, err := newFailedOperation(deployplan.Recreate, "main.some_schema", json.RawMessage(`{"state":{"catalog_name":"main"}}`), errors.New("Catalog 'mainx' does not exist")) + require.NoError(t, err) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) + assert.Equal(t, "main.some_schema", op.resourceID) + assert.JSONEq(t, `{"state":{"catalog_name":"main"}}`, string(op.state)) +} + +func TestNewFailedOperationTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + op, err := newFailedOperation(deployplan.Update, "job-123", nil, errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) + require.NoError(t, err) + + assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) +} + +func TestDeployActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := deployActionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Skip and Undefined never reach a recorder and are rejected. + _, err := deployActionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = deployActionToSDK(deployplan.Undefined) + assert.Error(t, err) +} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..03864af5da2 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -44,6 +44,13 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // OpRec uploads each applied resource operation to the deployment metadata + // service (DMS). It is nil unless the bundle opts into recording deployment + // history, in which case the phases package sets it after CreateVersion. + // Apply queues the operations and drains them before returning, so the + // uploads do not block the resources being deployed. + OpRec operationUploader } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/env/dms.go b/bundle/env/dms.go new file mode 100644 index 00000000000..812d492d4bc --- /dev/null +++ b/bundle/env/dms.go @@ -0,0 +1,37 @@ +package env + +import "context" + +// DMSVariable names the environment variable that turns on deployment history +// recording without setting experimental.record_deployment_history in the bundle. +// It exists for the CLI's own acceptance tests, which run the whole bundle suite +// with DMS enabled: setting it here beats adding the field to every databricks.yml. +// +// Like ForceAllowRecordDeploymentHistoryVariable it is deliberately undocumented; see +// validate.ValidateRecordDeploymentHistory for why the feature is still gated off. +const DMSVariable = "DATABRICKS_BUNDLE_DMS" + +// DMS reports whether the environment turns on deployment history recording. +func DMS(ctx context.Context) bool { + value, ok := get(ctx, []string{DMSVariable}) + return ok && value != "" && value != "0" && value != "false" +} + +// RecordsDeploymentHistory reports whether this deploy records deployment history, +// from either the bundle setting or DMSVariable. It is the single predicate the +// recording code paths branch on, so the env var and the config field cannot drift. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + return configured || DMS(ctx) +} + +// DMSAllowExistingResourcesVariable names the environment variable that lets a bundle +// with resources already in its state file be recorded, which is otherwise refused +// (see dstate.DMSSource.AllowExistingResources for what that costs). +const DMSAllowExistingResourcesVariable = "DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES" + +// DMSAllowExistingResources reports whether the environment allows recording a bundle +// that already tracks resources. +func DMSAllowExistingResources(ctx context.Context) bool { + value, ok := get(ctx, []string{DMSAllowExistingResourcesVariable}) + return ok && value != "" && value != "0" && value != "false" +} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go new file mode 100644 index 00000000000..f449d5ec6b9 --- /dev/null +++ b/bundle/env/dms_test.go @@ -0,0 +1,38 @@ +package env + +import ( + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" +) + +func TestDMS(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"true", true}, + {"1", true}, + {"", false}, + {"0", false}, + {"false", false}, + } { + ctx := env.Set(t.Context(), DMSVariable, tc.value) + assert.Equal(t, tc.want, DMS(ctx), "value %q", tc.value) + } +} + +func TestDMSUnset(t *testing.T) { + assert.False(t, DMS(t.Context())) +} + +func TestRecordsDeploymentHistory(t *testing.T) { + // The bundle setting alone is enough, and so is the environment; the env var + // exists so the acceptance suite can record without touching every databricks.yml. + assert.True(t, RecordsDeploymentHistory(t.Context(), true)) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + + ctx := env.Set(t.Context(), DMSVariable, "true") + assert.True(t, RecordsDeploymentHistory(ctx, false)) +} diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index c08cf01c31f..459c14a0b6d 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -233,7 +233,9 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + // Migration rebuilds local state from terraform's; nothing is deployed, and + // the DMS sink is never set on this state, so the action is not reported. + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn, deployplan.Create); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index e585fe51609..d0faca36bca 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -24,6 +24,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -75,7 +76,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting) { +func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, recorder *dms.Recorder) { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. cmdio.LogString(ctx, "Deploying resources...") @@ -121,6 +122,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") + logDeploymentHistory(ctx, b, recorder) } // Once the deploy is complete, dry-run the migration to the direct engine @@ -164,7 +166,21 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // Set up DMS recording of this deployment as a version. The version is not + // created until the plan is approved (below), so a cancelled deploy records + // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. + // CompleteVersion is deferred before lock.Release so it runs while the lock + // is still held (defers run last-in-first-out). + recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -215,6 +231,26 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + // Create the version before planning: the plan snapshots the resource config, so + // the version has to be stamped on before it is computed or the applied resources + // would not carry it. A cancelled deploy therefore leaves a version behind, + // completed as a failure by the deferred CompleteVersion. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + // The deployment ID is stamped earlier, when the state is opened; only the + // version is new here. A first deploy has no ID until now, so stamp both. + bundle.ApplySeqContext(ctx, b, + metadata.AnnotateDeployment(recorder.DeploymentID()), + metadata.AnnotateDeploymentVersion(recorder.Version()), + ) + if logdiag.HasError(ctx) { + return + } + } + planFromFile := plan != nil if plan == nil { // State is already open for read by process.go (for direct engine) @@ -258,7 +294,10 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { - deployCore(ctx, b, plan, stateEngine, requestedEngine) + // Record operations under the version created before planning, so DMS holds + // the deployed resource state. + setOperationRecorder(ctx, b, recorder) + deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) } else { cmdio.LogString(ctx, "Deployment cancelled!") return diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2b52d575344..e09d5716f5f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -80,7 +81,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { +func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType, recorder *dms.Recorder) { if engine.IsDirect() { b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { @@ -104,6 +105,15 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // Complete the version before deleting the remote files. The deployment is a + // node under the state directory, so files.Delete removes it and any later call + // fails with 404. CompleteVersion is idempotent, so the deferred call in Destroy + // is a no-op after this. + if err := recorder.CompleteVersion(ctx, true); err != nil { + logdiag.LogError(ctx, err) + return + } + bundle.ApplyContext(ctx, b, files.Delete()) if !logdiag.HasError(ctx) { @@ -131,7 +141,19 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // Set up DMS recording of this destroy as a version. The version is not + // created until the destroy is approved (below), so a cancelled destroy + // records nothing; the deferred CompleteVersion is a no-op until then. It is + // deferred before lock.Release so it runs while the lock is still held. + recorder, err := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() @@ -188,7 +210,14 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } - destroyCore(ctx, b, plan, engine) + // Record the DMS version now that the destroy is approved and the state WAL + // has been opened, then record each delete operation under it. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + setOperationRecorder(ctx, b, recorder) + destroyCore(ctx, b, plan, engine, recorder) } else { cmdio.LogString(ctx, "Destroy cancelled!") } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..8ca73fac94e --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,159 @@ +package phases + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// newDeploymentRecorder returns a dms.Recorder for the current deployment, or +// nil when DMS recording does not apply. A nil recorder is a no-op, so callers +// do not need to branch on it. +// +// Recording is enabled only when the bundle asks for it (see +// recordsDeploymentHistory) AND the engine is direct: DMS resource state is tracked +// per direct-engine deployment. Returning nil for terraform leaves those untouched. +// +// The deployment ID is resolved from the workspace, not local state (see +// dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is +// held, so it sees any deployment a concurrent deploy created. It is empty on the +// first recorded deploy, where the recorder creates the deployment instead. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { + if !recordsDeploymentHistory(ctx, b) { + return nil, nil + } + if !eng.IsDirect() { + return nil, nil + } + + statePath := b.Config.Workspace.StatePath + deploymentID, err := dms.ResolveDeploymentID(ctx, b.WorkspaceClient(ctx), statePath) + if err != nil { + return nil, err + } + apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + if err != nil { + return nil, err + } + return dms.NewRecorder(dms.RecorderOptions{ + Service: b.WorkspaceClient(ctx).BundleDeployments, + Versions: dms.NewAPIVersionCreator(apiClient), + DeploymentID: deploymentID, + StatePath: statePath, + VersionType: versionType, + Metadata: deploymentMetadata(b), + }), nil +} + +// recordsDeploymentHistory reports whether this bundle records deployment history, +// from experimental.record_deployment_history or DATABRICKS_BUNDLE_DMS. +func recordsDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + +// setOperationRecorder points the deployment at the version the recorder claimed, so +// the state writes during apply are recorded under it. A nil recorder means recording +// is off and leaves the deployment's uploader unset. +func setOperationRecorder(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { + if recorder == nil { + return + } + + apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + b.DeploymentBundle.OpRec = direct.NewOperationRecorder(apiClient, recorder.DeploymentID(), recorder.Version()) +} + +// logDeploymentHistory links to the deployment this deploy was recorded under, so +// the user can open its history without hunting for the ID. A nil recorder means +// recording is off, and a zero version means the version was never created. +// +// The workspace ID is left out of the URL: the page redirects correctly without it, +// and omitting it keeps the line short enough to stay clickable in a terminal. +func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { + if recorder == nil || recorder.Version() == 0 { + return + } + + baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) + if err != nil { + // Only the link is lost, so report the deployment without it rather than + // failing a deploy that already succeeded. + log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) + cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d", recorder.DeploymentID(), recorder.Version())) + return + } + + cmdio.LogString(ctx, "Deployment history: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) +} + +// deploymentMetadata describes the bundle this deploy came from and where it +// landed, mirroring what bundle/deploy/metadata computes for the metadata file. +func deploymentMetadata(b *bundle.Bundle) dms.Metadata { + p := dms.Metadata{ + DisplayName: b.Config.Bundle.Name, + TargetName: b.Config.Bundle.Target, + Mode: deploymentModeToSDK(b.Config.Bundle.Mode), + } + + git := b.Config.Bundle.Git + if git.Branch != "" || git.Commit != "" || git.OriginURL != "" { + p.Git = &bundledeployments.GitInfo{ + Branch: git.Branch, + Commit: git.Commit, + OriginUrl: git.OriginURL, + } + } + + ws := &bundledeployments.WorkspaceInfo{ + RootPath: b.Config.Workspace.RootPath, + FilePath: b.Config.Workspace.FilePath, + } + // In a source-linked deployment files are not copied, so resources read them + // from the sync root instead of file_path (see bundle/deploy/metadata.Compute). + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + ws.FilePath = b.SyncRootPath + ws.SourceLinked = true + } + // Only a deploy from a Databricks Git folder has one; a local worktree does not. + // bundle_root_path is relative to it, so the service requires both or neither. + if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { + ws.GitFolderPath = b.WorktreeRoot.Native() + ws.BundleRootPath = git.BundleRootPath + } + p.Workspace = ws + + return p +} + +// deploymentModeToSDK maps the bundle target's mode to the DMS enum. An unset mode +// maps to empty, which the service reads as "not reported". +func deploymentModeToSDK(mode config.Mode) bundledeployments.DeploymentMode { + switch mode { + case config.Development: + return bundledeployments.DeploymentModeDeploymentModeDevelopment + case config.Production: + return bundledeployments.DeploymentModeDeploymentModeProduction + default: + return "" + } +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 873dc31dc07..ce03411d0f7 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,6 +185,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) + // Rejects experimental.record_deployment_history: the feature is not usable yet. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index b3a55a607cc..d533517f998 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -27,10 +27,11 @@ Useful after deployment to see what was created and where to find it.`, cmd.RunE = func(cmd *cobra.Command, args []string) error { b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ - ReadState: true, - AlwaysPull: forcePull, - IncludeLocations: includeLocations, - InitIDs: true, + ReadState: true, + AlwaysPull: forcePull, + IncludeLocations: includeLocations, + InitIDs: true, + InitDeploymentHistory: true, }) if err != nil { return err diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e4f232605ce..f2b1750205e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -11,15 +11,18 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -55,6 +58,11 @@ type ProcessOptions struct { // Implies ReadState InitIDs bool + // If true, calls InitializeDeploymentHistory() to look up the bundle's recorded + // deployment. Independent of InitIDs, and costs its own API calls. + // Implies ReadState + InitDeploymentHistory bool + // if true, pass ErrorOnEmptyState to statemgmt.Load // Implies ReadState ErrorOnEmptyState bool @@ -179,7 +187,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, nil, err } - shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.InitDeploymentHistory || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" if shouldReadState { // PullResourcesState depends on stateFiler which needs b.Config.Workspace.StatePath which is set in phases.Initialize @@ -211,7 +219,36 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + // When the bundle records deployment history, the deployment metadata + // service owns resource state, so hand Open a DMS source to read it from + // there instead of the file. The local identity (lineage/serial) still + // comes from the file. Reads open the state write-disabled, so no lineage + // is minted here. + var dmsSource *dstate.DMSSource + if env.RecordsDeploymentHistory(ctx, b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory) { + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + dmsSource = &dstate.DMSSource{ + Client: w.BundleDeployments, + DeploymentID: deploymentID, + AllowExistingResources: env.DMSAllowExistingResources(ctx), + } + + // Stamp the deployment onto the resources before anything diffs them. + // The workspace has it, so a plan that left it unset would report drift + // on a resource nobody touched. The version is stamped by the deploy + // phase instead, once it claims one. + bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(deploymentID)) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -246,6 +283,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } } + + // Independent of the resource IDs above: this reads the deployment record, not + // the state. It makes its own API calls, so only 'bundle summary' asks for it. + if opts.InitDeploymentHistory { + bundle.ApplyContext(ctx, b, mutator.InitializeDeploymentHistory()) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } } var plan *deployplan.Plan diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go new file mode 100644 index 00000000000..001126f4fb6 --- /dev/null +++ b/libs/dms/recorder.go @@ -0,0 +1,350 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The server expires a version's lease if it does not receive a heartbeat +// within a 2-minute TTL; we heartbeat well inside that window. +const defaultHeartbeatInterval = 30 * time.Second + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// createVersionRequest is the CreateVersion request body. Hand-written because the +// generated struct has no previous_version_id, which the service needs as its +// concurrency check - without it every deploy after the first is rejected. +type createVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + TargetName string `json:"target_name,omitempty"` + // DisplayName names the deployment in the UI. The service keeps it on the + // deployment's node, so a version that omits it leaves the deployment unnamed. + DisplayName string `json:"display_name,omitempty"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` + // DeploymentMode is the bundle target's mode, unset when the target sets none. + DeploymentMode bundledeployments.DeploymentMode `json:"deployment_mode,omitempty"` + // GitInfo and WorkspaceInfo record where the deployed source came from and + // where it landed. The service denormalizes both onto the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + WorkspaceInfo *bundledeployments.WorkspaceInfo `json:"workspace_info,omitempty"` +} + +// versionCreator creates a version under a deployment. It exists because the +// generated client cannot express the request body (see createVersionRequest). +type versionCreator interface { + CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) +} + +// apiVersionCreator creates versions through the workspace API client. +type apiVersionCreator struct { + client *client.DatabricksClient +} + +// NewAPIVersionCreator returns a versionCreator that posts to the DMS API. +func NewAPIVersionCreator(c *client.DatabricksClient) versionCreator { + return &apiVersionCreator{client: c} +} + +func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { + var version bundledeployments.Version + path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions", deploymentID) + err := a.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + +// Recorder records a single deploy/destroy as a version with DMS. The server +// assigns the deployment ID on the first deploy and later deploys reuse it; a +// destroy deletes the record, so the next deploy starts over (see +// ResolveDeploymentID). +type Recorder struct { + svc bundledeployments.BundleDeploymentsInterface + versions versionCreator + deploymentID string + statePath string + versionType VersionType + metadata Metadata + + // populated by CreateVersion + versionNum int64 + stopHeartbeat context.CancelFunc + + // completed makes CompleteVersion idempotent, so a caller that completes the + // version early can still defer it unconditionally. + completed bool +} + +// RecorderOptions are the dependencies and deployment identity a Recorder needs. +type RecorderOptions struct { + // Service handles every DMS call except CreateVersion. + Service bundledeployments.BundleDeploymentsInterface + // Versions handles CreateVersion; see versionCreator. + Versions versionCreator + // DeploymentID is resolved from the deployment's workspace node, empty until the + // first recorded deploy (CreateVersion assigns one then). + DeploymentID string + // StatePath is the bundle's remote state directory, under which DMS registers + // the deployment node. + StatePath string + VersionType VersionType + // Metadata is what the version records about the deploy; see Metadata. + Metadata Metadata +} + +// Metadata is what a version records about the bundle, its source and where it +// landed. The service copies these onto the deployment, so they describe it as of +// its most recent version. +type Metadata struct { + // DisplayName is the bundle's name, which the deployment is listed under. + DisplayName string + // TargetName is the bundle target that was deployed. + TargetName string + // Mode is the bundle target's mode, empty when the target sets none. + Mode bundledeployments.DeploymentMode + Git *bundledeployments.GitInfo + Workspace *bundledeployments.WorkspaceInfo +} + +// NewRecorder returns a Recorder for the deployment described by opts. +func NewRecorder(opts RecorderOptions) *Recorder { + return &Recorder{ + svc: opts.Service, + versions: opts.Versions, + deploymentID: opts.DeploymentID, + statePath: opts.StatePath, + versionType: opts.VersionType, + metadata: opts.Metadata, + } +} + +// DeploymentID returns the DMS deployment ID this recorder is bound to. It is +// empty until CreateVersion has created the deployment record (on a first +// deploy) and non-empty afterwards, so callers can parent operations under it. +func (r *Recorder) DeploymentID() string { + if r == nil { + return "" + } + return r.deploymentID +} + +// Version returns the version number claimed by CreateVersion. It is zero until +// CreateVersion has run; callers use it to parent operations under the version. +func (r *Recorder) Version() int64 { + if r == nil { + return 0 + } + return r.versionNum +} + +// CreateVersion registers a new version with DMS, claiming it for the duration +// of the deployment. A nil Recorder is a no-op, so callers can leave it nil +// when recording is disabled. +func (r *Recorder) CreateVersion(ctx context.Context) error { + if r == nil { + return nil + } + + versionID, err := r.createDeploymentVersion(ctx) + if err != nil { + return err + } + + versionNum, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + } + r.versionNum = versionNum + r.stopHeartbeat = startHeartbeat(ctx, r.svc, r.deploymentID, versionID) + return nil +} + +// CompleteVersion finalizes the version created by CreateVersion. It is a no-op +// when CreateVersion never ran, which is what lets callers defer it and still not +// complete a version a cancelled deploy never created. +func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { + if r == nil || r.versionNum == 0 || r.completed { + return nil + } + r.completed = true + + r.stopHeartbeat() + + versionIDStr := strconv.FormatInt(r.versionNum, 10) + versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + + _, err := r.svc.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName, + CompletionReason: reason, + }) + if err != nil { + return err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%s reason=%s", r.deploymentID, versionIDStr, reason) + + // For destroy operations, delete the deployment record after the version + // completes successfully. + if success && r.versionType == VersionTypeDestroy { + err = r.svc.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if err != nil { + return fmt.Errorf("failed to delete deployment: %w", err) + } + } + + return nil +} + +// createDeploymentVersion ensures the deployment record exists, then creates a new +// version under it: with no ID it creates the deployment, otherwise it reads the +// existing one for the next version number. +func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + // The version this one supersedes, sent as the concurrency check. Empty for a + // deployment's first version. + var previousVersionID string + if r.deploymentID != "" { + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by + // design the service has a deployment for every such node, so a not-found + // here means that invariant is broken rather than anything the user did. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + switch { + case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): + return "", fmt.Errorf("internal error: no deployment found for the file with object id %s: %w", r.deploymentID, getErr) + case getErr != nil: + return "", fmt.Errorf("failed to get deployment: %w", getErr) + case dep.LastVersionId == "": + // The record exists but carries no version: a deploy whose first version was + // rejected still leaves the record behind. Retry at version 1. + versionID = "1" + default: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + previousVersionID = dep.LastVersionId + } + } else { + // First deploy: create the deployment so the server assigns an ID. + // initial_parent_path is required - the node the service creates under it is + // what ResolveDeploymentID reads back later. + dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ + Deployment: bundledeployments.Deployment{ + InitialParentPath: r.statePath, + TargetName: r.metadata.TargetName, + }, + }) + if createErr != nil { + return "", fmt.Errorf("failed to create deployment: %w", createErr) + } + id, idErr := deploymentIDFromName(dep.Name) + if idErr != nil { + return "", idErr + } + r.deploymentID = id + versionID = "1" + } + + // The server rejects the call unless versionID is numerically greater than + // last_version_id and previous_version_id matches it, so a deploy racing + // another is rejected rather than overwriting it. + version, versionErr := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.metadata.TargetName, + DisplayName: r.metadata.DisplayName, + PreviousVersionId: previousVersionID, + DeploymentMode: r.metadata.Mode, + GitInfo: r.metadata.Git, + WorkspaceInfo: r.metadata.Workspace, + }) + if versionErr != nil { + return "", fmt.Errorf("failed to create deployment version: %w", versionErr) + } + + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) + return versionID, nil +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, "deployments/") + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// startHeartbeat starts a background goroutine that sends heartbeats to keep +// the deployment version's lease alive. Returns a cancel function to stop it. +func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeploymentsInterface, deploymentID, versionID string) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + versionName := fmt.Sprintf("deployments/%s/versions/%s", deploymentID, versionID) + + go func() { + ticker := time.NewTicker(defaultHeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := svc.Heartbeat(ctx, bundledeployments.HeartbeatRequest{Name: versionName}) + if err != nil { + // A 409 ABORTED is expected if the version was completed + // between the ticker firing and the heartbeat. + if isAbortedErr(err) { + log.Debugf(ctx, "Heartbeat stopped: version already completed") + return + } + log.Warnf(ctx, "Failed to send deployment heartbeat: %v", err) + } else { + log.Debugf(ctx, "Deployment heartbeat sent: deployment=%s version=%s", deploymentID, versionID) + } + } + } + }() + + return cancel +} + +// isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. +func isAbortedErr(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusConflict && apiErr.ErrorCode == "ABORTED" +} diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go new file mode 100644 index 00000000000..51ce63e5971 --- /dev/null +++ b/libs/dms/recorder_test.go @@ -0,0 +1,250 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testStatePath is the bundle state directory the recorder registers the +// deployment node under; several tests assert it round-trips to the service. +const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" + +// testDisplayName is the bundle name the recorder sends as the version's display +// name; the service copies it onto the deployment's workspace node. +const testDisplayName = "proj" + +// fakeDMS records the calls the recorder makes and lets a test script the +// server-side responses. It embeds the SDK interface so it satisfies it while +// only overriding the methods the recorder uses. +type fakeDMS struct { + bundledeployments.BundleDeploymentsInterface + + // scripted behavior + getDeployment func(id string) (*bundledeployments.Deployment, error) + + // assigned deployment ID for CreateDeployment (server-generated flow) + assignedID string + + // captured requests + created []bundledeployments.CreateDeploymentRequest + versions []fakeVersionRequest + completed []bundledeployments.CompleteVersionRequest + deleted []string +} + +// fakeVersionRequest is a CreateVersion call captured by fakeVersions. +type fakeVersionRequest struct { + deploymentID string + versionID string + body createVersionRequest +} + +// fakeVersions captures CreateVersion calls. It is separate from fakeDMS because +// the CLI does not create versions through the generated client (see +// createVersionRequest), so the two use different signatures. +type fakeVersions struct { + requests *[]fakeVersionRequest +} + +func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { + *f.requests = append(*f.requests, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) + return &bundledeployments.Version{VersionId: versionID}, nil +} + +func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { + f.created = append(f.created, req) + // The server always assigns the ID; it is the ID of the workspace node it + // creates under initial_parent_path. + return &bundledeployments.Deployment{Name: "deployments/" + f.assignedID}, nil +} + +func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { + id := req.Name[len("deployments/"):] + return f.getDeployment(id) +} + +func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { + f.completed = append(f.completed, req) + return &bundledeployments.Version{}, nil +} + +func (f *fakeDMS) DeleteDeployment(ctx context.Context, req bundledeployments.DeleteDeploymentRequest) error { + f.deleted = append(f.deleted, req.Name) + return nil +} + +func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.HeartbeatRequest) (*bundledeployments.HeartbeatResponse, error) { + return &bundledeployments.HeartbeatResponse{}, nil +} + +func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { + f := &fakeDMS{assignedID: "server-generated-id"} + // A first deploy resolves no deployment ID from the workspace. + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + require.NoError(t, r.CreateVersion(t.Context())) + + // The server assigned the ID, and the recorder exposes it for the rest of the + // deploy (it parents the operations recorded under this version). + require.Len(t, f.created, 1) + assert.Equal(t, "server-generated-id", r.DeploymentID()) + // initial_parent_path is required: the service creates the deployment node + // under it, and that node is what ResolveDeploymentID looks up later. + assert.Equal(t, testStatePath, f.created[0].Deployment.InitialParentPath) + + // The first version is 1, parented under the assigned deployment. + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].versionID) + assert.Equal(t, "server-generated-id", f.versions[0].deploymentID) + assert.Equal(t, int64(1), r.Version()) + + // The service copies display_name onto the deployment's workspace node, which is + // where GetDeployment reads it from; a version that omits it leaves the deployment + // unnamed in the UI. + assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) + // A first version supersedes nothing, so previous_version_id is unset. + assert.Empty(t, f.versions[0].body.PreviousVersionId) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.Len(t, f.completed, 1) + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) + assert.Empty(t, f.deleted) +} + +func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + // A subsequent deploy passes the stored deployment ID. + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + require.NoError(t, r.CreateVersion(t.Context())) + + // No new deployment is created; the version increments to last_version_id + 1. + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "5", f.versions[0].versionID) + assert.Equal(t, "stored-id", r.DeploymentID()) + // The version it supersedes is the concurrency check; without it the service + // rejects every deploy after the first. + assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) +} + +func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + +func TestRecorderMissingDeploymentIsInternalError(t *testing.T) { + // The service has a deployment for every BUNDLE_DEPLOYMENT node, so a not-found + // for a node get-status just returned is a broken invariant, not a state the + // deploy can recover from. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") + assert.Empty(t, f.created) + assert.Empty(t, f.versions) +} + +func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + // A successful destroy deletes the deployment record. + require.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + +func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), false)) + + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) + // A failed destroy leaves the deployment in place. + assert.Empty(t, f.deleted) +} + +func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { + // Destroy completes the version before deleting the remote files, because that + // deletes the deployment's node, and still defers CompleteVersion. The second + // call must not reach the server, which would fail with 404. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.NoError(t, r.CompleteVersion(t.Context(), true)) + + assert.Len(t, f.completed, 1) + // The destroy deletes the deployment record once, not once per call. + assert.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + +func TestNilRecorderIsNoOp(t *testing.T) { + var r *Recorder + assert.NoError(t, r.CreateVersion(t.Context())) + assert.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, r.DeploymentID()) + assert.Zero(t, r.Version()) +} + +func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { + f := &fakeDMS{} + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + // CompleteVersion before CreateVersion is a no-op (nothing was claimed). + require.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, f.completed) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go new file mode 100644 index 00000000000..9bbf5a84b56 --- /dev/null +++ b/libs/dms/resolve.go @@ -0,0 +1,40 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// ResolveDeploymentID returns the DMS deployment ID for the bundle whose state +// lives under statePath, or empty if it has never recorded a deployment. +// +// The CLI stores the ID nowhere: DMS registers the deployment as a workspace +// node and that node's ID *is* the deployment ID, so a get-status is the lookup. +// This keeps the workspace the single source of truth — a destroyed deployment +// reports absent instead of leaving a dangling ID in the local state file. +func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if err != nil { + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil + } + return "", fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + + if obj.ObjectId == 0 { + return "", fmt.Errorf("deployment at %s has no object ID", nodePath) + } + return strconv.FormatInt(obj.ObjectId, 10), nil +} diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go new file mode 100644 index 00000000000..6838f4dd15c --- /dev/null +++ b/libs/dms/resolve_test.go @@ -0,0 +1,69 @@ +package dms + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestClient returns a workspace client pointed at a server that serves a +// single get-status response. +func newTestClient(t *testing.T, statusCode int, body string) *databricks.WorkspaceClient { + t.Helper() + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Query().Get("path") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { + assert.Equal(t, nodePath, gotPath) + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: srv.URL, + Token: "token", + Credentials: config.PatCredentials{}, + }) + require.NoError(t, err) + return w +} + +const nodePath = "/Workspace/state/" + DeploymentNodeName + +func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { + w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) + + // The workspace node ID is the deployment ID, so no local state is consulted. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Equal(t, "123456789", id) +} + +func TestResolveDeploymentIDAbsentWhenNodeMissing(t *testing.T) { + w := newTestClient(t, http.StatusNotFound, `{"error_code":"RESOURCE_DOES_NOT_EXIST","message":"Path (/Workspace/state/`+DeploymentNodeName+`) doesn't exist."}`) + + // A bundle that never recorded a deployment, or whose deployment was + // destroyed (the service trashes the node), has no ID rather than an error. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Empty(t, id) +} + +func TestResolveDeploymentIDPropagatesOtherErrors(t *testing.T) { + w := newTestClient(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"nope"}`) + + // Anything other than a missing node is fatal: silently treating it as absent + // would create a second deployment for a bundle that already has one. + _, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.Error(t, err) + assert.ErrorContains(t, err, "looking up deployment at /Workspace/state/"+DeploymentNodeName) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go new file mode 100644 index 00000000000..574570ce194 --- /dev/null +++ b/libs/testserver/bundle.go @@ -0,0 +1,461 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "path" + "slices" + "strconv" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. +// State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. + +// dmsDeploymentNodeName is the name of the workspace node the service creates +// for every deployment. It must match DEPLOYMENT_NODE_NAME on the service side +// (DeploymentWhsClient); the literal is repeated here rather than shared with +// the CLI so a test would catch the CLI drifting from the service. +const dmsDeploymentNodeName = "resources.deployment.json" + +// dmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type dmsDeployment struct { + deployment bundledeployments.Deployment + versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + resources map[string]bundledeployments.Resource + // operations holds the recorded operations by resource name. The service keeps + // one per resource per version, so a resource written twice in a version updates + // its operation rather than adding another. + operations map[string]*bundledeployments.Operation + // lastSuccessfulVersionID is the highest version that completed + // successfully. The server advances last_successful_version_id only on + // success (unlike last_version_id), and the read path treats a non-empty + // value as "DMS owns the state". Tracked separately because the SDK + // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). + lastSuccessfulVersionID string +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } + + defer s.LockUnlock()() + + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: nodePath, + ObjectId: objectID, + }, + } + + // The record is created together with the node, so a get on it always resolves + // for a node that exists. It carries no version yet: last_version_id stays empty + // until the first CreateVersion, which is how a client that registers a + // deployment and then fails leaves a record with no versions. + deploymentID := strconv.FormatInt(objectID, 10) + s.dmsDeploymentNodes[deploymentID] = nodePath + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.dmsDeployments[deploymentID] = &dmsDeployment{ + deployment: dep, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + operations: map[string]*bundledeployments.Operation{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment the way the real server does: the typed +// fields plus last_successful_version_id, which the generated SDK struct does +// not carry yet (still stage:DEVELOPMENT) but the read path reads off the raw +// JSON. +// +// The extra field cannot be added by embedding Deployment in a wrapper struct: +// Deployment has its own MarshalJSON, which is promoted to the wrapper and +// silently drops any sibling field. +func deploymentBody(d *dmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.lastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.lastSuccessfulVersionID + } + return body, nil +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if nodePath, ok := s.dmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) + } + delete(s.dmsDeploymentNodes, deploymentID) + delete(s.dmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + // previous_version_id is absent from the generated struct, so read it separately. + var concurrency struct { + PreviousVersionId string `json:"previous_version_id"` + } + if err := json.Unmarshal(req.Body, &concurrency); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Mirror the server-side checks: version_id must be numerically greater than + // the most recent version (not exactly one more), and previous_version_id must + // name that version, which is what detects a concurrent deploy. + next, err := strconv.ParseInt(versionID, 10, 64) + if err != nil || next < 1 { + return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) + } + var last int64 + if d.deployment.LastVersionId != "" { + last, _ = strconv.ParseInt(d.deployment.LastVersionId, 10, 64) + } + if next <= last { + return dmsInvalidArgument("version_id " + versionID + " must be greater than the most recent version " + d.deployment.LastVersionId) + } + if concurrency.PreviousVersionId != d.deployment.LastVersionId { + return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) + } + + // bundle_root_path is relative to git_folder_path, so the service rejects a + // workspace_info that carries one without the other. + if ws := version.WorkspaceInfo; ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { + return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together") + } + + d.deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.versions[versionID] = &version + + // The service denormalizes the version's provenance onto the deployment, which + // is where the read APIs serve it from. display_name is excluded: the service + // keeps that on the deployment's workspace node instead. + d.deployment.TargetName = version.TargetName + d.deployment.DeploymentMode = version.DeploymentMode + d.deployment.GitInfo = version.GitInfo + d.deployment.WorkspaceInfo = version.WorkspaceInfo + + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.lastSuccessfulVersionID = versionID + } + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID string) Response { + resourceKey := req.URL.Query().Get("resource_key") + + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // A delete carries no state, so resource_id is the only thing identifying which + // resource it refers to; the service rejects a delete without one. A failed + // delete is exempt only for create-flavored actions, which may not have an ID. + if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") + } + + failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && op.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // The service names operations after the resource key, so it keeps one per + // resource per version: creating a second one for the same resource conflicts, + // and the caller has to use UpdateOperation instead. + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + if _, exists := d.operations[opName]; exists { + return Response{ + StatusCode: 409, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": "operation for " + resourceKey + " already exists in this version"}, + } + } + + op.Name = opName + op.ResourceKey = resourceKey + op.SequenceId = 1 + d.operations[opName] = &op + + // The service sends sequence_id as a JSON string (proto3 encodes 64-bit ints + // that way) while the SDK struct types it as an int64, so the response is built + // by hand to match the wire format the CLI actually parses. + body, err := operationBody(&op) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Reflect the operation onto the deployment-level resource set the way the + // backend does: a delete removes the resource, anything else upserts it. + // + // A failed operation is upserted too, matching the service, but it carries + // neither a resource_id nor state, so the read path treats the resource as not + // yet created rather than as existing state (verified against the service: a + // failed create is listed with an empty resource_id and no state). + switch { + case op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed: + delete(d.resources, resourceKey) + default: + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: op.ResourceId, + ResourceType: op.ResourceType, + LastActionType: op.ActionType, + LastVersionId: versionID, + State: op.State, + } + } + return Response{Body: body} +} + +// operationBody renders an operation the way the service does: sequence_id as a +// JSON string, which the SDK struct cannot express (it types the field int64). +func operationBody(op *bundledeployments.Operation) (map[string]any, error) { + raw, err := json.Marshal(op) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + body["sequence_id"] = strconv.FormatInt(op.SequenceId, 10) + return body, nil +} + +// UpdateOperation applies a later write for a resource already recorded in this +// version. sequence_id is the concurrency precondition and increments on success. +func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { + // sequence_id arrives as a string, which the SDK struct cannot hold (it types the + // field int64), so read the body twice: once for the typed fields and once for the + // precondition. + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var precondition struct { + SequenceId string `json:"sequence_id"` + } + if err := json.Unmarshal(req.Body, &precondition); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + existing, ok := d.operations[opName] + if !ok { + return dmsNotFound("operation " + opName) + } + if precondition.SequenceId != strconv.FormatInt(existing.SequenceId, 10) { + return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) + } + + failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && op.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // Only the mutable fields change; action_type and resource_key stay as created. + existing.State = op.State + existing.ErrorMessage = op.ErrorMessage + existing.ResourceId = op.ResourceId + existing.Status = op.Status + existing.SequenceId++ + + body, err := operationBody(existing) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Mirror onto the resource set the same way CreateOperation does, so the read + // path reflects the newest write. + if existing.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } + } + + return Response{Body: body} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.resources)) + for key := range d.resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + // Content-Type is required for the SDK to parse the body into a typed error, + // which is what callers match against apierr.ErrResourceDoesNotExist. + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsInvalidArgument(message string) Response { + return Response{ + StatusCode: 400, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } +} + +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index e2577730395..d951b61b284 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -242,6 +242,16 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + dmsDeployments map[string]*dmsDeployment + + // dmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment + // registered for it. A deployment appears here before it has a record in + // dmsDeployments: the record is created by its first version, so the node is + // what makes an ID valid in between. + dmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -410,6 +420,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + dmsDeployments: map[string]*dmsDeployment{}, + dmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 2ef3ad0c7a2..1c9f7b50bb8 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -283,6 +283,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { + return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { + return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 4839c1e4273..cd4ce5026a7 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "strings" ) @@ -70,6 +71,28 @@ func ResourceTypes() []string { return names } +// DeploymentURL returns the workspace URL for a bundle deployment recorded with +// the deployment metadata service, of the form +// +// /deployments/?version= +// +// The version pins the page to the deploy that produced it. It is separate from +// ResourceURL because a deployment is not a bundle resource type: it has no entry +// in resourceURLPatterns and takes a query parameter none of those do. +func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { + if deploymentID == "" { + return "" + } + + baseURL.Path = "deployments/" + deploymentID + if version > 0 { + values := baseURL.Query() + values.Set("version", strconv.FormatInt(version, 10)) + baseURL.RawQuery = values.Encode() + } + return baseURL.String() +} + // JobRunPath returns the modern workspace path for a job run, of the form // // jobs//runs/ diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index a398167e027..b92df037c84 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -141,6 +141,55 @@ func TestResourceURL(t *testing.T) { } } +func TestDeploymentURL(t *testing.T) { + tests := []struct { + name string + deploymentID string + version int64 + base url.URL + expected string + }{ + { + name: "id and version", + deploymentID: "996980114684409", + version: 2, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409?version=2", + }, + { + // The version is only known once CreateVersion has run, so link to the + // deployment itself rather than emitting version=0. + name: "zero version omits the query", + deploymentID: "996980114684409", + version: 0, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409", + }, + { + // A base URL carrying ?w= keeps it, so a vanity or legacy + // host still addresses the right workspace. + name: "preserves an existing query", + deploymentID: "42", + version: 7, + base: url.URL{Scheme: "https", Host: "host.com", RawQuery: "w=123"}, + expected: "https://host.com/deployments/42?version=7&w=123", + }, + { + name: "empty id returns empty", + deploymentID: "", + version: 1, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, DeploymentURL(tt.base, tt.deploymentID, tt.version)) + }) + } +} + func TestHasWorkspaceIDInHostname(t *testing.T) { tests := []struct { name string