From 0cc9bcf5a17bdc8714c4a2d54ab296c9ab01d35a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 12:58:55 +0000 Subject: [PATCH 01/28] Fix model serving telemetry_config never converging and silently dropping updates telemetry_config is only carried by CreateServingEndpoint. The serving endpoints GET API never echoes it back, and no update API accepts it, so classify it as input_only for remote changes and no_update_api for local ones. --- ...odel-serving-telemetry-config-converges.md | 1 + ...serving_endpoint_telemetry_config.yml.tmpl | 10 ++ .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + .../drift/telemetry_config/databricks.yml | 25 +++++ .../drift/telemetry_config/out.test.toml | 3 + .../drift/telemetry_config/output.txt | 95 +++++++++++++++++++ .../drift/telemetry_config/script | 18 ++++ .../drift/telemetry_config/test.toml | 4 + bundle/direct/dresources/resources.yml | 11 +++ 14 files changed, 173 insertions(+) create mode 100644 .nextchanges/bundles/model-serving-telemetry-config-converges.md create mode 100644 acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md new file mode 100644 index 00000000000..43c60d9cd61 --- /dev/null +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -0,0 +1 @@ +Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the field is only accepted on create and is never returned by the serving endpoints GET API, so every subsequent `bundle plan` reported the endpoint as a perpetual update. Changing `telemetry_config` on a deployed endpoint no longer plans an update that cannot be applied, since the API has no method to update it. diff --git a/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl new file mode 100644 index 00000000000..21cc542afe9 --- /dev/null +++ b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + model_serving_endpoints: + foo: + name: test-endpoint-$UNIQUE_NAME + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index ca367650f71..8399dc90ce2 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -23,6 +23,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 9b8f7ea4435..d9e7e10caca 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 9b8f7ea4435..d9e7e10caca 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 3fdd9871ac2..cb8cf38cc64 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -21,6 +21,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 9b8f7ea4435..d9e7e10caca 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 32a0808dbac..c56b4f264ff 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -48,6 +48,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml new file mode 100644 index 00000000000..703699db5ed --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + endpoint1: + name: test-endpoint + config: + served_entities: + - name: prod + external_model: + name: gpt-4o-mini + provider: openai + task: llm/v1/chat + openai_config: + openai_api_key_plaintext: sk-test-plaintext-key + traffic_config: + routes: + - served_model_name: prod + traffic_percentage: 100 + # Accepted by the create API, never returned on GET, and there is no + # update API for it. + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt new file mode 100644 index 00000000000..28bd2b2ef12 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -0,0 +1,95 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op even though telemetry_config does not round-trip +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== telemetry_config is skipped rather than planned as an update the API cannot make +>>> [CLI] bundle plan --output json +{ + "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "action": "skip", + "reason": "input_only", + "old": "sk-test-plaintext-key", + "new": "sk-test-plaintext-key", + "remote": "" + }, + "description": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "route_optimized": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "telemetry_config": { + "action": "skip", + "reason": "input_only", + "old": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + }, + "new": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} + +=== Changing telemetry_config does not plan an update: it can only be set at create time +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy is a no-op (no config/ai-gateway/tags update calls) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //serving-endpoints +{ + "method": "POST", + "path": "/api/2.0/serving-endpoints", + "body": { + "config": { + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key_plaintext": "sk-test-plaintext-key" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ], + "traffic_config": { + "routes": [ + { + "served_model_name": "prod", + "traffic_percentage": 100 + } + ] + } + }, + "name": "test-endpoint", + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script new file mode 100644 index 00000000000..59a8e2ffbd8 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op even though telemetry_config does not round-trip" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "telemetry_config is skipped rather than planned as an update the API cannot make" +trace $CLI bundle plan --output json | jq '.plan[].changes' + +title "Changing telemetry_config does not plan an update: it can only be set at create time" +update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" +trace $CLI bundle deploy +trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml new file mode 100644 index 00000000000..a43d26a50dd --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -0,0 +1,4 @@ +RecordRequests = true + +# Scope this regression to the direct engine where the classification lives. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 4099c536281..ffed535a074 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,6 +285,11 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only + # Accepted by the create API but GET never echoes it back, so the remote + # always reports empty and every deploy would otherwise replan the same + # update forever. + - field: telemetry_config + reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext reason: input_only @@ -315,6 +320,12 @@ resources: reason: no_update_api - field: rate_limits reason: not_implemented + # telemetry_config can only be set at create time: there is no update API + # for it on ServingEndpointsAPI, and no update request body in the OpenAPI + # spec carries the field. Without this rule the plan promises an update + # that DoUpdate has no call to make, so the change is silently dropped. + - field: telemetry_config + reason: no_update_api backend_defaults: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 # common.CustomizeSchemaPath(m, "config", "served_entities", "name").SetComputed() From 2ff048275609cc4879bfa45c1ee50d87517cfa77 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 13:09:43 +0000 Subject: [PATCH 02/28] Shorten comments --- .../drift/telemetry_config/databricks.yml | 2 -- .../drift/telemetry_config/output.txt | 4 ++-- .../drift/telemetry_config/script | 4 ++-- bundle/direct/dresources/resources.yml | 9 ++------- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml index 703699db5ed..5b40d000814 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -18,8 +18,6 @@ resources: routes: - served_model_name: prod traffic_percentage: 100 - # Accepted by the create API, never returned on GET, and there is no - # update API for it. telemetry_config: inference_table_config: sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index 28bd2b2ef12..6107891c48e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -10,7 +10,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== telemetry_config is skipped rather than planned as an update the API cannot make +=== telemetry_config is skipped (confirms the matched rules) >>> [CLI] bundle plan --output json { "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { @@ -46,7 +46,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } -=== Changing telemetry_config does not plan an update: it can only be set at create time +=== Changing telemetry_config plans nothing: it is create-only >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index 59a8e2ffbd8..e4f0ed5bbf8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -6,10 +6,10 @@ trace $CLI bundle deploy title "Plan is a no-op even though telemetry_config does not round-trip" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "telemetry_config is skipped rather than planned as an update the API cannot make" +title "telemetry_config is skipped (confirms the matched rules)" trace $CLI bundle plan --output json | jq '.plan[].changes' -title "Changing telemetry_config does not plan an update: it can only be set at create time" +title "Changing telemetry_config plans nothing: it is create-only" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index ffed535a074..b8f58c67e1d 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,9 +285,7 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # Accepted by the create API but GET never echoes it back, so the remote - # always reports empty and every deploy would otherwise replan the same - # update forever. + # Remote always reads back empty, so without this it never converges. - field: telemetry_config reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -320,10 +318,7 @@ resources: reason: no_update_api - field: rate_limits reason: not_implemented - # telemetry_config can only be set at create time: there is no update API - # for it on ServingEndpointsAPI, and no update request body in the OpenAPI - # spec carries the field. Without this rule the plan promises an update - # that DoUpdate has no call to make, so the change is silently dropped. + # Only exists on CreateServingEndpoint; no update API accepts it. - field: telemetry_config reason: no_update_api backend_defaults: From 64f5d2eeb00530c6fd868ea984b4745557e35bf7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 3 Aug 2026 12:52:27 +0000 Subject: [PATCH 03/28] Apply telemetry_config instead of suppressing it The previous approach classified telemetry_config as input_only/no_update_api on the premise that GET never returns it and no API updates it. Neither holds: ServingEndpointDetailed carries the field, and ServingEndpoints.PatchTelemetryConfig updates it. Suppressing the whole subtree converged only by making genuine sampling_fraction edits invisible and silently dropping them. The actual remote-only difference is telemetry_profile_id, which the backend assigns and echoes back, so suppress just that as a backend default and wire the PATCH API into DoUpdate. Edits now converge and are applied. The testserver fake dropped telemetry_config on create, which is why the earlier suppression looked correct locally; it now echoes the field and populates the fields the backend assigns. --- ...odel-serving-telemetry-config-converges.md | 2 +- .../bundle/invariant/migrate/out.test.toml | 1 - acceptance/bundle/invariant/migrate/test.toml | 7 +++ .../drift/telemetry_config/output.txt | 51 +++++++++++------- .../drift/telemetry_config/script | 13 ++--- .../dresources/model_serving_endpoint.go | 22 ++++++++ bundle/direct/dresources/resources.yml | 11 ++-- libs/testserver/handlers.go | 4 ++ libs/testserver/serving_endpoints.go | 52 +++++++++++++++++++ 9 files changed, 130 insertions(+), 33 deletions(-) diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index 43c60d9cd61..e726224fce6 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the field is only accepted on create and is never returned by the serving endpoints GET API, so every subsequent `bundle plan` reported the endpoint as a perpetual update. Changing `telemetry_config` on a deployed endpoint no longer plans an update that cannot be applied, since the API has no method to update it. +Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the backend provisions a telemetry profile for the endpoint and returns its `telemetry_profile_id` on GET, which was reported as drift, so every `bundle plan` after a deploy showed the endpoint as a perpetual update. Changes to `telemetry_config` are now applied through the serving endpoints telemetry configuration API instead of being dropped. diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index cb8cf38cc64..3fdd9871ac2 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -21,7 +21,6 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index bb2337b32aa..f64cea6e586 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -30,6 +30,13 @@ EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] # SQL warehouses currently failing with migration with permanent drift. TODO: fix this. EnvMatrixExclude.no_sql_warehouse = ["INPUT_CONFIG=sql_warehouse.yml.tmpl"] +# The terraform provider types telemetry_config.inference_table_config.sampling_fraction +# as an integer (see ResourceModelServingTelemetryConfigInferenceTableConfig), but the API +# takes a fraction in [0.0, 1.0]. The terraform deploy that seeds the migration truncates +# 0.5 to 0 and omits it, so the migrated plan reports drift the direct engine cannot +# reconcile. Covered by no_drift on direct. +EnvMatrixExclude.no_telemetry_config = ["INPUT_CONFIG=model_serving_endpoint_telemetry_config.yml.tmpl"] + # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (deploy + migrate + plan at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index 6107891c48e..d97deeab6fa 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -6,11 +6,11 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Plan is a no-op even though telemetry_config does not round-trip +=== Plan converges: the profile ID and inference table name the backend assigns are not drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== telemetry_config is skipped (confirms the matched rules) +=== Editing sampling_fraction is reported as a real change >>> [CLI] bundle plan --output json { "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { @@ -30,33 +30,35 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "reason": "empty", "remote": false }, - "telemetry_config": { + "telemetry_config.inference_table_config.name": { "action": "skip", - "reason": "input_only", - "old": { - "inference_table_config": { - "sampling_fraction": 0.5 - } - }, - "new": { - "inference_table_config": { - "sampling_fraction": 0.5 - } - } + "reason": "spec:output_only", + "remote": "main.default.test-endpoint_payload" + }, + "telemetry_config.inference_table_config.sampling_fraction": { + "action": "update", + "old": 0.5, + "new": 0.9, + "remote": 0.5 + }, + "telemetry_config.telemetry_profile_id": { + "action": "skip", + "reason": "backend_default", + "remote": "[UUID]" } } -=== Changing telemetry_config plans nothing: it is create-only ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Redeploy is a no-op (no config/ai-gateway/tags update calls) +=== Deploy applies the edit through the telemetry-config API >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... Deployment complete! +=== Plan converges again once the edit is applied +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> print_requests.py //serving-endpoints { "method": "POST", @@ -93,3 +95,14 @@ Deployment complete! } } } +{ + "method": "PATCH", + "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", + "body": { + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.9 + } + } + } +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index e4f0ed5bbf8..0117c06eeb2 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -3,16 +3,17 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Plan is a no-op even though telemetry_config does not round-trip" +title "Plan converges: the profile ID and inference table name the backend assigns are not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "telemetry_config is skipped (confirms the matched rules)" +title "Editing sampling_fraction is reported as a real change" +update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle plan --output json | jq '.plan[].changes' -title "Changing telemetry_config plans nothing: it is create-only" -update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" +title "Deploy applies the edit through the telemetry-config API" +trace $CLI bundle deploy + +title "Plan converges again once the edit is applied" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" -trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index e8a1917c2f2..92b94e77ff7 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -18,6 +18,7 @@ var ( pathAiGateway = structpath.MustParsePath("ai_gateway") pathConfig = structpath.MustParsePath("config") pathEmailNotifications = structpath.MustParsePath("email_notifications") + pathTelemetryConfig = structpath.MustParsePath("telemetry_config") ) type ResourceModelServingEndpoint struct { @@ -246,6 +247,20 @@ func (r *ResourceModelServingEndpoint) updateNotifications(ctx context.Context, return nil } +// updateTelemetryConfig applies telemetry_config through its dedicated PATCH API. +// A nil config removes the telemetry configuration from the endpoint. +func (r *ResourceModelServingEndpoint) updateTelemetryConfig(ctx context.Context, id string, telemetryConfig *serving.TelemetryConfig) error { + req := serving.PatchTelemetryConfigRequest{ + Name: id, + TelemetryConfig: telemetryConfig, + } + _, err := r.client.ServingEndpoints.PatchTelemetryConfig(ctx, req) + if err != nil { + return fmt.Errorf("failed to update telemetry config: %w", err) + } + return nil +} + func diffTags(currentTags, desiredTags []serving.EndpointTag) (addTags []serving.EndpointTag, deleteTags []string) { addTags = make([]serving.EndpointTag, 0) @@ -343,6 +358,13 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, } } + if entry.Changes.HasChange(pathTelemetryConfig) { + err = r.updateTelemetryConfig(ctx, id, config.TelemetryConfig) + if err != nil { + return nil, err + } + } + return nil, nil } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b8f58c67e1d..7f3f82bb4ed 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,9 +285,6 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # Remote always reads back empty, so without this it never converges. - - field: telemetry_config - reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext reason: input_only @@ -318,10 +315,12 @@ resources: reason: no_update_api - field: rate_limits reason: not_implemented - # Only exists on CreateServingEndpoint; no update API accepts it. - - field: telemetry_config - reason: no_update_api backend_defaults: + # The backend provisions a telemetry profile for the endpoint and echoes its + # ID back on GET. Users may also pin an existing profile, so suppress only the + # backend-assigned case and keep drift detection when the ID is set locally. + - field: telemetry_config.telemetry_profile_id + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 # common.CustomizeSchemaPath(m, "config", "served_entities", "name").SetComputed() - field: config.served_entities[*].name diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 2ef3ad0c7a2..db4e0697791 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -984,6 +984,10 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ServingEndpointPatchTags(req, req.Vars["name"]) }) + server.Handle("PATCH", "/api/2.0/serving-endpoints/{name}/telemetry-config", func(req Request) any { + return req.Workspace.ServingEndpointPatchTelemetryConfig(req, req.Vars["name"]) + }) + // Vector Search Endpoints: server.Handle("POST", "/api/2.0/vector-search/endpoints", func(req Request) any { diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 41a6b9766b5..b91c0a58dcc 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,6 +35,27 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } +// applyTelemetryConfig mirrors the backend: it provisions a telemetry profile for +// the endpoint and echoes back the assigned profile ID and the full name of the +// inference table it created, neither of which the caller supplies. +func applyTelemetryConfig(endpointName string, config *serving.TelemetryConfig) *serving.TelemetryConfig { + if config == nil { + return nil + } + + applied := *config + if applied.TelemetryProfileId == "" { + applied.TelemetryProfileId = nextUUID() + } + if applied.InferenceTableConfig != nil { + inferenceTable := *applied.InferenceTableConfig + inferenceTable.Name = "main.default." + endpointName + "_payload" + applied.InferenceTableConfig = &inferenceTable + } + + return &applied +} + // clearExternalModelSecrets mirrors the backend, which persists the *_plaintext // API keys as secrets and never returns them on GET. func clearExternalModelSecrets(em *serving.ExternalModel) *serving.ExternalModel { @@ -225,6 +246,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { PermissionLevel: serving.ServingEndpointDetailedPermissionLevelCanManage, RouteOptimized: createReq.RouteOptimized, Tags: createReq.Tags, + TelemetryConfig: applyTelemetryConfig(createReq.Name, createReq.TelemetryConfig), State: &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, Ready: serving.EndpointStateReadyNotReady, @@ -369,6 +391,36 @@ func (s *FakeWorkspace) ServingEndpointUpdateNotifications(req Request, name str } } +func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name string) Response { + defer s.LockUnlock()() + + var patchReq serving.PatchTelemetryConfigRequest + err := json.Unmarshal(req.Body, &patchReq) + if err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: 400, + } + } + + endpoint, exists := s.ServingEndpoints[name] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": fmt.Sprintf("Serving endpoint with name %s not found", name)}, + } + } + + // An omitted telemetry_config removes the configuration from the endpoint. + endpoint.TelemetryConfig = applyTelemetryConfig(name, patchReq.TelemetryConfig) + endpoint.LastUpdatedTimestamp = nowMilli() + s.ServingEndpoints[name] = endpoint + + return Response{ + Body: endpoint, + } +} + func (s *FakeWorkspace) ServingEndpointPatchTags(req Request, name string) Response { defer s.LockUnlock()() From f4eda68a6a474f6f449fccf819f68d780c7c1d31 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 3 Aug 2026 14:09:45 +0000 Subject: [PATCH 04/28] Tighten comments and test step titles --- acceptance/bundle/invariant/migrate/test.toml | 8 +++----- .../drift/telemetry_config/output.txt | 8 ++++---- .../model_serving_endpoints/drift/telemetry_config/script | 8 ++++---- .../drift/telemetry_config/test.toml | 2 +- bundle/direct/dresources/model_serving_endpoint.go | 3 +-- bundle/direct/dresources/resources.yml | 5 ++--- libs/testserver/serving_endpoints.go | 5 ++--- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index f64cea6e586..d3db6741390 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -30,11 +30,9 @@ EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] # SQL warehouses currently failing with migration with permanent drift. TODO: fix this. EnvMatrixExclude.no_sql_warehouse = ["INPUT_CONFIG=sql_warehouse.yml.tmpl"] -# The terraform provider types telemetry_config.inference_table_config.sampling_fraction -# as an integer (see ResourceModelServingTelemetryConfigInferenceTableConfig), but the API -# takes a fraction in [0.0, 1.0]. The terraform deploy that seeds the migration truncates -# 0.5 to 0 and omits it, so the migrated plan reports drift the direct engine cannot -# reconcile. Covered by no_drift on direct. +# The terraform provider types sampling_fraction as an integer while the API takes a +# fraction in [0.0, 1.0], so the terraform deploy that seeds the migration truncates +# 0.5 to 0 and drops it. Covered by no_drift on direct. EnvMatrixExclude.no_telemetry_config = ["INPUT_CONFIG=model_serving_endpoint_telemetry_config.yml.tmpl"] # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index d97deeab6fa..b5ae4ba51af 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -6,11 +6,11 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Plan converges: the profile ID and inference table name the backend assigns are not drift +=== Plan converges: backend-assigned fields are not drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== Editing sampling_fraction is reported as a real change +=== Editing sampling_fraction is a real change >>> [CLI] bundle plan --output json { "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { @@ -48,14 +48,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } -=== Deploy applies the edit through the telemetry-config API +=== Deploy applies it via the telemetry-config API >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... Deployment complete! -=== Plan converges again once the edit is applied +=== Plan converges again >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index 0117c06eeb2..3d46ccf88dc 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -3,17 +3,17 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Plan converges: the profile ID and inference table name the backend assigns are not drift" +title "Plan converges: backend-assigned fields are not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "Editing sampling_fraction is reported as a real change" +title "Editing sampling_fraction is a real change" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle plan --output json | jq '.plan[].changes' -title "Deploy applies the edit through the telemetry-config API" +title "Deploy applies it via the telemetry-config API" trace $CLI bundle deploy -title "Plan converges again once the edit is applied" +title "Plan converges again" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml index a43d26a50dd..d9fe064aa30 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -1,4 +1,4 @@ RecordRequests = true -# Scope this regression to the direct engine where the classification lives. +# Covers the direct engine's telemetry_config classification and update path. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 92b94e77ff7..6b9c091f691 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -247,8 +247,7 @@ func (r *ResourceModelServingEndpoint) updateNotifications(ctx context.Context, return nil } -// updateTelemetryConfig applies telemetry_config through its dedicated PATCH API. -// A nil config removes the telemetry configuration from the endpoint. +// updateTelemetryConfig applies telemetry_config; a nil config removes it. func (r *ResourceModelServingEndpoint) updateTelemetryConfig(ctx context.Context, id string, telemetryConfig *serving.TelemetryConfig) error { req := serving.PatchTelemetryConfigRequest{ Name: id, diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 7f3f82bb4ed..b6d55cfaed1 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -316,9 +316,8 @@ resources: - field: rate_limits reason: not_implemented backend_defaults: - # The backend provisions a telemetry profile for the endpoint and echoes its - # ID back on GET. Users may also pin an existing profile, so suppress only the - # backend-assigned case and keep drift detection when the ID is set locally. + # The backend assigns a telemetry profile and echoes its ID back on GET. Users may + # also pin an existing profile, so drift still applies when it is set locally. - field: telemetry_config.telemetry_profile_id # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index b91c0a58dcc..23fea03c82d 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,9 +35,8 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend: it provisions a telemetry profile for -// the endpoint and echoes back the assigned profile ID and the full name of the -// inference table it created, neither of which the caller supplies. +// applyTelemetryConfig mirrors the backend, which assigns the profile ID and the +// inference table name that the caller never supplies. func applyTelemetryConfig(endpointName string, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil From ebdd8d1d75bc3d42b2f7b9e6a2a9532f58f8057f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 11:02:58 +0000 Subject: [PATCH 05/28] Drop telemetry_config from the invariant configs The backend only supports telemetry configuration on endpoints with custom served models. The invariant suite deploys a config-less endpoint, so the create silently dropped telemetry_config and every subsequent plan reported an update that the telemetry-config API then rejected with "Telemetry configuration is not supported for endpoint type 'NO_CONFIG'". Serving a custom model needs a registered model version and real serving compute, which is too slow for this suite, and the drift/telemetry_config test already asserts the same convergence locally. The terraform truncation note moves to that test, which is where the engine matrix is now restricted. --- .../model_serving_endpoint_telemetry_config.yml.tmpl | 10 ---------- acceptance/bundle/invariant/continue_293/out.test.toml | 1 - .../bundle/invariant/delete_idempotent/out.test.toml | 1 - .../bundle/invariant/destroy_idempotent/out.test.toml | 1 - acceptance/bundle/invariant/migrate/test.toml | 5 ----- acceptance/bundle/invariant/no_drift/out.test.toml | 1 - acceptance/bundle/invariant/test.toml | 2 -- .../drift/telemetry_config/test.toml | 4 +++- 8 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl deleted file mode 100644 index 21cc542afe9..00000000000 --- a/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -bundle: - name: test-bundle-$UNIQUE_NAME - -resources: - model_serving_endpoints: - foo: - name: test-endpoint-$UNIQUE_NAME - telemetry_config: - inference_table_config: - sampling_fraction: 0.5 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8399dc90ce2..ca367650f71 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -23,7 +23,6 @@ EnvMatrix.INPUT_CONFIG = [ "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index d9e7e10caca..9b8f7ea4435 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -30,7 +30,6 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index d9e7e10caca..9b8f7ea4435 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -30,7 +30,6 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index d3db6741390..bb2337b32aa 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -30,11 +30,6 @@ EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] # SQL warehouses currently failing with migration with permanent drift. TODO: fix this. EnvMatrixExclude.no_sql_warehouse = ["INPUT_CONFIG=sql_warehouse.yml.tmpl"] -# The terraform provider types sampling_fraction as an integer while the API takes a -# fraction in [0.0, 1.0], so the terraform deploy that seeds the migration truncates -# 0.5 to 0 and drops it. Covered by no_drift on direct. -EnvMatrixExclude.no_telemetry_config = ["INPUT_CONFIG=model_serving_endpoint_telemetry_config.yml.tmpl"] - # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (deploy + migrate + plan at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index d9e7e10caca..9b8f7ea4435 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -30,7 +30,6 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index c56b4f264ff..a206e32586f 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -48,7 +48,6 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", @@ -110,7 +109,6 @@ no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_loc # External volumes reference external locations; excluded from cloud for the same reason no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"] - # A vector search index takes 10-20 min to create, and these tests deploy it twice # (deploy, then re-plan), making it by far the slowest variant on cloud. The dedicated # vector_search_indexes resource test already covers it on cloud (CloudSlow), so exclude diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml index d9fe064aa30..0728417344d 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -1,4 +1,6 @@ RecordRequests = true -# Covers the direct engine's telemetry_config classification and update path. +# Covers the direct engine's telemetry_config classification and update path. Terraform is +# excluded because the provider types sampling_fraction as an integer while the API takes a +# fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0 and drops it. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From fcc8cec7af230daab655f1d2c9a6cc1cb1ab2d57 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 11:54:35 +0000 Subject: [PATCH 06/28] Skip a remote-only telemetry_config instead of removing it The backend default was declared at telemetry_config.telemetry_profile_id, but when telemetry_config is absent from the bundle, structdiff reports the populated remote as a single change on the parent telemetry_config path. Neither the rule prefix nor the map fallback in shouldSkipBackendDefault matches a struct at that path, so the change was planned as an update and DoUpdate sent a PATCH with no telemetry_config, deleting telemetry from an endpoint the bundle never configured. Adopting an endpoint that already has telemetry reaches this. Declaring the rule on the whole subtree covers the parent path. A genuine removal is unaffected because it carries a non-nil old value, which shouldSkipBackendDefault requires to be nil. The new telemetry_config_unmanaged test covers it. With the rule back at the child path it fails with the endpoint's telemetry_config reading back as null and a second PATCH carrying an empty body. --- .../telemetry_config_unmanaged/databricks.yml | 20 ++++++ .../telemetry_config_unmanaged/out.test.toml | 3 + .../telemetry_config_unmanaged/output.txt | 71 +++++++++++++++++++ .../drift/telemetry_config_unmanaged/script | 17 +++++ .../telemetry_config_unmanaged/test.toml | 7 ++ bundle/direct/dresources/resources.yml | 9 ++- 6 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml new file mode 100644 index 00000000000..b3d2d9420aa --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + endpoint1: + name: test-endpoint + config: + served_entities: + - name: prod + external_model: + name: gpt-4o-mini + provider: openai + task: llm/v1/chat + openai_config: + openai_api_key_plaintext: sk-test-plaintext-key + traffic_config: + routes: + - served_model_name: prod + traffic_percentage: 100 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt new file mode 100644 index 00000000000..5c33c95d56e --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt @@ -0,0 +1,71 @@ + +=== Deploy an endpoint that does not declare telemetry_config +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Telemetry gets configured out of band, as on an adopted endpoint +=== A remote-only telemetry_config is a backend default, not drift +>>> [CLI] bundle plan --output json +{ + "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "action": "skip", + "reason": "input_only", + "old": "sk-test-plaintext-key", + "new": "sk-test-plaintext-key", + "remote": "" + }, + "description": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "route_optimized": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "telemetry_config": { + "action": "skip", + "reason": "backend_default", + "remote": { + "inference_table_config": { + "name": "main.default.test-endpoint_payload", + "sampling_fraction": 0.5 + }, + "telemetry_profile_id": "[UUID]" + } + } +} + +=== Deploying does not remove it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] api get /api/2.0/serving-endpoints/test-endpoint +{ + "inference_table_config": { + "name": "main.default.test-endpoint_payload", + "sampling_fraction": 0.5 + }, + "telemetry_profile_id": "[UUID]" +} + +=== The out-of-band call above is the only telemetry-config PATCH +>>> print_requests.py --method PATCH //serving-endpoints +{ + "method": "PATCH", + "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", + "body": { + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script new file mode 100644 index 00000000000..89bba593392 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -0,0 +1,17 @@ +echo "*" > .gitignore + +title "Deploy an endpoint that does not declare telemetry_config" +trace $CLI bundle deploy + +title "Telemetry gets configured out of band, as on an adopted endpoint" +MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}}}' > /dev/null + +title "A remote-only telemetry_config is a backend default, not drift" +trace $CLI bundle plan --output json | jq '.plan[].changes' + +title "Deploying does not remove it" +trace $CLI bundle deploy +trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config + +title "The out-of-band call above is the only telemetry-config PATCH" +trace print_requests.py --method PATCH //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml new file mode 100644 index 00000000000..3402a8f8840 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -0,0 +1,7 @@ +RecordRequests = true + +# Guards the backend_defaults rule at the telemetry_config parent path. Without it the +# remote-only config is planned as an update and DoUpdate sends a PATCH with no +# telemetry_config, deleting a configuration the bundle never managed. The classification +# under test belongs to the direct engine, so terraform is excluded. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b6d55cfaed1..1347cb014d7 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -318,7 +318,14 @@ resources: backend_defaults: # The backend assigns a telemetry profile and echoes its ID back on GET. Users may # also pin an existing profile, so drift still applies when it is set locally. - - field: telemetry_config.telemetry_profile_id + # + # The rule covers the whole subtree rather than just telemetry_profile_id: when + # telemetry_config is absent locally, structdiff reports the populated remote as one + # change on the parent path, which no child rule matches. Without this the change + # becomes an update and DoUpdate sends a PATCH with no telemetry_config, removing a + # configuration the bundle never managed. A genuine removal is unaffected because it + # has a non-nil old value. + - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 # common.CustomizeSchemaPath(m, "config", "served_entities", "name").SetComputed() From f29ba546c48e729999b5e94b031759f3269fb1b0 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 11:54:45 +0000 Subject: [PATCH 07/28] Restore blank line dropped from the invariant test.toml Leftover from adding and then removing the telemetry_config invariant config. --- acceptance/bundle/invariant/test.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a206e32586f..32a0808dbac 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -109,6 +109,7 @@ no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_loc # External volumes reference external locations; excluded from cloud for the same reason no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"] + # A vector search index takes 10-20 min to create, and these tests deploy it twice # (deploy, then re-plan), making it by far the slowest variant on cloud. The dedicated # vector_search_indexes resource test already covers it on cloud (CloudSlow), so exclude From 271f236b9a3c8c713ba3c72eb0432dd310bd0fb4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 12:24:12 +0000 Subject: [PATCH 08/28] Stop Git Bash rewriting the api get path on Windows The telemetry_config_unmanaged test reads the endpoint back with `api get` and a leading-slash path, which MSYS rewrites to C:\Program Files\Git\api\... before the CLI sees it, so the request missed the testserver stub and the test failed on Windows only. The out-of-band `api patch` above it already guarded against this. Setting MSYS_NO_PATHCONV for the whole script via test.toml would instead break the print_requests.py invocation on the last line, the same reason script.prepare unsets it around its Python helpers. --- .../drift/telemetry_config_unmanaged/script | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index 89bba593392..bd26a492b81 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -4,6 +4,8 @@ title "Deploy an endpoint that does not declare telemetry_config" trace $CLI bundle deploy title "Telemetry gets configured out of band, as on an adopted endpoint" +# MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash +# API path to C:\Program Files\Git\api\..., which misses the testserver stub. MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}}}' > /dev/null title "A remote-only telemetry_config is a backend default, not drift" @@ -11,7 +13,7 @@ trace $CLI bundle plan --output json | jq '.plan[].changes' title "Deploying does not remove it" trace $CLI bundle deploy -trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config +MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config title "The out-of-band call above is the only telemetry-config PATCH" trace print_requests.py --method PATCH //serving-endpoints From 2c0fcf343889f667036e72c21b2263572d16acde Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 12:30:39 +0000 Subject: [PATCH 09/28] Cover removing telemetry_config and document the behavior change Nothing exercised the removal path, which is the least certain part of this change: dropping telemetry_config from the bundle makes DoUpdate PATCH with the field omitted, and whether that deletes the configuration or is a no-op is the backend's call. The drift test now removes it, asserts the endpoint reads back null, and records the empty PATCH body in the golden file so the assumption is visible rather than buried in the fake. The fake also minted a fresh telemetry_profile_id on every PATCH instead of keeping the endpoint's profile. UUID masking hides that in the output, so it could only ever mislead a future test; the patch now carries the existing ID forward. The changelog gained the user-visible regression: an endpoint the backend types as NO_CONFIG that declares telemetry_config used to deploy because the field was dropped, and now fails on every deploy. --- ...odel-serving-telemetry-config-converges.md | 2 +- .../drift/telemetry_config/output.txt | 55 +++++++++++++++++++ .../drift/telemetry_config/script | 11 ++++ libs/testserver/serving_endpoints.go | 14 +++-- 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index e726224fce6..541a6adaeaf 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the backend provisions a telemetry profile for the endpoint and returns its `telemetry_profile_id` on GET, which was reported as drift, so every `bundle plan` after a deploy showed the endpoint as a perpetual update. Changes to `telemetry_config` are now applied through the serving endpoints telemetry configuration API instead of being dropped. +Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the backend provisions a telemetry profile for the endpoint and returns its `telemetry_profile_id` on GET, which was reported as drift, so every `bundle plan` after a deploy showed the endpoint as a perpetual update. Changes to `telemetry_config` are now applied through the serving endpoints telemetry configuration API instead of being dropped, and telemetry the bundle does not declare is left in place instead of being deleted on the next deploy. Note that the backend rejects telemetry configuration on an endpoint it types as `NO_CONFIG`: such an endpoint declaring `telemetry_config` used to deploy because the backend dropped the field, and now fails with `Telemetry configuration is not supported for endpoint type 'NO_CONFIG'` ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index b5ae4ba51af..5a5cc46263c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -59,6 +59,56 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== Removing telemetry_config from the bundle removes it remotely +>>> [CLI] bundle plan --output json +{ + "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "action": "skip", + "reason": "input_only", + "old": "sk-test-plaintext-key", + "new": "sk-test-plaintext-key", + "remote": "" + }, + "description": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "route_optimized": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "telemetry_config": { + "action": "update", + "old": { + "inference_table_config": { + "sampling_fraction": 0.9 + } + }, + "remote": { + "inference_table_config": { + "name": "main.default.test-endpoint_payload", + "sampling_fraction": 0.9 + }, + "telemetry_profile_id": "[UUID]" + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] api get /api/2.0/serving-endpoints/test-endpoint +null + +=== Plan converges with telemetry_config removed +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> print_requests.py //serving-endpoints { "method": "POST", @@ -106,3 +156,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } } +{ + "method": "PATCH", + "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", + "body": {} +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index 3d46ccf88dc..7bc4ae3d9c8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -16,4 +16,15 @@ trace $CLI bundle deploy title "Plan converges again" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" +title "Removing telemetry_config from the bundle removes it remotely" +update_file.py databricks.yml $' telemetry_config:\n inference_table_config:\n sampling_fraction: 0.9\n' '' +trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle deploy +# MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash +# API path to C:\Program Files\Git\api\..., which misses the testserver stub. +MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config + +title "Plan converges with telemetry_config removed" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + trace print_requests.py //serving-endpoints diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 23fea03c82d..8418415d9f1 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -36,13 +36,19 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se } // applyTelemetryConfig mirrors the backend, which assigns the profile ID and the -// inference table name that the caller never supplies. -func applyTelemetryConfig(endpointName string, config *serving.TelemetryConfig) *serving.TelemetryConfig { +// inference table name that the caller never supplies. previous is the endpoint's +// current telemetry config, if any: the profile belongs to the endpoint, so a patch +// that does not name one updates the existing profile rather than provisioning +// another. +func applyTelemetryConfig(endpointName string, previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil } applied := *config + if applied.TelemetryProfileId == "" && previous != nil { + applied.TelemetryProfileId = previous.TelemetryProfileId + } if applied.TelemetryProfileId == "" { applied.TelemetryProfileId = nextUUID() } @@ -245,7 +251,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { PermissionLevel: serving.ServingEndpointDetailedPermissionLevelCanManage, RouteOptimized: createReq.RouteOptimized, Tags: createReq.Tags, - TelemetryConfig: applyTelemetryConfig(createReq.Name, createReq.TelemetryConfig), + TelemetryConfig: applyTelemetryConfig(createReq.Name, nil, createReq.TelemetryConfig), State: &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, Ready: serving.EndpointStateReadyNotReady, @@ -411,7 +417,7 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } // An omitted telemetry_config removes the configuration from the endpoint. - endpoint.TelemetryConfig = applyTelemetryConfig(name, patchReq.TelemetryConfig) + endpoint.TelemetryConfig = applyTelemetryConfig(name, endpoint.TelemetryConfig, patchReq.TelemetryConfig) endpoint.LastUpdatedTimestamp = nowMilli() s.ServingEndpoints[name] = endpoint From e586f1bd09850be9fe0f76493dc898121e49598a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 12:50:43 +0000 Subject: [PATCH 10/28] Reject telemetry_config on an endpoint that serves nothing Wiring the telemetry-config API into DoUpdate turned a silent no-op into a hard failure for endpoints the backend types as NO_CONFIG. Create drops telemetry_config on such an endpoint instead of failing, so the bundle deployed once and then failed on every later deploy, when the plan applied the field the create had thrown away. The field can never be honored there, so keep failing, but fail at validation time naming the field and the fix instead of surfacing a 400 from the second deploy on. --- ...odel-serving-telemetry-config-converges.md | 2 +- .../databricks.yml | 10 ++ .../out.test.toml | 3 + .../output.txt | 15 +++ .../script | 1 + .../validate_serving_telemetry_config.go | 55 ++++++++++ .../validate_serving_telemetry_config_test.go | 101 ++++++++++++++++++ bundle/phases/initialize.go | 4 + 8 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_served_entity/out.test.toml create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_served_entity/script create mode 100644 bundle/config/validate/validate_serving_telemetry_config.go create mode 100644 bundle/config/validate/validate_serving_telemetry_config_test.go diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index 541a6adaeaf..b38395de0c5 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the backend provisions a telemetry profile for the endpoint and returns its `telemetry_profile_id` on GET, which was reported as drift, so every `bundle plan` after a deploy showed the endpoint as a perpetual update. Changes to `telemetry_config` are now applied through the serving endpoints telemetry configuration API instead of being dropped, and telemetry the bundle does not declare is left in place instead of being deleted on the next deploy. Note that the backend rejects telemetry configuration on an endpoint it types as `NO_CONFIG`: such an endpoint declaring `telemetry_config` used to deploy because the backend dropped the field, and now fails with `Telemetry configuration is not supported for endpoint type 'NO_CONFIG'` ([#6106](https://github.com/databricks/cli/pull/6106)). +Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the backend provisions a telemetry profile for the endpoint and returns its `telemetry_profile_id` on GET, which was reported as drift, so every `bundle plan` after a deploy showed the endpoint as a perpetual update. Changes to `telemetry_config` are now applied through the serving endpoints telemetry configuration API instead of being dropped, and telemetry the bundle does not declare is left in place instead of being deleted on the next deploy. Setting `telemetry_config` on an endpoint that serves nothing is now an error from `bundle validate`: the backend types such an endpoint as `NO_CONFIG` and rejects telemetry configuration on it, so the field used to be accepted at deploy time and then ignored ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml new file mode 100644 index 00000000000..b9a89a54634 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: "model_serving_telemetry" + +resources: + model_serving_endpoints: + endpoint1: + name: "test-endpoint" + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt new file mode 100644 index 00000000000..18affd17138 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt @@ -0,0 +1,15 @@ +Error: telemetry_config is not supported on an endpoint that does not serve a model + at resources.model_serving_endpoints.endpoint1.telemetry_config + in databricks.yml:9:9 + +The serving endpoints API rejects telemetry configuration on an endpoint with no served entities. Add an entry to config.served_entities, or remove telemetry_config. + +Name: model_serving_telemetry +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/script b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go new file mode 100644 index 00000000000..eea8eede716 --- /dev/null +++ b/bundle/config/validate/validate_serving_telemetry_config.go @@ -0,0 +1,55 @@ +package validate + +import ( + "cmp" + "context" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func ValidateServingTelemetryConfig() bundle.ReadOnlyMutator { + return &validateServingTelemetryConfig{} +} + +type validateServingTelemetryConfig struct{ bundle.RO } + +func (v *validateServingTelemetryConfig) Name() string { + return "validate:validate_serving_telemetry_config" +} + +func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // The serving endpoints API types an endpoint that serves nothing as NO_CONFIG and + // rejects telemetry configuration on it with "Telemetry configuration is not supported + // for endpoint type 'NO_CONFIG'". Create drops the field instead of failing, so without + // this the bundle deploys once and then fails on every later deploy, when the plan + // finally applies telemetry_config through the telemetry-config API. + for key, endpoint := range b.Config.Resources.ModelServingEndpoints { + if endpoint.TelemetryConfig == nil { + continue + } + if endpoint.Config != nil && (len(endpoint.Config.ServedEntities) > 0 || len(endpoint.Config.ServedModels) > 0) { + continue + } + + path := "resources.model_serving_endpoints." + key + ".telemetry_config" + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "telemetry_config is not supported on an endpoint that does not serve a model", + Detail: "The serving endpoints API rejects telemetry configuration on an endpoint with no served entities. Add an entry to config.served_entities, or remove telemetry_config.", + Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocations(path), + }) + } + + // Map iteration order is randomized; sort by path for stable output. + slices.SortFunc(diags, func(x, y diag.Diagnostic) int { + return cmp.Compare(x.Paths[0].String(), y.Paths[0].String()) + }) + + return diags +} diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go new file mode 100644 index 00000000000..c315b716181 --- /dev/null +++ b/bundle/config/validate/validate_serving_telemetry_config_test.go @@ -0,0 +1,101 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "my_endpoint": {CreateServingEndpoint: endpoint}, + }, + }, + }, + } +} + +func telemetryConfig() *serving.TelemetryConfig { + return &serving.TelemetryConfig{ + InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, + } +} + +func TestValidateServingTelemetryConfig(t *testing.T) { + tests := []struct { + name string + endpoint serving.CreateServingEndpoint + wantErr bool + }{ + { + name: "telemetry with a served entity", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: telemetryConfig(), + Config: &serving.EndpointCoreConfigInput{ + ServedEntities: []serving.ServedEntityInput{{Name: "prod"}}, + }, + }, + }, + { + // served_models is converted to served_entities by ModelServingEndpointFixups, + // so accept either to stay independent of mutator order. + name: "telemetry with a served model", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: telemetryConfig(), + Config: &serving.EndpointCoreConfigInput{ + ServedModels: []serving.ServedModelInput{{ModelName: "m", ModelVersion: "1"}}, + }, + }, + }, + { + name: "no telemetry and nothing served", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + }, + }, + { + name: "telemetry and no config", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: telemetryConfig(), + }, + wantErr: true, + }, + { + name: "telemetry and a config that serves nothing", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: telemetryConfig(), + Config: &serving.EndpointCoreConfigInput{}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + diags := ValidateServingTelemetryConfig().Apply(t.Context(), endpointBundle(tt.endpoint)) + + if !tt.wantErr { + assert.Empty(t, diags) + return + } + + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "telemetry_config is not supported on an endpoint that does not serve a model", diags[0].Summary) + assert.Equal(t, "resources.model_serving_endpoints.my_endpoint.telemetry_config", diags[0].Paths[0].String()) + }) + } +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 873dc31dc07..b91e2d68689 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,6 +185,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Validate that telemetry_config is only set on serving endpoints that serve a model. + // The API rejects it on any other endpoint, and only once the plan applies it. + validate.ValidateServingTelemetryConfig(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. From 585ea0a167dee2d9656ee0fd58e0ccd377611549 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 5 Aug 2026 12:10:02 +0000 Subject: [PATCH 11/28] Correct telemetry_config against the verified serving API The classification here had been rewritten twice on differing assumptions about what the telemetry API accepts and returns, with the testserver fake reshaped to match each one, so the acceptance tests could not tell whether any of them held. Checked against a workspace instead: - A telemetry_config naming neither table_names nor telemetry_profile_id is discarded; create and the telemetry API both report success and apply nothing. - GET echoes telemetry_profile_id, and inference_table_config once the config was accepted. table_names is never echoed. - Telemetry is rejected on endpoints typed NO_CONFIG and EXTERNAL_MODELS, and create rejects a foundation model outright. - An empty PATCH body removes the configuration. - The telemetry API refuses to run while the endpoint is still applying an earlier update, and every update call starts one. So table_names is input_only but inference_table_config is not: sampling_fraction round trips and needs ordinary drift detection. Suppressing the subtree would silently swallow the edits users actually make. DoUpdate now waits for the endpoint to settle before patching telemetry when an earlier call in the same pass ran. WaitAfterUpdate only runs once DoUpdate has returned, so without this a deploy touching both config and telemetry_config fails on the telemetry call. Validation covers both forms the API discards, and the fake models the endpoint-type rejection so a fixture the backend would refuse cannot pass. The drift fixture moves off an external model, which the API rejects, onto a custom served model and gains a sampling_fraction edit. Both drift tests stay local: telemetry needs a registered UC model and roughly ten minutes to provision, and each PATCH locks the endpoint for another two to three. --- ...odel-serving-telemetry-config-converges.md | 2 +- ...del-serving-telemetry-config-validation.md | 1 + .../drift/telemetry_config/databricks.yml | 19 +- .../drift/telemetry_config/output.txt | 183 +++++++++++++++--- .../drift/telemetry_config/script | 14 +- .../drift/telemetry_config/test.toml | 6 + .../telemetry_config_unmanaged/databricks.yml | 10 +- .../telemetry_config_unmanaged/output.txt | 19 +- .../drift/telemetry_config_unmanaged/script | 2 +- .../databricks.yml | 28 +++ .../out.test.toml | 0 .../output.txt | 33 ++++ .../script | 0 .../databricks.yml | 17 ++ .../out.test.toml | 3 + .../output.txt | 15 ++ .../script | 1 + .../databricks.yml | 10 - .../output.txt | 15 -- .../validate_serving_telemetry_config.go | 85 ++++++-- .../validate_serving_telemetry_config_test.go | 97 +++++++--- .../dresources/model_serving_endpoint.go | 16 ++ bundle/direct/dresources/resources.yml | 12 +- libs/testserver/serving_endpoints.go | 71 +++++-- 24 files changed, 503 insertions(+), 156 deletions(-) create mode 100644 .nextchanges/notable-changes/model-serving-telemetry-config-validation.md create mode 100644 acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml rename acceptance/bundle/validate/{model_serving_telemetry_without_served_entity => model_serving_telemetry_unsupported_endpoint}/out.test.toml (100%) create mode 100644 acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt rename acceptance/bundle/validate/{model_serving_telemetry_without_served_entity => model_serving_telemetry_unsupported_endpoint}/script (100%) create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/script delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index b38395de0c5..9f8423adc7e 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the backend provisions a telemetry profile for the endpoint and returns its `telemetry_profile_id` on GET, which was reported as drift, so every `bundle plan` after a deploy showed the endpoint as a perpetual update. Changes to `telemetry_config` are now applied through the serving endpoints telemetry configuration API instead of being dropped, and telemetry the bundle does not declare is left in place instead of being deleted on the next deploy. Setting `telemetry_config` on an endpoint that serves nothing is now an error from `bundle validate`: the backend types such an endpoint as `NO_CONFIG` and rejects telemetry configuration on it, so the field used to be accepted at deploy time and then ignored ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: `telemetry_config` on model serving endpoints now converges, and edits to it are applied. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id` on GET, so every `bundle plan` after a deploy reported the endpoint as a perpetual update. Edits were planned but then silently dropped instead of being sent to the telemetry configuration API, telemetry the bundle does not declare is now left in place instead of being deleted on the next deploy, and a deploy that changes both `config` and `telemetry_config` now waits for the endpoint to settle instead of failing because the endpoint is still updating ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md new file mode 100644 index 00000000000..82c1fa39fa5 --- /dev/null +++ b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md @@ -0,0 +1 @@ +`bundle validate` now rejects a `telemetry_config` on a model serving endpoint that the serving API cannot apply it to: one that serves nothing or only external models, and one that sets neither `table_names` nor `telemetry_profile_id`. Both forms used to be accepted and then silently discarded by the backend, so the bundle appeared to deploy while the endpoint received no telemetry. A bundle already deployed in either shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml index 5b40d000814..a5a41597efb 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -8,16 +8,15 @@ resources: config: served_entities: - name: prod - external_model: - name: gpt-4o-mini - provider: openai - task: llm/v1/chat - openai_config: - openai_api_key_plaintext: sk-test-plaintext-key - traffic_config: - routes: - - served_model_name: prod - traffic_percentage: 100 + entity_name: main.default.test_model + entity_version: "1" + workload_size: Small + scale_to_zero_enabled: true telemetry_config: + table_names: + logs_table: main.default.test_logs + traces_table: main.default.test_traces + metrics_table: main.default.test_metrics + annotations_table: main.default.test_annotations inference_table_config: sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index 5a5cc46263c..866094077d0 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -6,19 +6,25 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Plan converges: backend-assigned fields are not drift +=== Plan converges: the profile and inference table the backend named are not drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== Editing sampling_fraction is a real change +=== Editing sampling_fraction is a real change: it round trips through GET >>> [CLI] bundle plan --output json { - "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "config.traffic_config": { "action": "skip", - "reason": "input_only", - "old": "sk-test-plaintext-key", - "new": "sk-test-plaintext-key", - "remote": "" + "reason": "backend_default", + "remote": { + "routes": [ + { + "served_entity_name": "prod", + "served_model_name": "prod", + "traffic_percentage": 100 + } + ] + } }, "description": { "action": "skip", @@ -33,7 +39,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "telemetry_config.inference_table_config.name": { "action": "skip", "reason": "spec:output_only", - "remote": "main.default.test-endpoint_payload" + "remote": "main.default.test_logs_payload" }, "telemetry_config.inference_table_config.sampling_fraction": { "action": "update", @@ -41,6 +47,22 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "new": 0.9, "remote": 0.5 }, + "telemetry_config.table_names": { + "action": "skip", + "reason": "input_only", + "old": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" + }, + "new": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" + } + }, "telemetry_config.telemetry_profile_id": { "action": "skip", "reason": "backend_default", @@ -59,16 +81,90 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== Removing telemetry_config from the bundle removes it remotely +=== Editing table_names is a real change even though GET never echoes it >>> [CLI] bundle plan --output json { - "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "config.traffic_config": { "action": "skip", - "reason": "input_only", - "old": "sk-test-plaintext-key", - "new": "sk-test-plaintext-key", + "reason": "backend_default", + "remote": { + "routes": [ + { + "served_entity_name": "prod", + "served_model_name": "prod", + "traffic_percentage": 100 + } + ] + } + }, + "description": { + "action": "skip", + "reason": "empty", "remote": "" }, + "route_optimized": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "telemetry_config.inference_table_config.name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "main.default.test_logs_payload" + }, + "telemetry_config.table_names": { + "action": "update", + "old": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" + }, + "new": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs_v2", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" + } + }, + "telemetry_config.table_names.logs_table": { + "action": "update", + "old": "main.default.test_logs", + "new": "main.default.test_logs_v2" + }, + "telemetry_config.telemetry_profile_id": { + "action": "skip", + "reason": "backend_default", + "remote": "[UUID]" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan converges after the tables changed +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Removing telemetry_config from the bundle removes it remotely +>>> [CLI] bundle plan --output json +{ + "config.traffic_config": { + "action": "skip", + "reason": "backend_default", + "remote": { + "routes": [ + { + "served_entity_name": "prod", + "served_model_name": "prod", + "traffic_percentage": 100 + } + ] + } + }, "description": { "action": "skip", "reason": "empty", @@ -84,11 +180,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "old": { "inference_table_config": { "sampling_fraction": 0.9 + }, + "table_names": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs_v2", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" } }, "remote": { "inference_table_config": { - "name": "main.default.test-endpoint_payload", + "name": "main.default.test_logs_v2_payload", "sampling_fraction": 0.9 }, "telemetry_profile_id": "[UUID]" @@ -117,30 +219,41 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "config": { "served_entities": [ { - "external_model": { - "name": "gpt-4o-mini", - "openai_config": { - "openai_api_key_plaintext": "sk-test-plaintext-key" - }, - "provider": "openai", - "task": "llm/v1/chat" - }, - "name": "prod" + "entity_name": "main.default.test_model", + "entity_version": "1", + "name": "prod", + "scale_to_zero_enabled": true, + "workload_size": "Small" } - ], - "traffic_config": { - "routes": [ - { - "served_model_name": "prod", - "traffic_percentage": 100 - } - ] - } + ] }, "name": "test-endpoint", "telemetry_config": { "inference_table_config": { "sampling_fraction": 0.5 + }, + "table_names": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" + } + } + } +} +{ + "method": "PATCH", + "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", + "body": { + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.9 + }, + "table_names": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" } } } @@ -152,6 +265,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "telemetry_config": { "inference_table_config": { "sampling_fraction": 0.9 + }, + "table_names": { + "annotations_table": "main.default.test_annotations", + "logs_table": "main.default.test_logs_v2", + "metrics_table": "main.default.test_metrics", + "traces_table": "main.default.test_traces" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index 7bc4ae3d9c8..b13d7322cec 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -3,10 +3,10 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Plan converges: backend-assigned fields are not drift" +title "Plan converges: the profile and inference table the backend named are not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "Editing sampling_fraction is a real change" +title "Editing sampling_fraction is a real change: it round trips through GET" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle plan --output json | jq '.plan[].changes' @@ -16,8 +16,16 @@ trace $CLI bundle deploy title "Plan converges again" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" +title "Editing table_names is a real change even though GET never echoes it" +update_file.py databricks.yml "main.default.test_logs" "main.default.test_logs_v2" +trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle deploy + +title "Plan converges after the tables changed" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + title "Removing telemetry_config from the bundle removes it remotely" -update_file.py databricks.yml $' telemetry_config:\n inference_table_config:\n sampling_fraction: 0.9\n' '' +update_file.py databricks.yml $' telemetry_config:\n table_names:\n logs_table: main.default.test_logs_v2\n traces_table: main.default.test_traces\n metrics_table: main.default.test_metrics\n annotations_table: main.default.test_annotations\n inference_table_config:\n sampling_fraction: 0.9\n' '' trace $CLI bundle plan --output json | jq '.plan[].changes' trace $CLI bundle deploy # MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml index 0728417344d..c8350f18910 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -3,4 +3,10 @@ RecordRequests = true # Covers the direct engine's telemetry_config classification and update path. Terraform is # excluded because the provider types sampling_fraction as an integer while the API takes a # fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0 and drops it. +# +# Local only: telemetry is supported only on an endpoint with a custom served model, which +# needs a registered UC model and roughly ten minutes to provision, and every telemetry +# PATCH then locks the endpoint for another two to three minutes. The fake's behaviour was +# recorded against a real workspace instead; see the comments in +# libs/testserver/serving_endpoints.go. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml index b3d2d9420aa..1ccacd2fee0 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml @@ -8,12 +8,10 @@ resources: config: served_entities: - name: prod - external_model: - name: gpt-4o-mini - provider: openai - task: llm/v1/chat - openai_config: - openai_api_key_plaintext: sk-test-plaintext-key + entity_name: main.default.test_model + entity_version: "1" + workload_size: Small + scale_to_zero_enabled: true traffic_config: routes: - served_model_name: prod diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt index 5c33c95d56e..88735a92023 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt @@ -10,13 +10,6 @@ Deployment complete! === A remote-only telemetry_config is a backend default, not drift >>> [CLI] bundle plan --output json { - "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { - "action": "skip", - "reason": "input_only", - "old": "sk-test-plaintext-key", - "new": "sk-test-plaintext-key", - "remote": "" - }, "description": { "action": "skip", "reason": "empty", @@ -31,10 +24,6 @@ Deployment complete! "action": "skip", "reason": "backend_default", "remote": { - "inference_table_config": { - "name": "main.default.test-endpoint_payload", - "sampling_fraction": 0.5 - }, "telemetry_profile_id": "[UUID]" } } @@ -49,10 +38,6 @@ Deployment complete! >>> [CLI] api get /api/2.0/serving-endpoints/test-endpoint { - "inference_table_config": { - "name": "main.default.test-endpoint_payload", - "sampling_fraction": 0.5 - }, "telemetry_profile_id": "[UUID]" } @@ -63,8 +48,8 @@ Deployment complete! "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": { "telemetry_config": { - "inference_table_config": { - "sampling_fraction": 0.5 + "table_names": { + "logs_table": "main.default.other_logs" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index bd26a492b81..c14a95944de 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -6,7 +6,7 @@ trace $CLI bundle deploy title "Telemetry gets configured out of band, as on an adopted endpoint" # MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash # API path to C:\Program Files\Git\api\..., which misses the testserver stub. -MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "A remote-only telemetry_config is a backend default, not drift" trace $CLI bundle plan --output json | jq '.plan[].changes' diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml new file mode 100644 index 00000000000..c79e4b32f19 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: "model_serving_telemetry" + +resources: + model_serving_endpoints: + # Typed NO_CONFIG by the API: nothing is served. + endpoint1: + name: "test-endpoint-1" + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 + + # Typed EXTERNAL_MODELS by the API: it serves something, but telemetry is still + # only supported for custom served models. + endpoint2: + name: "test-endpoint-2" + config: + served_entities: + - name: prod + external_model: + name: gpt-4o-mini + provider: openai + task: llm/v1/chat + openai_config: + openai_api_key_plaintext: sk-test-plaintext-key + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/out.test.toml similarity index 100% rename from acceptance/bundle/validate/model_serving_telemetry_without_served_entity/out.test.toml rename to acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/out.test.toml diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt new file mode 100644 index 00000000000..b3a36dcd973 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt @@ -0,0 +1,33 @@ +Error: telemetry_config is only supported on an endpoint that serves a registered model + at resources.model_serving_endpoints.endpoint1.telemetry_config + in databricks.yml:10:9 + +The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. + +Error: telemetry_config is only supported on an endpoint that serves a registered model + at resources.model_serving_endpoints.endpoint2.telemetry_config + in databricks.yml:27:9 + +The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. + +Error: telemetry_config must set table_names or telemetry_profile_id + at resources.model_serving_endpoints.endpoint1.telemetry_config + in databricks.yml:10:9 + +The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. + +Error: telemetry_config must set table_names or telemetry_profile_id + at resources.model_serving_endpoints.endpoint2.telemetry_config + in databricks.yml:27:9 + +The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. + +Name: model_serving_telemetry +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default + +Found 4 errors + +Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/script b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/script similarity index 100% rename from acceptance/bundle/validate/model_serving_telemetry_without_served_entity/script rename to acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/script diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml new file mode 100644 index 00000000000..c17b9b334d3 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: "model_serving_telemetry" + +resources: + model_serving_endpoints: + # The endpoint supports telemetry, but an inference_table_config on its own names + # no profile, so the API discards it instead of applying the sampling fraction. + endpoint1: + name: "test-endpoint" + config: + served_entities: + - name: prod + entity_name: main.default.test_model + entity_version: "1" + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt new file mode 100644 index 00000000000..31e0b42ac5e --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt @@ -0,0 +1,15 @@ +Error: telemetry_config must set table_names or telemetry_profile_id + at resources.model_serving_endpoints.endpoint1.telemetry_config + in databricks.yml:16:9 + +The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. + +Name: model_serving_telemetry +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/script b/acceptance/bundle/validate/model_serving_telemetry_without_profile/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml deleted file mode 100644 index b9a89a54634..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/databricks.yml +++ /dev/null @@ -1,10 +0,0 @@ -bundle: - name: "model_serving_telemetry" - -resources: - model_serving_endpoints: - endpoint1: - name: "test-endpoint" - telemetry_config: - inference_table_config: - sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt deleted file mode 100644 index 18affd17138..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_served_entity/output.txt +++ /dev/null @@ -1,15 +0,0 @@ -Error: telemetry_config is not supported on an endpoint that does not serve a model - at resources.model_serving_endpoints.endpoint1.telemetry_config - in databricks.yml:9:9 - -The serving endpoints API rejects telemetry configuration on an endpoint with no served entities. Add an entry to config.served_entities, or remove telemetry_config. - -Name: model_serving_telemetry -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default - -Found 1 error - -Exit code: 1 diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go index eea8eede716..dcb6c71841e 100644 --- a/bundle/config/validate/validate_serving_telemetry_config.go +++ b/bundle/config/validate/validate_serving_telemetry_config.go @@ -1,13 +1,12 @@ package validate import ( - "cmp" "context" - "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/serving" ) func ValidateServingTelemetryConfig() bundle.ReadOnlyMutator { @@ -23,33 +22,77 @@ func (v *validateServingTelemetryConfig) Name() string { func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - // The serving endpoints API types an endpoint that serves nothing as NO_CONFIG and - // rejects telemetry configuration on it with "Telemetry configuration is not supported - // for endpoint type 'NO_CONFIG'". Create drops the field instead of failing, so without - // this the bundle deploys once and then fails on every later deploy, when the plan - // finally applies telemetry_config through the telemetry-config API. for key, endpoint := range b.Config.Resources.ModelServingEndpoints { if endpoint.TelemetryConfig == nil { continue } - if endpoint.Config != nil && (len(endpoint.Config.ServedEntities) > 0 || len(endpoint.Config.ServedModels) > 0) { - continue - } path := "resources.model_serving_endpoints." + key + ".telemetry_config" - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: "telemetry_config is not supported on an endpoint that does not serve a model", - Detail: "The serving endpoints API rejects telemetry configuration on an endpoint with no served entities. Add an entry to config.served_entities, or remove telemetry_config.", - Paths: []dyn.Path{dyn.MustPathFromString(path)}, - Locations: b.Config.GetLocations(path), - }) + addDiag := func(summary, detail string) { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Detail: detail, + Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocations(path), + }) + } + + // The telemetry configuration API only supports endpoints with custom served + // models: it rejects an endpoint that serves nothing ("Telemetry configuration is + // not supported for endpoint type 'NO_CONFIG'") and one that only proxies external + // models (the same error with 'EXTERNAL_MODELS'). Create drops telemetry_config + // instead of failing, so without this the bundle deploys once and then fails on + // every later deploy, when the plan finally applies the field through that API. + if !servesRegisteredModel(endpoint.Config) { + addDiag( + "telemetry_config is only supported on an endpoint that serves a registered model", + "The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config.", + ) + } + + // The API identifies a telemetry configuration by an existing profile or by the + // tables to create one from. Given neither, both create and the telemetry + // configuration API return success and apply nothing, so an inference_table_config + // on its own is silently discarded. Left unchecked the CLI would record it in state + // and report a perpetual update against an endpoint that never received it. + if !identifiesTelemetryProfile(endpoint.TelemetryConfig) { + addDiag( + "telemetry_config must set table_names or telemetry_profile_id", + "The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it.", + ) + } } - // Map iteration order is randomized; sort by path for stable output. - slices.SortFunc(diags, func(x, y diag.Diagnostic) int { - return cmp.Compare(x.Paths[0].String(), y.Paths[0].String()) - }) + sortDiagnostics(diags) return diags } + +// identifiesTelemetryProfile reports whether the configuration names the telemetry +// profile to use or the tables to create one from. +func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { + return config.TableNames != nil || config.TelemetryProfileId != "" +} + +// servesRegisteredModel reports whether the endpoint serves at least one registered +// model, as opposed to nothing at all or only external models. A foundation model is +// indistinguishable from a custom model here (both are named by entity_name), so it is +// accepted; create rejects it outright with "Telemetry configuration is supported only +// for custom CPU or GPU models", which is already a clear failure at the first deploy. +// +// ModelServingEndpointFixups has converted served_models to served_entities by the time +// validation runs, so only served_entities is inspected. +func servesRegisteredModel(config *serving.EndpointCoreConfigInput) bool { + if config == nil { + return false + } + + for _, entity := range config.ServedEntities { + if entity.ExternalModel == nil && entity.EntityName != "" { + return true + } + } + + return false +} diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go index c315b716181..72895b1a6b1 100644 --- a/bundle/config/validate/validate_serving_telemetry_config_test.go +++ b/bundle/config/validate/validate_serving_telemetry_config_test.go @@ -12,6 +12,11 @@ import ( "github.com/stretchr/testify/require" ) +const ( + unsupportedEndpoint = "telemetry_config is only supported on an endpoint that serves a registered model" + noProfile = "telemetry_config must set table_names or telemetry_profile_id" +) + func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { return &bundle.Bundle{ Config: config.Root{ @@ -24,9 +29,21 @@ func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { } } +// telemetryConfig returns a configuration that names the tables to create a profile +// from, which is the form the API accepts. func telemetryConfig() *serving.TelemetryConfig { return &serving.TelemetryConfig{ - InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, + TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}, + } +} + +func servedModelConfig() *serving.EndpointCoreConfigInput { + return &serving.EndpointCoreConfigInput{ + ServedEntities: []serving.ServedEntityInput{{ + Name: "prod", + EntityName: "main.default.my_model", + EntityVersion: "1", + }}, } } @@ -34,28 +51,22 @@ func TestValidateServingTelemetryConfig(t *testing.T) { tests := []struct { name string endpoint serving.CreateServingEndpoint - wantErr bool + want []string }{ { - name: "telemetry with a served entity", + name: "telemetry with a served registered model", endpoint: serving.CreateServingEndpoint{ Name: "my_endpoint", TelemetryConfig: telemetryConfig(), - Config: &serving.EndpointCoreConfigInput{ - ServedEntities: []serving.ServedEntityInput{{Name: "prod"}}, - }, + Config: servedModelConfig(), }, }, { - // served_models is converted to served_entities by ModelServingEndpointFixups, - // so accept either to stay independent of mutator order. - name: "telemetry with a served model", + name: "telemetry pinned to an existing profile", endpoint: serving.CreateServingEndpoint{ Name: "my_endpoint", - TelemetryConfig: telemetryConfig(), - Config: &serving.EndpointCoreConfigInput{ - ServedModels: []serving.ServedModelInput{{ModelName: "m", ModelVersion: "1"}}, - }, + TelemetryConfig: &serving.TelemetryConfig{TelemetryProfileId: "abc"}, + Config: servedModelConfig(), }, }, { @@ -70,7 +81,7 @@ func TestValidateServingTelemetryConfig(t *testing.T) { Name: "my_endpoint", TelemetryConfig: telemetryConfig(), }, - wantErr: true, + want: []string{unsupportedEndpoint}, }, { name: "telemetry and a config that serves nothing", @@ -79,7 +90,50 @@ func TestValidateServingTelemetryConfig(t *testing.T) { TelemetryConfig: telemetryConfig(), Config: &serving.EndpointCoreConfigInput{}, }, - wantErr: true, + want: []string{unsupportedEndpoint}, + }, + { + // The API types this endpoint as EXTERNAL_MODELS and rejects telemetry on it, + // even though it does serve something. + name: "telemetry with only an external model", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: telemetryConfig(), + Config: &serving.EndpointCoreConfigInput{ + ServedEntities: []serving.ServedEntityInput{{ + Name: "prod", + ExternalModel: &serving.ExternalModel{ + Name: "gpt-4o-mini", + Provider: "openai", + Task: "llm/v1/chat", + }, + }}, + }, + }, + want: []string{unsupportedEndpoint}, + }, + { + // The API discards a configuration that names no profile, so sampling on its + // own never takes effect. + name: "telemetry with only an inference table config", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, + }, + Config: servedModelConfig(), + }, + want: []string{noProfile}, + }, + { + name: "an endpoint can fail both rules", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, + }, + }, + want: []string{unsupportedEndpoint, noProfile}, }, } @@ -87,15 +141,12 @@ func TestValidateServingTelemetryConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { diags := ValidateServingTelemetryConfig().Apply(t.Context(), endpointBundle(tt.endpoint)) - if !tt.wantErr { - assert.Empty(t, diags) - return + require.Len(t, diags, len(tt.want)) + for i, summary := range tt.want { + assert.Equal(t, diag.Error, diags[i].Severity) + assert.Equal(t, summary, diags[i].Summary) + assert.Equal(t, "resources.model_serving_endpoints.my_endpoint.telemetry_config", diags[i].Paths[0].String()) } - - require.Len(t, diags, 1) - assert.Equal(t, diag.Error, diags[0].Severity) - assert.Equal(t, "telemetry_config is not supported on an endpoint that does not serve a model", diags[0].Summary) - assert.Equal(t, "resources.model_serving_endpoints.my_endpoint.telemetry_config", diags[0].Paths[0].String()) }) } } diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 6b9c091f691..f6a0d57417e 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -325,6 +325,7 @@ func (r *ResourceModelServingEndpoint) updateTags(ctx context.Context, id string func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, config *serving.CreateServingEndpoint, entry *PlanEntry) (*ModelServingEndpointRemote, error) { var err error + updated := false // Terraform makes these API calls sequentially. We do the same here. // It's an unknown as of 1st Dec 2025 if these APIs are safe to make in parallel. (we did not check) @@ -334,6 +335,7 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } + updated = true } if entry.Changes.HasChange(pathAiGateway) { @@ -341,6 +343,7 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } + updated = true } if entry.Changes.HasChange(pathConfig) { @@ -348,6 +351,7 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } + updated = true } if entry.Changes.HasChange(pathEmailNotifications) { @@ -355,9 +359,21 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } + updated = true } if entry.Changes.HasChange(pathTelemetryConfig) { + // Unlike the calls above, the telemetry configuration API refuses to run + // concurrently with an in-flight update ("Endpoint is currently being + // updated"), and each of those calls starts one. Wait for the endpoint to settle + // first; WaitAfterUpdate only runs once DoUpdate has returned. + if updated { + _, err = r.client.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, id, 35*time.Minute, nil) + if err != nil { + return nil, err + } + } + err = r.updateTelemetryConfig(ctx, id, config.TelemetryConfig) if err != nil { return nil, err diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 1347cb014d7..3d0c4fa87dc 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,6 +285,13 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only + # The backend creates a telemetry profile from these tables and reports back the + # resulting telemetry_profile_id instead; the table names themselves are never + # echoed, so remote reads back empty and a config that sets them never converges. + # inference_table_config is deliberately not listed: sampling_fraction does round + # trip, so it is subject to normal drift detection. + - field: telemetry_config.table_names + reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext reason: input_only @@ -316,8 +323,9 @@ resources: - field: rate_limits reason: not_implemented backend_defaults: - # The backend assigns a telemetry profile and echoes its ID back on GET. Users may - # also pin an existing profile, so drift still applies when it is set locally. + # The backend creates a telemetry profile from table_names and echoes its ID back on + # GET. Users may also pin an existing profile, so drift still applies when + # telemetry_profile_id is set locally. # # The rule covers the whole subtree rather than just telemetry_profile_id: when # telemetry_config is absent locally, structdiff reports the populated remote as one diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 8418415d9f1..30857b5a974 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,32 +35,57 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend, which assigns the profile ID and the -// inference table name that the caller never supplies. previous is the endpoint's -// current telemetry config, if any: the profile belongs to the endpoint, so a patch -// that does not name one updates the existing profile rather than provisioning -// another. -func applyTelemetryConfig(endpointName string, previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { +// applyTelemetryConfig mirrors the backend: a telemetry configuration is identified by an +// existing telemetry_profile_id or by the table_names to create a profile from. Given +// neither, the API returns 200 and applies nothing, so an inference_table_config on its +// own leaves the endpoint's telemetry untouched rather than setting or clearing it. +// +// What is echoed back is the profile ID plus, when the caller supplied one, the inference +// table config carrying the table name the backend derives from logs_table. table_names +// itself is never returned. +func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil } - - applied := *config - if applied.TelemetryProfileId == "" && previous != nil { - applied.TelemetryProfileId = previous.TelemetryProfileId + if config.TableNames == nil && config.TelemetryProfileId == "" { + return previous } + + applied := serving.TelemetryConfig{TelemetryProfileId: config.TelemetryProfileId} if applied.TelemetryProfileId == "" { + // New tables mean a new profile, so this does not carry over the endpoint's + // current profile ID. applied.TelemetryProfileId = nextUUID() } - if applied.InferenceTableConfig != nil { - inferenceTable := *applied.InferenceTableConfig - inferenceTable.Name = "main.default." + endpointName + "_payload" + if config.InferenceTableConfig != nil { + inferenceTable := *config.InferenceTableConfig + // Only the table_names form was observed against a real workspace. A patch that + // reuses a profile by ID does not name the tables, and what the backend reports as + // the inference table name in that case is unverified, so leave it empty. + if config.TableNames != nil { + inferenceTable.Name = config.TableNames.LogsTable + "_payload" + } applied.InferenceTableConfig = &inferenceTable } return &applied } +// telemetrySupported mirrors the backend, which serves telemetry only for custom served +// models and rejects an endpoint that serves nothing ('NO_CONFIG') or only proxies +// external models ('EXTERNAL_MODELS'). +func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) { + if endpoint.Config == nil || len(endpoint.Config.ServedEntities) == 0 { + return "NO_CONFIG", false + } + for _, entity := range endpoint.Config.ServedEntities { + if entity.ExternalModel == nil { + return "", true + } + } + return "EXTERNAL_MODELS", false +} + // clearExternalModelSecrets mirrors the backend, which persists the *_plaintext // API keys as secrets and never returns them on GET. func clearExternalModelSecrets(em *serving.ExternalModel) *serving.ExternalModel { @@ -251,7 +276,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { PermissionLevel: serving.ServingEndpointDetailedPermissionLevelCanManage, RouteOptimized: createReq.RouteOptimized, Tags: createReq.Tags, - TelemetryConfig: applyTelemetryConfig(createReq.Name, nil, createReq.TelemetryConfig), + TelemetryConfig: applyTelemetryConfig(nil, createReq.TelemetryConfig), State: &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, Ready: serving.EndpointStateReadyNotReady, @@ -261,6 +286,12 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { ForceSendFields: append(createReq.ForceSendFields, "PermissionLevel", "RouteOptimized", "Description"), } + // Unlike the telemetry configuration API, create does not fail on an endpoint that + // cannot carry telemetry; it drops the field. + if _, ok := telemetrySupported(endpoint); !ok { + endpoint.TelemetryConfig = nil + } + s.ServingEndpoints[createReq.Name] = endpoint return Response{ @@ -416,8 +447,18 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } } + if endpointType, ok := telemetrySupported(endpoint); !ok { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": fmt.Sprintf("Telemetry configuration is not supported for endpoint type '%s'. This API only supports endpoints with custom served models.", endpointType), + }, + } + } + // An omitted telemetry_config removes the configuration from the endpoint. - endpoint.TelemetryConfig = applyTelemetryConfig(name, endpoint.TelemetryConfig, patchReq.TelemetryConfig) + endpoint.TelemetryConfig = applyTelemetryConfig(endpoint.TelemetryConfig, patchReq.TelemetryConfig) endpoint.LastUpdatedTimestamp = nowMilli() s.ServingEndpoints[name] = endpoint From b4d708b8cccd128188f43883ee570e469d167129 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 5 Aug 2026 12:20:33 +0000 Subject: [PATCH 12/28] Shorten the comments added by this change They had accumulated the reasoning behind each revision, which belongs in the commit messages. Each one now states the API behaviour the code depends on and stops there. --- ...odel-serving-telemetry-config-converges.md | 2 +- ...del-serving-telemetry-config-validation.md | 2 +- .../drift/telemetry_config/test.toml | 13 ++++----- .../telemetry_config_unmanaged/test.toml | 7 ++--- .../databricks.yml | 3 +- .../output.txt | 4 +-- .../databricks.yml | 3 +- .../output.txt | 2 +- .../validate_serving_telemetry_config.go | 24 ++++++---------- .../validate_serving_telemetry_config_test.go | 9 ++---- .../dresources/model_serving_endpoint.go | 7 ++--- bundle/direct/dresources/resources.yml | 22 ++++++--------- bundle/phases/initialize.go | 4 +-- libs/testserver/serving_endpoints.go | 28 +++++++------------ 14 files changed, 49 insertions(+), 81 deletions(-) diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index 9f8423adc7e..69fe957d034 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -direct: `telemetry_config` on model serving endpoints now converges, and edits to it are applied. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id` on GET, so every `bundle plan` after a deploy reported the endpoint as a perpetual update. Edits were planned but then silently dropped instead of being sent to the telemetry configuration API, telemetry the bundle does not declare is now left in place instead of being deleted on the next deploy, and a deploy that changes both `config` and `telemetry_config` now waits for the endpoint to settle instead of failing because the endpoint is still updating ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id` on GET, so every `bundle plan` after a deploy reported a perpetual update. Telemetry the bundle does not declare is now left in place instead of being deleted on the next deploy, and a deploy that changes both `config` and `telemetry_config` waits for the endpoint to settle instead of failing on the telemetry call ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md index 82c1fa39fa5..b787d1b303d 100644 --- a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md +++ b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md @@ -1 +1 @@ -`bundle validate` now rejects a `telemetry_config` on a model serving endpoint that the serving API cannot apply it to: one that serves nothing or only external models, and one that sets neither `table_names` nor `telemetry_profile_id`. Both forms used to be accepted and then silently discarded by the backend, so the bundle appeared to deploy while the endpoint received no telemetry. A bundle already deployed in either shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). +`bundle validate` now rejects a `telemetry_config` the serving API cannot apply: one on an endpoint that serves nothing or only external models, and one that sets neither `table_names` nor `telemetry_profile_id`. Both used to be accepted and then silently discarded by the backend, so the bundle appeared to deploy while the endpoint received no telemetry. A bundle already deployed in either shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml index c8350f18910..32b1abc44bb 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -1,12 +1,9 @@ RecordRequests = true -# Covers the direct engine's telemetry_config classification and update path. Terraform is -# excluded because the provider types sampling_fraction as an integer while the API takes a -# fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0 and drops it. +# Terraform is excluded: the provider types sampling_fraction as an integer while the API +# takes a fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0. # -# Local only: telemetry is supported only on an endpoint with a custom served model, which -# needs a registered UC model and roughly ten minutes to provision, and every telemetry -# PATCH then locks the endpoint for another two to three minutes. The fake's behaviour was -# recorded against a real workspace instead; see the comments in -# libs/testserver/serving_endpoints.go. +# Local only: telemetry needs an endpoint with a custom served model, which takes a +# registered UC model and ~10 minutes to provision, and each PATCH then locks it for +# another 2-3. The fake was checked against a real workspace instead. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml index 3402a8f8840..4f6d8de4369 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -1,7 +1,6 @@ RecordRequests = true -# Guards the backend_defaults rule at the telemetry_config parent path. Without it the -# remote-only config is planned as an update and DoUpdate sends a PATCH with no -# telemetry_config, deleting a configuration the bundle never managed. The classification -# under test belongs to the direct engine, so terraform is excluded. +# Guards the backend_defaults rule at the telemetry_config parent path: without it the +# remote-only config is planned as an update and DoUpdate PATCHes it away. Direct engine +# only, since that is where the classification lives. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml index c79e4b32f19..1f226dce244 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml @@ -10,8 +10,7 @@ resources: inference_table_config: sampling_fraction: 0.5 - # Typed EXTERNAL_MODELS by the API: it serves something, but telemetry is still - # only supported for custom served models. + # Typed EXTERNAL_MODELS by the API: it serves something, but not a custom model. endpoint2: name: "test-endpoint-2" config: diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt index b3a36dcd973..cc817b8ab62 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt @@ -6,7 +6,7 @@ The serving endpoints telemetry configuration API only supports endpoints with c Error: telemetry_config is only supported on an endpoint that serves a registered model at resources.model_serving_endpoints.endpoint2.telemetry_config - in databricks.yml:27:9 + in databricks.yml:26:9 The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. @@ -18,7 +18,7 @@ The serving endpoints API identifies a telemetry configuration by the Unity Cata Error: telemetry_config must set table_names or telemetry_profile_id at resources.model_serving_endpoints.endpoint2.telemetry_config - in databricks.yml:27:9 + in databricks.yml:26:9 The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml index c17b9b334d3..ac509878ced 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml @@ -3,8 +3,7 @@ bundle: resources: model_serving_endpoints: - # The endpoint supports telemetry, but an inference_table_config on its own names - # no profile, so the API discards it instead of applying the sampling fraction. + # Telemetry is supported here, but a config naming no profile is discarded. endpoint1: name: "test-endpoint" config: diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt index 31e0b42ac5e..113eb9af02f 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt @@ -1,6 +1,6 @@ Error: telemetry_config must set table_names or telemetry_profile_id at resources.model_serving_endpoints.endpoint1.telemetry_config - in databricks.yml:16:9 + in databricks.yml:15:9 The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go index dcb6c71841e..b4b881a95bb 100644 --- a/bundle/config/validate/validate_serving_telemetry_config.go +++ b/bundle/config/validate/validate_serving_telemetry_config.go @@ -38,12 +38,9 @@ func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bund }) } - // The telemetry configuration API only supports endpoints with custom served - // models: it rejects an endpoint that serves nothing ("Telemetry configuration is - // not supported for endpoint type 'NO_CONFIG'") and one that only proxies external - // models (the same error with 'EXTERNAL_MODELS'). Create drops telemetry_config - // instead of failing, so without this the bundle deploys once and then fails on - // every later deploy, when the plan finally applies the field through that API. + // The telemetry API rejects endpoints typed NO_CONFIG or EXTERNAL_MODELS, but create + // drops the field instead of failing, so without this the bundle deploys once and + // then fails on every later deploy. if !servesRegisteredModel(endpoint.Config) { addDiag( "telemetry_config is only supported on an endpoint that serves a registered model", @@ -51,11 +48,9 @@ func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bund ) } - // The API identifies a telemetry configuration by an existing profile or by the - // tables to create one from. Given neither, both create and the telemetry - // configuration API return success and apply nothing, so an inference_table_config - // on its own is silently discarded. Left unchecked the CLI would record it in state - // and report a perpetual update against an endpoint that never received it. + // Create and the telemetry API both report success and apply nothing when the + // configuration names no profile, so an inference_table_config on its own would land + // in state and show as a perpetual update against an endpoint that never got it. if !identifiesTelemetryProfile(endpoint.TelemetryConfig) { addDiag( "telemetry_config must set table_names or telemetry_profile_id", @@ -77,12 +72,9 @@ func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { // servesRegisteredModel reports whether the endpoint serves at least one registered // model, as opposed to nothing at all or only external models. A foundation model is -// indistinguishable from a custom model here (both are named by entity_name), so it is -// accepted; create rejects it outright with "Telemetry configuration is supported only -// for custom CPU or GPU models", which is already a clear failure at the first deploy. +// named the same way and is accepted here; create rejects it outright with a clear error. // -// ModelServingEndpointFixups has converted served_models to served_entities by the time -// validation runs, so only served_entities is inspected. +// ModelServingEndpointFixups has already folded served_models into served_entities. func servesRegisteredModel(config *serving.EndpointCoreConfigInput) bool { if config == nil { return false diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go index 72895b1a6b1..b4831d7d5b3 100644 --- a/bundle/config/validate/validate_serving_telemetry_config_test.go +++ b/bundle/config/validate/validate_serving_telemetry_config_test.go @@ -29,8 +29,7 @@ func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { } } -// telemetryConfig returns a configuration that names the tables to create a profile -// from, which is the form the API accepts. +// telemetryConfig names the tables to create a profile from, the form the API accepts. func telemetryConfig() *serving.TelemetryConfig { return &serving.TelemetryConfig{ TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}, @@ -93,8 +92,7 @@ func TestValidateServingTelemetryConfig(t *testing.T) { want: []string{unsupportedEndpoint}, }, { - // The API types this endpoint as EXTERNAL_MODELS and rejects telemetry on it, - // even though it does serve something. + // Rejected as EXTERNAL_MODELS, even though the endpoint does serve something. name: "telemetry with only an external model", endpoint: serving.CreateServingEndpoint{ Name: "my_endpoint", @@ -113,8 +111,7 @@ func TestValidateServingTelemetryConfig(t *testing.T) { want: []string{unsupportedEndpoint}, }, { - // The API discards a configuration that names no profile, so sampling on its - // own never takes effect. + // Discarded by the API, so the sampling fraction never takes effect. name: "telemetry with only an inference table config", endpoint: serving.CreateServingEndpoint{ Name: "my_endpoint", diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index f6a0d57417e..bc8772603d0 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -363,10 +363,9 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, } if entry.Changes.HasChange(pathTelemetryConfig) { - // Unlike the calls above, the telemetry configuration API refuses to run - // concurrently with an in-flight update ("Endpoint is currently being - // updated"), and each of those calls starts one. Wait for the endpoint to settle - // first; WaitAfterUpdate only runs once DoUpdate has returned. + // The telemetry API rejects an endpoint that is still applying an earlier update, + // and every call above starts one. WaitAfterUpdate runs only after DoUpdate + // returns, so settle here. if updated { _, err = r.client.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, id, 35*time.Minute, nil) if err != nil { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 3d0c4fa87dc..386e609493c 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,11 +285,9 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # The backend creates a telemetry profile from these tables and reports back the - # resulting telemetry_profile_id instead; the table names themselves are never - # echoed, so remote reads back empty and a config that sets them never converges. - # inference_table_config is deliberately not listed: sampling_fraction does round - # trip, so it is subject to normal drift detection. + # The backend reports back only the telemetry_profile_id it creates from these tables, + # never the tables themselves, so remote reads back empty. inference_table_config is + # deliberately not listed: sampling_fraction does round trip. - field: telemetry_config.table_names reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -323,16 +321,12 @@ resources: - field: rate_limits reason: not_implemented backend_defaults: - # The backend creates a telemetry profile from table_names and echoes its ID back on - # GET. Users may also pin an existing profile, so drift still applies when - # telemetry_profile_id is set locally. + # The backend assigns telemetry_profile_id when it creates the profile from table_names. # - # The rule covers the whole subtree rather than just telemetry_profile_id: when - # telemetry_config is absent locally, structdiff reports the populated remote as one - # change on the parent path, which no child rule matches. Without this the change - # becomes an update and DoUpdate sends a PATCH with no telemetry_config, removing a - # configuration the bundle never managed. A genuine removal is unaffected because it - # has a non-nil old value. + # The rule covers the whole subtree, not just that field: with no local + # telemetry_config, structdiff reports the remote as one change on the parent path, + # which no child rule matches, and DoUpdate would then PATCH it away. A genuine + # removal still applies, since it carries a non-nil old value. - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b91e2d68689..b5eec1d25d4 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,8 +185,8 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), - // Validate that telemetry_config is only set on serving endpoints that serve a model. - // The API rejects it on any other endpoint, and only once the plan applies it. + // Validate that telemetry_config is in a form the serving endpoints API applies. + // It rejects or silently discards the rest, and only once the plan applies it. validate.ValidateServingTelemetryConfig(), // Reads (dynamic): * (strings) (searches for ${resources.*} references) diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 30857b5a974..05c494893a0 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,14 +35,10 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend: a telemetry configuration is identified by an -// existing telemetry_profile_id or by the table_names to create a profile from. Given -// neither, the API returns 200 and applies nothing, so an inference_table_config on its -// own leaves the endpoint's telemetry untouched rather than setting or clearing it. -// -// What is echoed back is the profile ID plus, when the caller supplied one, the inference -// table config carrying the table name the backend derives from logs_table. table_names -// itself is never returned. +// applyTelemetryConfig mirrors the backend: a configuration is identified by an existing +// telemetry_profile_id or by the table_names to create a profile from, and one naming +// neither returns 200 and applies nothing. What comes back is the profile ID, plus +// inference_table_config if the caller supplied one; never table_names. func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil @@ -53,15 +49,13 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te applied := serving.TelemetryConfig{TelemetryProfileId: config.TelemetryProfileId} if applied.TelemetryProfileId == "" { - // New tables mean a new profile, so this does not carry over the endpoint's - // current profile ID. + // New tables mean a new profile, not the endpoint's current one. applied.TelemetryProfileId = nextUUID() } if config.InferenceTableConfig != nil { inferenceTable := *config.InferenceTableConfig - // Only the table_names form was observed against a real workspace. A patch that - // reuses a profile by ID does not name the tables, and what the backend reports as - // the inference table name in that case is unverified, so leave it empty. + // Left empty for a patch that reuses a profile by ID: it does not name the tables, + // and what the backend reports as the name there was never observed. if config.TableNames != nil { inferenceTable.Name = config.TableNames.LogsTable + "_payload" } @@ -71,9 +65,8 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te return &applied } -// telemetrySupported mirrors the backend, which serves telemetry only for custom served -// models and rejects an endpoint that serves nothing ('NO_CONFIG') or only proxies -// external models ('EXTERNAL_MODELS'). +// telemetrySupported mirrors the backend, which supports telemetry only on custom served +// models, and returns the endpoint type the backend names when it rejects one. func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) { if endpoint.Config == nil || len(endpoint.Config.ServedEntities) == 0 { return "NO_CONFIG", false @@ -286,8 +279,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { ForceSendFields: append(createReq.ForceSendFields, "PermissionLevel", "RouteOptimized", "Description"), } - // Unlike the telemetry configuration API, create does not fail on an endpoint that - // cannot carry telemetry; it drops the field. + // Unlike the telemetry API, create drops telemetry it cannot apply instead of failing. if _, ok := telemetrySupported(endpoint); !ok { endpoint.TelemetryConfig = nil } From ce2a3b9633968269e8946d55d2ec2a5e560cc3ac Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 09:23:48 +0000 Subject: [PATCH 13/28] Cover the telemetry wait and reject a telemetry_config naming no tables Nothing exercised the wait before patching telemetry, and nothing could: the fake always reported NOT_UPDATING, so a test that reached that branch would have passed with the wait deleted. The fake now models the interaction the wait exists for. A config update leaves the endpoint IN_PROGRESS, the telemetry API rejects an endpoint in that state with a 409, and GET reports the update once before settling it, so a caller that polls converges and one that does not still sees it in flight. The new drift/telemetry_config_with_config_update test changes config and telemetry_config in one deploy and records the resulting PUT, poll and PATCH. Deleting the wait fails it. Validation accepted an empty table_names, which names no tables and so identifies a profile no better than omitting the field. It now requires one of the four table names, and the error says "name a table in table_names" rather than "set table_names", which was misleading for that case. telemetrySupported also disagreed with the validator on what counts as a custom served model: it accepted an entity with no external model but no entity_name either. Unreachable while validation runs first, but it left the fake claiming to mirror a rule it did not. --- ...del-serving-telemetry-config-validation.md | 2 +- .../databricks.yml | 19 ++++ .../out.test.toml | 3 + .../output.txt | 101 ++++++++++++++++++ .../script | 18 ++++ .../test.toml | 6 ++ .../output.txt | 4 +- .../output.txt | 2 +- .../validate_serving_telemetry_config.go | 26 ++--- .../validate_serving_telemetry_config_test.go | 24 ++++- libs/testserver/handlers.go | 2 +- libs/testserver/serving_endpoints.go | 60 +++++++++-- 12 files changed, 241 insertions(+), 26 deletions(-) create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml diff --git a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md index b787d1b303d..e7bc1847bc9 100644 --- a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md +++ b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md @@ -1 +1 @@ -`bundle validate` now rejects a `telemetry_config` the serving API cannot apply: one on an endpoint that serves nothing or only external models, and one that sets neither `table_names` nor `telemetry_profile_id`. Both used to be accepted and then silently discarded by the backend, so the bundle appeared to deploy while the endpoint received no telemetry. A bundle already deployed in either shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). +`bundle validate` now rejects a `telemetry_config` the serving API cannot apply: one on an endpoint that serves nothing or only external models, and one that names neither a table in `table_names` nor a `telemetry_profile_id`. Both used to be accepted and then silently discarded by the backend, so the bundle appeared to deploy while the endpoint received no telemetry. A bundle already deployed in either shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml new file mode 100644 index 00000000000..0ab171df466 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + endpoint1: + name: test-endpoint + config: + served_entities: + - name: prod + entity_name: main.default.test_model + entity_version: "1" + workload_size: Small + scale_to_zero_enabled: true + telemetry_config: + table_names: + logs_table: main.default.test_logs + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt new file mode 100644 index 00000000000..e75cabff52c --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -0,0 +1,101 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Change config and telemetry_config in the same deploy +>>> [CLI] bundle plan --output json +[ + "config.served_entities[0].workload_size", + "config.traffic_config", + "description", + "route_optimized", + "telemetry_config.inference_table_config.name", + "telemetry_config.inference_table_config.sampling_fraction", + "telemetry_config.table_names", + "telemetry_config.telemetry_profile_id" +] + +=== Deploy waits for the config update to settle before patching telemetry +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan converges +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== The config update is followed by a poll for NOT_UPDATING, then the telemetry patch +>>> print_requests.py --get --unique //serving-endpoints +{ + "method": "POST", + "path": "/api/2.0/serving-endpoints", + "body": { + "config": { + "served_entities": [ + { + "entity_name": "main.default.test_model", + "entity_version": "1", + "name": "prod", + "scale_to_zero_enabled": true, + "workload_size": "Small" + } + ] + }, + "name": "test-endpoint", + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.5 + }, + "table_names": { + "logs_table": "main.default.test_logs" + } + } + } +} +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} +{ + "method": "PUT", + "path": "/api/2.0/serving-endpoints/test-endpoint/config", + "body": { + "served_entities": [ + { + "entity_name": "main.default.test_model", + "entity_version": "1", + "name": "prod", + "scale_to_zero_enabled": true, + "workload_size": "Medium" + } + ] + } +} +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} +{ + "method": "PATCH", + "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", + "body": { + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.9 + }, + "table_names": { + "logs_table": "main.default.test_logs" + } + } + } +} +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script new file mode 100644 index 00000000000..2a3356d85cc --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Change config and telemetry_config in the same deploy" +update_file.py databricks.yml "workload_size: Small" "workload_size: Medium" +update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" +trace $CLI bundle plan --output json | jq '.plan[].changes | keys' + +title "Deploy waits for the config update to settle before patching telemetry" +trace $CLI bundle deploy + +title "Plan converges" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "The config update is followed by a poll for NOT_UPDATING, then the telemetry patch" +trace print_requests.py --get --unique //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml new file mode 100644 index 00000000000..f7acf50fa75 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml @@ -0,0 +1,6 @@ +RecordRequests = true + +# Guards the wait in DoUpdate: the telemetry API rejects an endpoint still applying an +# earlier update, so without the wait this deploy fails on the telemetry patch with a 409. +# Direct engine only, for the same reasons as drift/telemetry_config. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt index cc817b8ab62..6cef47eeff9 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt @@ -10,13 +10,13 @@ Error: telemetry_config is only supported on an endpoint that serves a registere The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. -Error: telemetry_config must set table_names or telemetry_profile_id +Error: telemetry_config must name a table in table_names or set telemetry_profile_id at resources.model_serving_endpoints.endpoint1.telemetry_config in databricks.yml:10:9 The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. -Error: telemetry_config must set table_names or telemetry_profile_id +Error: telemetry_config must name a table in table_names or set telemetry_profile_id at resources.model_serving_endpoints.endpoint2.telemetry_config in databricks.yml:26:9 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt index 113eb9af02f..5ca6943ebb5 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt @@ -1,4 +1,4 @@ -Error: telemetry_config must set table_names or telemetry_profile_id +Error: telemetry_config must name a table in table_names or set telemetry_profile_id at resources.model_serving_endpoints.endpoint1.telemetry_config in databricks.yml:15:9 diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go index b4b881a95bb..b76b810acea 100644 --- a/bundle/config/validate/validate_serving_telemetry_config.go +++ b/bundle/config/validate/validate_serving_telemetry_config.go @@ -39,8 +39,7 @@ func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bund } // The telemetry API rejects endpoints typed NO_CONFIG or EXTERNAL_MODELS, but create - // drops the field instead of failing, so without this the bundle deploys once and - // then fails on every later deploy. + // silently drops the field, so without this the bundle deploys once and fails after. if !servesRegisteredModel(endpoint.Config) { addDiag( "telemetry_config is only supported on an endpoint that serves a registered model", @@ -48,12 +47,11 @@ func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bund ) } - // Create and the telemetry API both report success and apply nothing when the - // configuration names no profile, so an inference_table_config on its own would land - // in state and show as a perpetual update against an endpoint that never got it. + // Create and the telemetry API both report success and apply nothing when the config + // names no profile, so an inference_table_config alone would drift forever. if !identifiesTelemetryProfile(endpoint.TelemetryConfig) { addDiag( - "telemetry_config must set table_names or telemetry_profile_id", + "telemetry_config must name a table in table_names or set telemetry_profile_id", "The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it.", ) } @@ -64,15 +62,19 @@ func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bund return diags } -// identifiesTelemetryProfile reports whether the configuration names the telemetry -// profile to use or the tables to create one from. +// identifiesTelemetryProfile reports whether the configuration names an existing profile +// or the tables to create one from. An empty table_names names neither. func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { - return config.TableNames != nil || config.TelemetryProfileId != "" + if config.TelemetryProfileId != "" { + return true + } + t := config.TableNames + return t != nil && (t.LogsTable != "" || t.TracesTable != "" || t.MetricsTable != "" || t.AnnotationsTable != "") } -// servesRegisteredModel reports whether the endpoint serves at least one registered -// model, as opposed to nothing at all or only external models. A foundation model is -// named the same way and is accepted here; create rejects it outright with a clear error. +// servesRegisteredModel reports whether the endpoint serves at least one registered model, +// as opposed to nothing or only external models. A foundation model is named the same way +// and is accepted here; create rejects it outright. // // ModelServingEndpointFixups has already folded served_models into served_entities. func servesRegisteredModel(config *serving.EndpointCoreConfigInput) bool { diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go index b4831d7d5b3..48ee3a774e4 100644 --- a/bundle/config/validate/validate_serving_telemetry_config_test.go +++ b/bundle/config/validate/validate_serving_telemetry_config_test.go @@ -14,7 +14,7 @@ import ( const ( unsupportedEndpoint = "telemetry_config is only supported on an endpoint that serves a registered model" - noProfile = "telemetry_config must set table_names or telemetry_profile_id" + noProfile = "telemetry_config must name a table in table_names or set telemetry_profile_id" ) func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { @@ -110,6 +110,28 @@ func TestValidateServingTelemetryConfig(t *testing.T) { }, want: []string{unsupportedEndpoint}, }, + { + // Names no tables, so nothing identifies the profile. + name: "telemetry with an empty table_names", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + TableNames: &serving.UnityCatalogTableNames{}, + }, + Config: servedModelConfig(), + }, + want: []string{noProfile}, + }, + { + name: "telemetry with only a traces table", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + TableNames: &serving.UnityCatalogTableNames{TracesTable: "main.default.traces"}, + }, + Config: servedModelConfig(), + }, + }, { // Discarded by the API, so the sampling fraction never takes effect. name: "telemetry with only an inference table config", diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index db4e0697791..4a5fc912350 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -957,7 +957,7 @@ func AddDefaultHandlers(server *Server) { // Serving Endpoints: server.Handle("GET", "/api/2.0/serving-endpoints/{name}", func(req Request) any { - return MapGet(req.Workspace, req.Workspace.ServingEndpoints, req.Vars["name"]) + return req.Workspace.ServingEndpointGet(req.Vars["name"]) }) server.Handle("POST", "/api/2.0/serving-endpoints", func(req Request) any { diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 05c494893a0..1f02284fc93 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,10 +35,9 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend: a configuration is identified by an existing -// telemetry_profile_id or by the table_names to create a profile from, and one naming -// neither returns 200 and applies nothing. What comes back is the profile ID, plus -// inference_table_config if the caller supplied one; never table_names. +// applyTelemetryConfig mirrors the backend: a config naming neither telemetry_profile_id +// nor table_names returns 200 and applies nothing. What comes back is the profile ID plus +// inference_table_config if the caller supplied one, never table_names. func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil @@ -54,8 +53,8 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te } if config.InferenceTableConfig != nil { inferenceTable := *config.InferenceTableConfig - // Left empty for a patch that reuses a profile by ID: it does not name the tables, - // and what the backend reports as the name there was never observed. + // Left empty when the patch reuses a profile by ID: it names no tables, and what + // the backend reports there was never observed. if config.TableNames != nil { inferenceTable.Name = config.TableNames.LogsTable + "_payload" } @@ -72,7 +71,7 @@ func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) return "NO_CONFIG", false } for _, entity := range endpoint.Config.ServedEntities { - if entity.ExternalModel == nil { + if entity.ExternalModel == nil && entity.EntityName != "" { return "", true } } @@ -291,6 +290,38 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { } } +// ServingEndpointGet reports an in-progress config update once and then settles the +// endpoint, so a caller that polls converges and one that does not sees it in flight. +func (s *FakeWorkspace) ServingEndpointGet(name string) Response { + defer s.LockUnlock()() + + endpoint, exists := s.ServingEndpoints[name] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]string{"message": fmt.Sprintf("Resource %T not found: %v", endpoint, name)}, + } + } + + if endpointUpdating(endpoint) { + // Settle the stored copy for the next read. The response keeps the in-progress + // State, since reassigning settled.State leaves endpoint.State pointing at it. + settled := endpoint + settled.State = &serving.EndpointState{ + ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, + Ready: endpoint.State.Ready, + } + s.ServingEndpoints[name] = settled + } + + return Response{Body: endpoint} +} + +// endpointUpdating reports whether the endpoint is still applying a config update. +func endpointUpdating(endpoint serving.ServingEndpointDetailed) bool { + return endpoint.State != nil && endpoint.State.ConfigUpdate == serving.EndpointStateConfigUpdateInProgress +} + func (s *FakeWorkspace) ServingEndpointUpdate(req Request, name string) Response { defer s.LockUnlock()() @@ -338,8 +369,10 @@ func (s *FakeWorkspace) ServingEndpointUpdate(req Request, name string) Response endpoint.Config = config endpoint.LastUpdatedTimestamp = nowMilli() + // A config update is asynchronous: until it settles the endpoint reports + // IN_PROGRESS, which is what makes a telemetry patch in the same pass fail. endpoint.State = &serving.EndpointState{ - ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, + ConfigUpdate: serving.EndpointStateConfigUpdateInProgress, Ready: serving.EndpointStateReadyNotReady, } @@ -449,6 +482,17 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } } + // The telemetry API refuses to run while an earlier update is still applying. + if endpointUpdating(endpoint) { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "error_code": "RESOURCE_CONFLICT", + "message": fmt.Sprintf("Endpoint %s is currently updating. Wait for the update to complete before updating its telemetry configuration.", name), + }, + } + } + // An omitted telemetry_config removes the configuration from the endpoint. endpoint.TelemetryConfig = applyTelemetryConfig(endpoint.TelemetryConfig, patchReq.TelemetryConfig) endpoint.LastUpdatedTimestamp = nowMilli() From e11ddb785503eb331ecbbcb30d2ffc6e4220b883 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 09:24:02 +0000 Subject: [PATCH 14/28] Shorten the remaining comments added by this change Several still carried the reasoning behind the revision that introduced them. Each now states the API behaviour or the decision the code depends on and stops there. --- .../drift/telemetry_config/test.toml | 6 +++--- .../drift/telemetry_config_unmanaged/test.toml | 4 ++-- bundle/direct/dresources/model_serving_endpoint.go | 5 ++--- bundle/direct/dresources/resources.yml | 13 +++++-------- bundle/phases/initialize.go | 2 +- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml index 32b1abc44bb..8431f2a3e8d 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -3,7 +3,7 @@ RecordRequests = true # Terraform is excluded: the provider types sampling_fraction as an integer while the API # takes a fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0. # -# Local only: telemetry needs an endpoint with a custom served model, which takes a -# registered UC model and ~10 minutes to provision, and each PATCH then locks it for -# another 2-3. The fake was checked against a real workspace instead. +# Local only: telemetry needs a custom served model, which takes a registered UC model and +# ~10 minutes to provision, and each PATCH locks the endpoint for another 2-3. The fake was +# checked against a real workspace instead. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml index 4f6d8de4369..809168409a5 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -1,6 +1,6 @@ RecordRequests = true # Guards the backend_defaults rule at the telemetry_config parent path: without it the -# remote-only config is planned as an update and DoUpdate PATCHes it away. Direct engine -# only, since that is where the classification lives. +# remote-only config is planned as an update and DoUpdate PATCHes it away. +# Direct engine only, since that is where the classification lives. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index bc8772603d0..eb05030318d 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -363,9 +363,8 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, } if entry.Changes.HasChange(pathTelemetryConfig) { - // The telemetry API rejects an endpoint that is still applying an earlier update, - // and every call above starts one. WaitAfterUpdate runs only after DoUpdate - // returns, so settle here. + // The telemetry API rejects an endpoint still applying an earlier update, and every + // call above starts one. WaitAfterUpdate only runs after DoUpdate returns. if updated { _, err = r.client.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, id, 35*time.Minute, nil) if err != nil { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 386e609493c..747253c4c5a 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,9 +285,8 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # The backend reports back only the telemetry_profile_id it creates from these tables, - # never the tables themselves, so remote reads back empty. inference_table_config is - # deliberately not listed: sampling_fraction does round trip. + # The backend returns only the telemetry_profile_id it creates from these tables, never + # the tables themselves. inference_table_config is not listed: sampling_fraction round trips. - field: telemetry_config.table_names reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -322,11 +321,9 @@ resources: reason: not_implemented backend_defaults: # The backend assigns telemetry_profile_id when it creates the profile from table_names. - # - # The rule covers the whole subtree, not just that field: with no local - # telemetry_config, structdiff reports the remote as one change on the parent path, - # which no child rule matches, and DoUpdate would then PATCH it away. A genuine - # removal still applies, since it carries a non-nil old value. + # Declared on the subtree, not that field alone: with no local telemetry_config, structdiff + # reports the remote as one change on the parent path, which no child rule matches, and + # DoUpdate would PATCH it away. A genuine removal carries a non-nil old value and still applies. - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b5eec1d25d4..4f06e3ca7db 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -186,7 +186,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.ValidateDeploymentFields(), // Validate that telemetry_config is in a form the serving endpoints API applies. - // It rejects or silently discards the rest, and only once the plan applies it. + // It rejects or silently discards anything else. validate.ValidateServingTelemetryConfig(), // Reads (dynamic): * (strings) (searches for ${resources.*} references) From 2fbbfa1662ba79f39e39002592cf3deda309e72d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 12:28:32 +0000 Subject: [PATCH 15/28] Wait unconditionally before patching telemetry_config The updated flag only tracked calls made in the same DoUpdate pass, so a deploy whose only change was telemetry_config skipped the wait and failed with a 409 when the endpoint was already applying an update from an out-of-band change or an interrupted deploy. Waiting always costs one GET on a settled endpoint, and WaitAfterUpdate already waits with the same timeout once DoUpdate returns, so this adds no new way to hang. It also drops a flag every future update call would have had to remember to set. The changelog claimed two fixes that were never broken on main: nothing deleted telemetry the bundle did not declare, and nothing failed a deploy changing both config and telemetry_config, because DoUpdate made no telemetry call at all. Both were introduced and repaired inside this branch, so they are dropped rather than announced as user-visible fixes. model_serving_telemetry_unsupported_endpoint tripped the profile-naming rule as well as the endpoint-type rule it is named for; its fixtures now name a table so the golden shows only the rejection under test. The request dump in telemetry_config_with_config_update drops --unique, which collapsed the poll between the config PUT and the telemetry PATCH that the step claims to show. --- ...model-serving-telemetry-config-converges.md | 2 +- .../output.txt | 18 +++++++++++++++++- .../telemetry_config_with_config_update/script | 2 +- .../databricks.yml | 8 ++++---- .../output.txt | 14 +------------- .../validate_serving_telemetry_config.go | 5 ++--- .../dresources/model_serving_endpoint.go | 18 ++++++------------ bundle/direct/dresources/resources.yml | 6 +++--- libs/testserver/serving_endpoints.go | 3 +-- 9 files changed, 36 insertions(+), 40 deletions(-) diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index 69fe957d034..e6f53d71f16 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id` on GET, so every `bundle plan` after a deploy reported a perpetual update. Telemetry the bundle does not declare is now left in place instead of being deleted on the next deploy, and a deploy that changes both `config` and `telemetry_config` waits for the endpoint to settle instead of failing on the telemetry call ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id` on GET, so every `bundle plan` after a deploy reported an update, and a `bundle deploy` that planned one applied nothing ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt index e75cabff52c..b60d31a3711 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -31,7 +31,7 @@ Deployment complete! Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The config update is followed by a poll for NOT_UPDATING, then the telemetry patch ->>> print_requests.py --get --unique //serving-endpoints +>>> print_requests.py --get //serving-endpoints { "method": "POST", "path": "/api/2.0/serving-endpoints", @@ -62,6 +62,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint" } +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} { "method": "PUT", "path": "/api/2.0/serving-endpoints/test-endpoint/config", @@ -81,6 +89,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint" } +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} { "method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", @@ -99,3 +111,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint" } +{ + "method": "GET", + "path": "/api/2.0/serving-endpoints/test-endpoint" +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script index 2a3356d85cc..abb93db4778 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script @@ -15,4 +15,4 @@ title "Plan converges" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The config update is followed by a poll for NOT_UPDATING, then the telemetry patch" -trace print_requests.py --get --unique //serving-endpoints +trace print_requests.py --get //serving-endpoints diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml index 1f226dce244..7b8ecd8426a 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml @@ -7,8 +7,8 @@ resources: endpoint1: name: "test-endpoint-1" telemetry_config: - inference_table_config: - sampling_fraction: 0.5 + table_names: + logs_table: main.default.test_logs # Typed EXTERNAL_MODELS by the API: it serves something, but not a custom model. endpoint2: @@ -23,5 +23,5 @@ resources: openai_config: openai_api_key_plaintext: sk-test-plaintext-key telemetry_config: - inference_table_config: - sampling_fraction: 0.5 + table_names: + logs_table: main.default.test_logs diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt index 6cef47eeff9..6993adea0b2 100644 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt +++ b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt @@ -10,24 +10,12 @@ Error: telemetry_config is only supported on an endpoint that serves a registere The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. -Error: telemetry_config must name a table in table_names or set telemetry_profile_id - at resources.model_serving_endpoints.endpoint1.telemetry_config - in databricks.yml:10:9 - -The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. - -Error: telemetry_config must name a table in table_names or set telemetry_profile_id - at resources.model_serving_endpoints.endpoint2.telemetry_config - in databricks.yml:26:9 - -The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. - Name: model_serving_telemetry Target: default Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default -Found 4 errors +Found 2 errors Exit code: 1 diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go index b76b810acea..84f1462ebbb 100644 --- a/bundle/config/validate/validate_serving_telemetry_config.go +++ b/bundle/config/validate/validate_serving_telemetry_config.go @@ -72,9 +72,8 @@ func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { return t != nil && (t.LogsTable != "" || t.TracesTable != "" || t.MetricsTable != "" || t.AnnotationsTable != "") } -// servesRegisteredModel reports whether the endpoint serves at least one registered model, -// as opposed to nothing or only external models. A foundation model is named the same way -// and is accepted here; create rejects it outright. +// servesRegisteredModel reports whether the endpoint serves at least one registered model. +// A foundation model is named the same way and is accepted here; create rejects it outright. // // ModelServingEndpointFixups has already folded served_models into served_entities. func servesRegisteredModel(config *serving.EndpointCoreConfigInput) bool { diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index eb05030318d..e87daf873b6 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -325,7 +325,6 @@ func (r *ResourceModelServingEndpoint) updateTags(ctx context.Context, id string func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, config *serving.CreateServingEndpoint, entry *PlanEntry) (*ModelServingEndpointRemote, error) { var err error - updated := false // Terraform makes these API calls sequentially. We do the same here. // It's an unknown as of 1st Dec 2025 if these APIs are safe to make in parallel. (we did not check) @@ -335,7 +334,6 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } - updated = true } if entry.Changes.HasChange(pathAiGateway) { @@ -343,7 +341,6 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } - updated = true } if entry.Changes.HasChange(pathConfig) { @@ -351,7 +348,6 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } - updated = true } if entry.Changes.HasChange(pathEmailNotifications) { @@ -359,17 +355,15 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if err != nil { return nil, err } - updated = true } if entry.Changes.HasChange(pathTelemetryConfig) { - // The telemetry API rejects an endpoint still applying an earlier update, and every - // call above starts one. WaitAfterUpdate only runs after DoUpdate returns. - if updated { - _, err = r.client.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, id, 35*time.Minute, nil) - if err != nil { - return nil, err - } + // The telemetry API rejects an endpoint that is still applying an update, which the + // calls above start and an interrupted deploy can leave behind. WaitAfterUpdate only + // runs after DoUpdate returns. + _, err = r.client.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, id, 35*time.Minute, nil) + if err != nil { + return nil, err } err = r.updateTelemetryConfig(ctx, id, config.TelemetryConfig) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 747253c4c5a..d0f22464355 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -321,9 +321,9 @@ resources: reason: not_implemented backend_defaults: # The backend assigns telemetry_profile_id when it creates the profile from table_names. - # Declared on the subtree, not that field alone: with no local telemetry_config, structdiff - # reports the remote as one change on the parent path, which no child rule matches, and - # DoUpdate would PATCH it away. A genuine removal carries a non-nil old value and still applies. + # Declared on the subtree: structdiff reports an undeclared telemetry_config as one change + # on the parent path, which no child rule matches, so DoUpdate would PATCH it away. + # Removal still applies: it carries a non-nil old value, which this rule requires to be nil. - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 1f02284fc93..e4d3345a9fe 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -304,8 +304,7 @@ func (s *FakeWorkspace) ServingEndpointGet(name string) Response { } if endpointUpdating(endpoint) { - // Settle the stored copy for the next read. The response keeps the in-progress - // State, since reassigning settled.State leaves endpoint.State pointing at it. + // Settle the stored copy for the next read; this response still reports IN_PROGRESS. settled := endpoint settled.State = &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, From fce68deb7036ba47d7b74b9e2d5c0c33b0a23cc8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 13:01:28 +0000 Subject: [PATCH 16/28] Narrow this change to the telemetry_config drift fix The bundle validate rules rejecting a telemetry_config the serving API cannot apply are a user-visible behavior change riding along with a bug fix, and only one of them relates to that fix at all: the endpoint-type rule covered a regression this branch introduces, while the profile-naming rule addresses a drift that predates it. Both are dropped, so a bundle in either shape now gets the serving API's own 400 at deploy time instead. telemetrySupported and the create branch that dropped telemetry from an unsupported endpoint existed only to back those rules, and no test reached either, so they go with them. The rest is tightening. DoUpdate reuses waitForEndpointReady rather than repeating its timeout, the removal step in drift/telemetry_config uses the grep -v sentinel the repo already uses instead of an escaped multi-line replacement, and that fixture drops to two telemetry tables now that nothing depends on there being four. --- ...odel-serving-telemetry-config-converges.md | 2 +- ...del-serving-telemetry-config-validation.md | 1 - .../drift/telemetry_config/databricks.yml | 14 +- .../drift/telemetry_config/output.txt | 16 -- .../drift/telemetry_config/script | 4 +- .../drift/telemetry_config/test.toml | 8 +- .../drift/telemetry_config_unmanaged/script | 2 - .../telemetry_config_unmanaged/test.toml | 6 +- .../test.toml | 6 +- .../databricks.yml | 27 --- .../out.test.toml | 3 - .../output.txt | 21 --- .../script | 1 - .../databricks.yml | 16 -- .../out.test.toml | 3 - .../output.txt | 15 -- .../script | 1 - .../validate_serving_telemetry_config.go | 91 ---------- .../validate_serving_telemetry_config_test.go | 171 ------------------ .../dresources/model_serving_endpoint.go | 4 +- bundle/direct/dresources/resources.yml | 11 +- bundle/phases/initialize.go | 4 - libs/testserver/serving_endpoints.go | 39 +--- 23 files changed, 28 insertions(+), 438 deletions(-) delete mode 100644 .nextchanges/notable-changes/model-serving-telemetry-config-validation.md delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/out.test.toml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/script delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/script delete mode 100644 bundle/config/validate/validate_serving_telemetry_config.go delete mode 100644 bundle/config/validate/validate_serving_telemetry_config_test.go diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index e6f53d71f16..af5dfea897c 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id` on GET, so every `bundle plan` after a deploy reported an update, and a `bundle deploy` that planned one applied nothing ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id`, so every `bundle plan` after a deploy reported an update, and the deploy that followed made no telemetry call at all ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md deleted file mode 100644 index e7bc1847bc9..00000000000 --- a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md +++ /dev/null @@ -1 +0,0 @@ -`bundle validate` now rejects a `telemetry_config` the serving API cannot apply: one on an endpoint that serves nothing or only external models, and one that names neither a table in `table_names` nor a `telemetry_profile_id`. Both used to be accepted and then silently discarded by the backend, so the bundle appeared to deploy while the endpoint received no telemetry. A bundle already deployed in either shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml index a5a41597efb..46818f14591 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -12,11 +12,9 @@ resources: entity_version: "1" workload_size: Small scale_to_zero_enabled: true - telemetry_config: - table_names: - logs_table: main.default.test_logs - traces_table: main.default.test_traces - metrics_table: main.default.test_metrics - annotations_table: main.default.test_annotations - inference_table_config: - sampling_fraction: 0.5 + telemetry_config: # TO_REMOVE + table_names: # TO_REMOVE + logs_table: main.default.test_logs # TO_REMOVE + traces_table: main.default.test_traces # TO_REMOVE + inference_table_config: # TO_REMOVE + sampling_fraction: 0.5 # TO_REMOVE diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index 866094077d0..6d1336e7d6a 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -51,15 +51,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "action": "skip", "reason": "input_only", "old": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" }, "new": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" } }, @@ -115,15 +111,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "telemetry_config.table_names": { "action": "update", "old": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" }, "new": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs_v2", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" } }, @@ -182,9 +174,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "sampling_fraction": 0.9 }, "table_names": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs_v2", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" } }, @@ -233,9 +223,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "sampling_fraction": 0.5 }, "table_names": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" } } @@ -250,9 +238,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "sampling_fraction": 0.9 }, "table_names": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" } } @@ -267,9 +253,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "sampling_fraction": 0.9 }, "table_names": { - "annotations_table": "main.default.test_annotations", "logs_table": "main.default.test_logs_v2", - "metrics_table": "main.default.test_metrics", "traces_table": "main.default.test_traces" } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index b13d7322cec..5d55d346a3b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -25,11 +25,9 @@ title "Plan converges after the tables changed" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "Removing telemetry_config from the bundle removes it remotely" -update_file.py databricks.yml $' telemetry_config:\n table_names:\n logs_table: main.default.test_logs_v2\n traces_table: main.default.test_traces\n metrics_table: main.default.test_metrics\n annotations_table: main.default.test_annotations\n inference_table_config:\n sampling_fraction: 0.9\n' '' +grep -v TO_REMOVE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle plan --output json | jq '.plan[].changes' trace $CLI bundle deploy -# MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash -# API path to C:\Program Files\Git\api\..., which misses the testserver stub. MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config title "Plan converges with telemetry_config removed" diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml index 8431f2a3e8d..0d20ee379d2 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -1,9 +1,5 @@ RecordRequests = true -# Terraform is excluded: the provider types sampling_fraction as an integer while the API -# takes a fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0. -# -# Local only: telemetry needs a custom served model, which takes a registered UC model and -# ~10 minutes to provision, and each PATCH locks the endpoint for another 2-3. The fake was -# checked against a real workspace instead. +# Direct engine only: the terraform provider types sampling_fraction as an integer while +# the API takes a fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index c14a95944de..e80552e8a37 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -4,8 +4,6 @@ title "Deploy an endpoint that does not declare telemetry_config" trace $CLI bundle deploy title "Telemetry gets configured out of band, as on an adopted endpoint" -# MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash -# API path to C:\Program Files\Git\api\..., which misses the testserver stub. MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "A remote-only telemetry_config is a backend default, not drift" diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml index 809168409a5..aaf5e57b677 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -1,6 +1,6 @@ RecordRequests = true -# Guards the backend_defaults rule at the telemetry_config parent path: without it the -# remote-only config is planned as an update and DoUpdate PATCHes it away. -# Direct engine only, since that is where the classification lives. +# Direct engine only, where the classification lives. Guards the backend_defaults rule on +# the telemetry_config subtree: without it the remote-only config is planned as an update +# and DoUpdate patches it away. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml index f7acf50fa75..a89d9b03a11 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml @@ -1,6 +1,6 @@ RecordRequests = true -# Guards the wait in DoUpdate: the telemetry API rejects an endpoint still applying an -# earlier update, so without the wait this deploy fails on the telemetry patch with a 409. -# Direct engine only, for the same reasons as drift/telemetry_config. +# Direct engine only, as with the sibling drift tests. Guards the wait in DoUpdate: the +# telemetry API rejects an endpoint still applying an earlier update, so without the wait +# this deploy fails on the telemetry patch with a 409. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml deleted file mode 100644 index 7b8ecd8426a..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/databricks.yml +++ /dev/null @@ -1,27 +0,0 @@ -bundle: - name: "model_serving_telemetry" - -resources: - model_serving_endpoints: - # Typed NO_CONFIG by the API: nothing is served. - endpoint1: - name: "test-endpoint-1" - telemetry_config: - table_names: - logs_table: main.default.test_logs - - # Typed EXTERNAL_MODELS by the API: it serves something, but not a custom model. - endpoint2: - name: "test-endpoint-2" - config: - served_entities: - - name: prod - external_model: - name: gpt-4o-mini - provider: openai - task: llm/v1/chat - openai_config: - openai_api_key_plaintext: sk-test-plaintext-key - telemetry_config: - table_names: - logs_table: main.default.test_logs diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/out.test.toml deleted file mode 100644 index f784a183258..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt deleted file mode 100644 index 6993adea0b2..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/output.txt +++ /dev/null @@ -1,21 +0,0 @@ -Error: telemetry_config is only supported on an endpoint that serves a registered model - at resources.model_serving_endpoints.endpoint1.telemetry_config - in databricks.yml:10:9 - -The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. - -Error: telemetry_config is only supported on an endpoint that serves a registered model - at resources.model_serving_endpoints.endpoint2.telemetry_config - in databricks.yml:26:9 - -The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config. - -Name: model_serving_telemetry -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default - -Found 2 errors - -Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/script b/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/script deleted file mode 100644 index 72555b332a4..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_unsupported_endpoint/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle validate diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml deleted file mode 100644 index ac509878ced..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml +++ /dev/null @@ -1,16 +0,0 @@ -bundle: - name: "model_serving_telemetry" - -resources: - model_serving_endpoints: - # Telemetry is supported here, but a config naming no profile is discarded. - endpoint1: - name: "test-endpoint" - config: - served_entities: - - name: prod - entity_name: main.default.test_model - entity_version: "1" - telemetry_config: - inference_table_config: - sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml deleted file mode 100644 index f784a183258..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt deleted file mode 100644 index 5ca6943ebb5..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt +++ /dev/null @@ -1,15 +0,0 @@ -Error: telemetry_config must name a table in table_names or set telemetry_profile_id - at resources.model_serving_endpoints.endpoint1.telemetry_config - in databricks.yml:15:9 - -The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. - -Name: model_serving_telemetry -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default - -Found 1 error - -Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/script b/acceptance/bundle/validate/model_serving_telemetry_without_profile/script deleted file mode 100644 index 72555b332a4..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle validate diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go deleted file mode 100644 index 84f1462ebbb..00000000000 --- a/bundle/config/validate/validate_serving_telemetry_config.go +++ /dev/null @@ -1,91 +0,0 @@ -package validate - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/serving" -) - -func ValidateServingTelemetryConfig() bundle.ReadOnlyMutator { - return &validateServingTelemetryConfig{} -} - -type validateServingTelemetryConfig struct{ bundle.RO } - -func (v *validateServingTelemetryConfig) Name() string { - return "validate:validate_serving_telemetry_config" -} - -func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - - for key, endpoint := range b.Config.Resources.ModelServingEndpoints { - if endpoint.TelemetryConfig == nil { - continue - } - - path := "resources.model_serving_endpoints." + key + ".telemetry_config" - addDiag := func(summary, detail string) { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: summary, - Detail: detail, - Paths: []dyn.Path{dyn.MustPathFromString(path)}, - Locations: b.Config.GetLocations(path), - }) - } - - // The telemetry API rejects endpoints typed NO_CONFIG or EXTERNAL_MODELS, but create - // silently drops the field, so without this the bundle deploys once and fails after. - if !servesRegisteredModel(endpoint.Config) { - addDiag( - "telemetry_config is only supported on an endpoint that serves a registered model", - "The serving endpoints telemetry configuration API only supports endpoints with custom served models, so it rejects an endpoint that serves nothing or only external models. Add a served entity with entity_name, or remove telemetry_config.", - ) - } - - // Create and the telemetry API both report success and apply nothing when the config - // names no profile, so an inference_table_config alone would drift forever. - if !identifiesTelemetryProfile(endpoint.TelemetryConfig) { - addDiag( - "telemetry_config must name a table in table_names or set telemetry_profile_id", - "The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it.", - ) - } - } - - sortDiagnostics(diags) - - return diags -} - -// identifiesTelemetryProfile reports whether the configuration names an existing profile -// or the tables to create one from. An empty table_names names neither. -func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { - if config.TelemetryProfileId != "" { - return true - } - t := config.TableNames - return t != nil && (t.LogsTable != "" || t.TracesTable != "" || t.MetricsTable != "" || t.AnnotationsTable != "") -} - -// servesRegisteredModel reports whether the endpoint serves at least one registered model. -// A foundation model is named the same way and is accepted here; create rejects it outright. -// -// ModelServingEndpointFixups has already folded served_models into served_entities. -func servesRegisteredModel(config *serving.EndpointCoreConfigInput) bool { - if config == nil { - return false - } - - for _, entity := range config.ServedEntities { - if entity.ExternalModel == nil && entity.EntityName != "" { - return true - } - } - - return false -} diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go deleted file mode 100644 index 48ee3a774e4..00000000000 --- a/bundle/config/validate/validate_serving_telemetry_config_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/serving" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - unsupportedEndpoint = "telemetry_config is only supported on an endpoint that serves a registered model" - noProfile = "telemetry_config must name a table in table_names or set telemetry_profile_id" -) - -func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { - return &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ - "my_endpoint": {CreateServingEndpoint: endpoint}, - }, - }, - }, - } -} - -// telemetryConfig names the tables to create a profile from, the form the API accepts. -func telemetryConfig() *serving.TelemetryConfig { - return &serving.TelemetryConfig{ - TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}, - } -} - -func servedModelConfig() *serving.EndpointCoreConfigInput { - return &serving.EndpointCoreConfigInput{ - ServedEntities: []serving.ServedEntityInput{{ - Name: "prod", - EntityName: "main.default.my_model", - EntityVersion: "1", - }}, - } -} - -func TestValidateServingTelemetryConfig(t *testing.T) { - tests := []struct { - name string - endpoint serving.CreateServingEndpoint - want []string - }{ - { - name: "telemetry with a served registered model", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: telemetryConfig(), - Config: servedModelConfig(), - }, - }, - { - name: "telemetry pinned to an existing profile", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{TelemetryProfileId: "abc"}, - Config: servedModelConfig(), - }, - }, - { - name: "no telemetry and nothing served", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - }, - }, - { - name: "telemetry and no config", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: telemetryConfig(), - }, - want: []string{unsupportedEndpoint}, - }, - { - name: "telemetry and a config that serves nothing", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: telemetryConfig(), - Config: &serving.EndpointCoreConfigInput{}, - }, - want: []string{unsupportedEndpoint}, - }, - { - // Rejected as EXTERNAL_MODELS, even though the endpoint does serve something. - name: "telemetry with only an external model", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: telemetryConfig(), - Config: &serving.EndpointCoreConfigInput{ - ServedEntities: []serving.ServedEntityInput{{ - Name: "prod", - ExternalModel: &serving.ExternalModel{ - Name: "gpt-4o-mini", - Provider: "openai", - Task: "llm/v1/chat", - }, - }}, - }, - }, - want: []string{unsupportedEndpoint}, - }, - { - // Names no tables, so nothing identifies the profile. - name: "telemetry with an empty table_names", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - TableNames: &serving.UnityCatalogTableNames{}, - }, - Config: servedModelConfig(), - }, - want: []string{noProfile}, - }, - { - name: "telemetry with only a traces table", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - TableNames: &serving.UnityCatalogTableNames{TracesTable: "main.default.traces"}, - }, - Config: servedModelConfig(), - }, - }, - { - // Discarded by the API, so the sampling fraction never takes effect. - name: "telemetry with only an inference table config", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, - }, - Config: servedModelConfig(), - }, - want: []string{noProfile}, - }, - { - name: "an endpoint can fail both rules", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, - }, - }, - want: []string{unsupportedEndpoint, noProfile}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - diags := ValidateServingTelemetryConfig().Apply(t.Context(), endpointBundle(tt.endpoint)) - - require.Len(t, diags, len(tt.want)) - for i, summary := range tt.want { - assert.Equal(t, diag.Error, diags[i].Severity) - assert.Equal(t, summary, diags[i].Summary) - assert.Equal(t, "resources.model_serving_endpoints.my_endpoint.telemetry_config", diags[i].Paths[0].String()) - } - }) - } -} diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index e87daf873b6..935899d7d64 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -360,8 +360,8 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, if entry.Changes.HasChange(pathTelemetryConfig) { // The telemetry API rejects an endpoint that is still applying an update, which the // calls above start and an interrupted deploy can leave behind. WaitAfterUpdate only - // runs after DoUpdate returns. - _, err = r.client.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, id, 35*time.Minute, nil) + // runs once DoUpdate has returned, so it cannot order these two calls. + _, err = r.waitForEndpointReady(ctx, id) if err != nil { return nil, err } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index d0f22464355..37f6ebf8890 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,8 +285,8 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # The backend returns only the telemetry_profile_id it creates from these tables, never - # the tables themselves. inference_table_config is not listed: sampling_fraction round trips. + # Consumed to create a telemetry profile and never echoed back; GET returns only the + # resulting telemetry_profile_id. inference_table_config is not listed: it round trips. - field: telemetry_config.table_names reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -320,10 +320,9 @@ resources: - field: rate_limits reason: not_implemented backend_defaults: - # The backend assigns telemetry_profile_id when it creates the profile from table_names. - # Declared on the subtree: structdiff reports an undeclared telemetry_config as one change - # on the parent path, which no child rule matches, so DoUpdate would PATCH it away. - # Removal still applies: it carries a non-nil old value, which this rule requires to be nil. + # telemetry_profile_id is assigned by the backend when it creates the profile. + # Declared on the subtree rather than that one field: a telemetry_config the bundle + # never set arrives as a single change on the parent path, which a child rule misses. - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 4f06e3ca7db..873dc31dc07 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,10 +185,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), - // Validate that telemetry_config is in a form the serving endpoints API applies. - // It rejects or silently discards anything else. - validate.ValidateServingTelemetryConfig(), - // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index e4d3345a9fe..fb4e0f23fe6 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,9 +35,10 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend: a config naming neither telemetry_profile_id -// nor table_names returns 200 and applies nothing. What comes back is the profile ID plus -// inference_table_config if the caller supplied one, never table_names. +// applyTelemetryConfig mirrors the backend: table_names is consumed to create a profile and +// never echoed back, so a GET returns the profile ID plus the inference_table_config the +// caller supplied. A config naming neither table_names nor telemetry_profile_id identifies +// no profile, and the backend reports success while applying nothing. func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil @@ -53,8 +54,7 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te } if config.InferenceTableConfig != nil { inferenceTable := *config.InferenceTableConfig - // Left empty when the patch reuses a profile by ID: it names no tables, and what - // the backend reports there was never observed. + // The backend names the payload table after the logs table it was given. if config.TableNames != nil { inferenceTable.Name = config.TableNames.LogsTable + "_payload" } @@ -64,20 +64,6 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te return &applied } -// telemetrySupported mirrors the backend, which supports telemetry only on custom served -// models, and returns the endpoint type the backend names when it rejects one. -func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) { - if endpoint.Config == nil || len(endpoint.Config.ServedEntities) == 0 { - return "NO_CONFIG", false - } - for _, entity := range endpoint.Config.ServedEntities { - if entity.ExternalModel == nil && entity.EntityName != "" { - return "", true - } - } - return "EXTERNAL_MODELS", false -} - // clearExternalModelSecrets mirrors the backend, which persists the *_plaintext // API keys as secrets and never returns them on GET. func clearExternalModelSecrets(em *serving.ExternalModel) *serving.ExternalModel { @@ -278,11 +264,6 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { ForceSendFields: append(createReq.ForceSendFields, "PermissionLevel", "RouteOptimized", "Description"), } - // Unlike the telemetry API, create drops telemetry it cannot apply instead of failing. - if _, ok := telemetrySupported(endpoint); !ok { - endpoint.TelemetryConfig = nil - } - s.ServingEndpoints[createReq.Name] = endpoint return Response{ @@ -471,16 +452,6 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } } - if endpointType, ok := telemetrySupported(endpoint); !ok { - return Response{ - StatusCode: 400, - Body: map[string]string{ - "error_code": "INVALID_PARAMETER_VALUE", - "message": fmt.Sprintf("Telemetry configuration is not supported for endpoint type '%s'. This API only supports endpoints with custom served models.", endpointType), - }, - } - } - // The telemetry API refuses to run while an earlier update is still applying. if endpointUpdating(endpoint) { return Response{ From e50622bb5a413372848398e7f49c75de8d65d88c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 14:24:25 +0000 Subject: [PATCH 17/28] Reject a telemetry_config that names no profile Dropping both validation rules assumed a bundle in either shape gets the serving API's own 400 at deploy time. That holds for the endpoint-type rule, but not for this one: a telemetry_config naming neither table_names nor telemetry_profile_id is discarded, and create and the telemetry API both report success. So the deploy succeeds, the endpoint receives no telemetry, and every plan afterwards reports an update the next deploy re-sends and the backend discards again -- the drift this branch exists to fix, left in place for that shape. The endpoint-type rule stays out, since the API does reject those outright. With one rule left there is nothing to repeat, so the addDiag closure goes and servesRegisteredModel goes with the rule it backed. The changelog regains the regression this branch introduces for endpoints typed NO_CONFIG or EXTERNAL_MODELS: the field used to be dropped at create with DoUpdate never calling the telemetry API, so those bundles deployed. They now fail. --- ...del-serving-telemetry-config-validation.md | 2 + .../databricks.yml | 16 +++ .../out.test.toml | 3 + .../output.txt | 15 +++ .../script | 1 + .../validate_serving_telemetry_config.go | 55 ++++++++ .../validate_serving_telemetry_config_test.go | 118 ++++++++++++++++++ bundle/phases/initialize.go | 4 + 8 files changed, 214 insertions(+) create mode 100644 .nextchanges/notable-changes/model-serving-telemetry-config-validation.md create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt create mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/script create mode 100644 bundle/config/validate/validate_serving_telemetry_config.go create mode 100644 bundle/config/validate/validate_serving_telemetry_config_test.go diff --git a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md new file mode 100644 index 00000000000..79872c1aecb --- /dev/null +++ b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md @@ -0,0 +1,2 @@ +* `bundle validate` now rejects a model serving `telemetry_config` that names neither a table in `table_names` nor a `telemetry_profile_id`. The serving API discards such a configuration instead of applying it, so the bundle appeared to deploy while the endpoint received no telemetry, and every later `bundle plan` reported an update that never converged. A bundle already deployed in that shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). +* direct: A `telemetry_config` on a model serving endpoint that serves nothing or only external models now fails `bundle deploy` with the error the serving endpoints telemetry API returns for it. Previously the field was silently dropped when the endpoint was created and never applied afterwards, so the deploy succeeded and the endpoint received no telemetry ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml new file mode 100644 index 00000000000..ac509878ced --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: "model_serving_telemetry" + +resources: + model_serving_endpoints: + # Telemetry is supported here, but a config naming no profile is discarded. + endpoint1: + name: "test-endpoint" + config: + served_entities: + - name: prod + entity_name: main.default.test_model + entity_version: "1" + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt new file mode 100644 index 00000000000..5ca6943ebb5 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt @@ -0,0 +1,15 @@ +Error: telemetry_config must name a table in table_names or set telemetry_profile_id + at resources.model_serving_endpoints.endpoint1.telemetry_config + in databricks.yml:15:9 + +The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. + +Name: model_serving_telemetry +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/script b/acceptance/bundle/validate/model_serving_telemetry_without_profile/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/model_serving_telemetry_without_profile/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go new file mode 100644 index 00000000000..08a2c08c2a6 --- /dev/null +++ b/bundle/config/validate/validate_serving_telemetry_config.go @@ -0,0 +1,55 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/serving" +) + +func ValidateServingTelemetryConfig() bundle.ReadOnlyMutator { + return &validateServingTelemetryConfig{} +} + +type validateServingTelemetryConfig struct{ bundle.RO } + +func (v *validateServingTelemetryConfig) Name() string { + return "validate:validate_serving_telemetry_config" +} + +func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + for key, endpoint := range b.Config.Resources.ModelServingEndpoints { + if endpoint.TelemetryConfig == nil || identifiesTelemetryProfile(endpoint.TelemetryConfig) { + continue + } + + // Create and the telemetry API both report success and apply nothing when the config + // names no profile, so the field would be planned and re-sent on every deploy. + path := "resources.model_serving_endpoints." + key + ".telemetry_config" + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "telemetry_config must name a table in table_names or set telemetry_profile_id", + Detail: "The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it.", + Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocations(path), + }) + } + + sortDiagnostics(diags) + + return diags +} + +// identifiesTelemetryProfile reports whether the configuration names an existing profile +// or the tables to create one from. An empty table_names names neither. +func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { + if config.TelemetryProfileId != "" { + return true + } + t := config.TableNames + return t != nil && (t.LogsTable != "" || t.TracesTable != "" || t.MetricsTable != "" || t.AnnotationsTable != "") +} diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go new file mode 100644 index 00000000000..5b6982c2a3a --- /dev/null +++ b/bundle/config/validate/validate_serving_telemetry_config_test.go @@ -0,0 +1,118 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const noProfile = "telemetry_config must name a table in table_names or set telemetry_profile_id" + +func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "my_endpoint": {CreateServingEndpoint: endpoint}, + }, + }, + }, + } +} + +func servedModelConfig() *serving.EndpointCoreConfigInput { + return &serving.EndpointCoreConfigInput{ + ServedEntities: []serving.ServedEntityInput{{ + Name: "prod", + EntityName: "main.default.my_model", + EntityVersion: "1", + }}, + } +} + +func TestValidateServingTelemetryConfig(t *testing.T) { + tests := []struct { + name string + endpoint serving.CreateServingEndpoint + want []string + }{ + { + name: "telemetry naming the tables to create a profile from", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}, + }, + Config: servedModelConfig(), + }, + }, + { + name: "telemetry pinned to an existing profile", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{TelemetryProfileId: "abc"}, + Config: servedModelConfig(), + }, + }, + { + name: "telemetry with only a traces table", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + TableNames: &serving.UnityCatalogTableNames{TracesTable: "main.default.traces"}, + }, + Config: servedModelConfig(), + }, + }, + { + name: "no telemetry at all", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + Config: servedModelConfig(), + }, + }, + { + // Names no tables, so nothing identifies the profile. + name: "telemetry with an empty table_names", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + TableNames: &serving.UnityCatalogTableNames{}, + }, + Config: servedModelConfig(), + }, + want: []string{noProfile}, + }, + { + // Discarded by the API, so the sampling fraction never takes effect. + name: "telemetry with only an inference table config", + endpoint: serving.CreateServingEndpoint{ + Name: "my_endpoint", + TelemetryConfig: &serving.TelemetryConfig{ + InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, + }, + Config: servedModelConfig(), + }, + want: []string{noProfile}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + diags := ValidateServingTelemetryConfig().Apply(t.Context(), endpointBundle(tt.endpoint)) + + require.Len(t, diags, len(tt.want)) + for i, summary := range tt.want { + assert.Equal(t, diag.Error, diags[i].Severity) + assert.Equal(t, summary, diags[i].Summary) + assert.Equal(t, "resources.model_serving_endpoints.my_endpoint.telemetry_config", diags[i].Paths[0].String()) + } + }) + } +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 873dc31dc07..8415d756f02 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,6 +185,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Validate that a model serving telemetry_config names the profile it configures. + // The serving API discards one that does not, instead of reporting an error. + validate.ValidateServingTelemetryConfig(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. From b08f4a813adf604d2cf7f59380d3ac8da6358f1c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 14:34:56 +0000 Subject: [PATCH 18/28] Move the telemetry_config validation rule out of this change Rejecting a telemetry_config that names no profile is a user-visible behavior change riding along with a bug fix, and it hard-errors bundle validate, which blocks destroy as well as deploy for anyone already deployed in that shape. It needs its own review, so it goes to a follow-up. That shape still never converges: the serving API discards the field and reports success, so every plan reports an update the next deploy re-sends. Unfixed here, not regressed by here. The changelog keeps the regression this change does introduce, for endpoints typed NO_CONFIG or EXTERNAL_MODELS, and the fragment is renamed to match now that it no longer describes a validation rule. --- ...del-serving-telemetry-config-validation.md | 2 - ...-serving-telemetry-unsupported-endpoint.md | 1 + .../databricks.yml | 16 --- .../out.test.toml | 3 - .../output.txt | 15 --- .../script | 1 - .../validate_serving_telemetry_config.go | 55 -------- .../validate_serving_telemetry_config_test.go | 118 ------------------ bundle/phases/initialize.go | 4 - 9 files changed, 1 insertion(+), 214 deletions(-) delete mode 100644 .nextchanges/notable-changes/model-serving-telemetry-config-validation.md create mode 100644 .nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt delete mode 100644 acceptance/bundle/validate/model_serving_telemetry_without_profile/script delete mode 100644 bundle/config/validate/validate_serving_telemetry_config.go delete mode 100644 bundle/config/validate/validate_serving_telemetry_config_test.go diff --git a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md b/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md deleted file mode 100644 index 79872c1aecb..00000000000 --- a/.nextchanges/notable-changes/model-serving-telemetry-config-validation.md +++ /dev/null @@ -1,2 +0,0 @@ -* `bundle validate` now rejects a model serving `telemetry_config` that names neither a table in `table_names` nor a `telemetry_profile_id`. The serving API discards such a configuration instead of applying it, so the bundle appeared to deploy while the endpoint received no telemetry, and every later `bundle plan` reported an update that never converged. A bundle already deployed in that shape has to fix or drop the field before `bundle deploy` or `bundle destroy` will run again ([#6106](https://github.com/databricks/cli/pull/6106)). -* direct: A `telemetry_config` on a model serving endpoint that serves nothing or only external models now fails `bundle deploy` with the error the serving endpoints telemetry API returns for it. Previously the field was silently dropped when the endpoint was created and never applied afterwards, so the deploy succeeded and the endpoint received no telemetry ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md b/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md new file mode 100644 index 00000000000..33c661ff0b7 --- /dev/null +++ b/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md @@ -0,0 +1 @@ +direct: A `telemetry_config` on a model serving endpoint that serves nothing or only external models now fails `bundle deploy` with the error the serving endpoints telemetry API returns for it. Previously the field was silently dropped when the endpoint was created and never applied afterwards, so the deploy succeeded and the endpoint received no telemetry ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml deleted file mode 100644 index ac509878ced..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/databricks.yml +++ /dev/null @@ -1,16 +0,0 @@ -bundle: - name: "model_serving_telemetry" - -resources: - model_serving_endpoints: - # Telemetry is supported here, but a config naming no profile is discarded. - endpoint1: - name: "test-endpoint" - config: - served_entities: - - name: prod - entity_name: main.default.test_model - entity_version: "1" - telemetry_config: - inference_table_config: - sampling_fraction: 0.5 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml b/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml deleted file mode 100644 index f784a183258..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt b/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt deleted file mode 100644 index 5ca6943ebb5..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/output.txt +++ /dev/null @@ -1,15 +0,0 @@ -Error: telemetry_config must name a table in table_names or set telemetry_profile_id - at resources.model_serving_endpoints.endpoint1.telemetry_config - in databricks.yml:15:9 - -The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it. - -Name: model_serving_telemetry -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/model_serving_telemetry/default - -Found 1 error - -Exit code: 1 diff --git a/acceptance/bundle/validate/model_serving_telemetry_without_profile/script b/acceptance/bundle/validate/model_serving_telemetry_without_profile/script deleted file mode 100644 index 72555b332a4..00000000000 --- a/acceptance/bundle/validate/model_serving_telemetry_without_profile/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle validate diff --git a/bundle/config/validate/validate_serving_telemetry_config.go b/bundle/config/validate/validate_serving_telemetry_config.go deleted file mode 100644 index 08a2c08c2a6..00000000000 --- a/bundle/config/validate/validate_serving_telemetry_config.go +++ /dev/null @@ -1,55 +0,0 @@ -package validate - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/serving" -) - -func ValidateServingTelemetryConfig() bundle.ReadOnlyMutator { - return &validateServingTelemetryConfig{} -} - -type validateServingTelemetryConfig struct{ bundle.RO } - -func (v *validateServingTelemetryConfig) Name() string { - return "validate:validate_serving_telemetry_config" -} - -func (v *validateServingTelemetryConfig) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - - for key, endpoint := range b.Config.Resources.ModelServingEndpoints { - if endpoint.TelemetryConfig == nil || identifiesTelemetryProfile(endpoint.TelemetryConfig) { - continue - } - - // Create and the telemetry API both report success and apply nothing when the config - // names no profile, so the field would be planned and re-sent on every deploy. - path := "resources.model_serving_endpoints." + key + ".telemetry_config" - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: "telemetry_config must name a table in table_names or set telemetry_profile_id", - Detail: "The serving endpoints API identifies a telemetry configuration by the Unity Catalog tables to create a profile from (table_names) or by an existing profile (telemetry_profile_id). Given neither, it discards the configuration instead of applying it.", - Paths: []dyn.Path{dyn.MustPathFromString(path)}, - Locations: b.Config.GetLocations(path), - }) - } - - sortDiagnostics(diags) - - return diags -} - -// identifiesTelemetryProfile reports whether the configuration names an existing profile -// or the tables to create one from. An empty table_names names neither. -func identifiesTelemetryProfile(config *serving.TelemetryConfig) bool { - if config.TelemetryProfileId != "" { - return true - } - t := config.TableNames - return t != nil && (t.LogsTable != "" || t.TracesTable != "" || t.MetricsTable != "" || t.AnnotationsTable != "") -} diff --git a/bundle/config/validate/validate_serving_telemetry_config_test.go b/bundle/config/validate/validate_serving_telemetry_config_test.go deleted file mode 100644 index 5b6982c2a3a..00000000000 --- a/bundle/config/validate/validate_serving_telemetry_config_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/serving" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const noProfile = "telemetry_config must name a table in table_names or set telemetry_profile_id" - -func endpointBundle(endpoint serving.CreateServingEndpoint) *bundle.Bundle { - return &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ - "my_endpoint": {CreateServingEndpoint: endpoint}, - }, - }, - }, - } -} - -func servedModelConfig() *serving.EndpointCoreConfigInput { - return &serving.EndpointCoreConfigInput{ - ServedEntities: []serving.ServedEntityInput{{ - Name: "prod", - EntityName: "main.default.my_model", - EntityVersion: "1", - }}, - } -} - -func TestValidateServingTelemetryConfig(t *testing.T) { - tests := []struct { - name string - endpoint serving.CreateServingEndpoint - want []string - }{ - { - name: "telemetry naming the tables to create a profile from", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}, - }, - Config: servedModelConfig(), - }, - }, - { - name: "telemetry pinned to an existing profile", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{TelemetryProfileId: "abc"}, - Config: servedModelConfig(), - }, - }, - { - name: "telemetry with only a traces table", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - TableNames: &serving.UnityCatalogTableNames{TracesTable: "main.default.traces"}, - }, - Config: servedModelConfig(), - }, - }, - { - name: "no telemetry at all", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - Config: servedModelConfig(), - }, - }, - { - // Names no tables, so nothing identifies the profile. - name: "telemetry with an empty table_names", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - TableNames: &serving.UnityCatalogTableNames{}, - }, - Config: servedModelConfig(), - }, - want: []string{noProfile}, - }, - { - // Discarded by the API, so the sampling fraction never takes effect. - name: "telemetry with only an inference table config", - endpoint: serving.CreateServingEndpoint{ - Name: "my_endpoint", - TelemetryConfig: &serving.TelemetryConfig{ - InferenceTableConfig: &serving.TelemetryInferenceTableConfig{SamplingFraction: 0.5}, - }, - Config: servedModelConfig(), - }, - want: []string{noProfile}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - diags := ValidateServingTelemetryConfig().Apply(t.Context(), endpointBundle(tt.endpoint)) - - require.Len(t, diags, len(tt.want)) - for i, summary := range tt.want { - assert.Equal(t, diag.Error, diags[i].Severity) - assert.Equal(t, summary, diags[i].Summary) - assert.Equal(t, "resources.model_serving_endpoints.my_endpoint.telemetry_config", diags[i].Paths[0].String()) - } - }) - } -} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8415d756f02..873dc31dc07 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,10 +185,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), - // Validate that a model serving telemetry_config names the profile it configures. - // The serving API discards one that does not, instead of reporting an error. - validate.ValidateServingTelemetryConfig(), - // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. From e2f8ede9aafa1b31effa98a9366deb1944a5481c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 14:42:45 +0000 Subject: [PATCH 19/28] Show only what each telemetry_config test is about The goldens were three quarters of this change, mostly because the scripts dumped more than the tests assert: every plan printed the whole change set, so config.traffic_config, description and route_optimized appeared in each of four dumps, and every request printed as a pretty block, so a poll of two identical GETs cost eight lines. The plan dumps now select the telemetry_config subtree and the request dumps print one request per line, both following what the apps and permissions tests already do. The request sequence the wait test exists to show reads better collapsed with uniq -c than it did spread over eighty lines. No scenario, step or assertion is dropped: 439 golden lines become 215. --- .../drift/telemetry_config/output.txt | 137 +----------------- .../drift/telemetry_config/script | 8 +- .../telemetry_config_unmanaged/output.txt | 24 +-- .../drift/telemetry_config_unmanaged/script | 4 +- .../output.txt | 91 +----------- .../script | 2 +- 6 files changed, 21 insertions(+), 245 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index 6d1336e7d6a..cd2d39004f3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -13,29 +13,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Editing sampling_fraction is a real change: it round trips through GET >>> [CLI] bundle plan --output json { - "config.traffic_config": { - "action": "skip", - "reason": "backend_default", - "remote": { - "routes": [ - { - "served_entity_name": "prod", - "served_model_name": "prod", - "traffic_percentage": 100 - } - ] - } - }, - "description": { - "action": "skip", - "reason": "empty", - "remote": "" - }, - "route_optimized": { - "action": "skip", - "reason": "empty", - "remote": false - }, "telemetry_config.inference_table_config.name": { "action": "skip", "reason": "spec:output_only", @@ -80,29 +57,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Editing table_names is a real change even though GET never echoes it >>> [CLI] bundle plan --output json { - "config.traffic_config": { - "action": "skip", - "reason": "backend_default", - "remote": { - "routes": [ - { - "served_entity_name": "prod", - "served_model_name": "prod", - "traffic_percentage": 100 - } - ] - } - }, - "description": { - "action": "skip", - "reason": "empty", - "remote": "" - }, - "route_optimized": { - "action": "skip", - "reason": "empty", - "remote": false - }, "telemetry_config.inference_table_config.name": { "action": "skip", "reason": "spec:output_only", @@ -144,29 +98,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Removing telemetry_config from the bundle removes it remotely >>> [CLI] bundle plan --output json { - "config.traffic_config": { - "action": "skip", - "reason": "backend_default", - "remote": { - "routes": [ - { - "served_entity_name": "prod", - "served_model_name": "prod", - "traffic_percentage": 100 - } - ] - } - }, - "description": { - "action": "skip", - "reason": "empty", - "remote": "" - }, - "route_optimized": { - "action": "skip", - "reason": "empty", - "remote": false - }, "telemetry_config": { "action": "update", "old": { @@ -201,66 +132,8 @@ null >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //serving-endpoints -{ - "method": "POST", - "path": "/api/2.0/serving-endpoints", - "body": { - "config": { - "served_entities": [ - { - "entity_name": "main.default.test_model", - "entity_version": "1", - "name": "prod", - "scale_to_zero_enabled": true, - "workload_size": "Small" - } - ] - }, - "name": "test-endpoint", - "telemetry_config": { - "inference_table_config": { - "sampling_fraction": 0.5 - }, - "table_names": { - "logs_table": "main.default.test_logs", - "traces_table": "main.default.test_traces" - } - } - } -} -{ - "method": "PATCH", - "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", - "body": { - "telemetry_config": { - "inference_table_config": { - "sampling_fraction": 0.9 - }, - "table_names": { - "logs_table": "main.default.test_logs", - "traces_table": "main.default.test_traces" - } - } - } -} -{ - "method": "PATCH", - "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", - "body": { - "telemetry_config": { - "inference_table_config": { - "sampling_fraction": 0.9 - }, - "table_names": { - "logs_table": "main.default.test_logs_v2", - "traces_table": "main.default.test_traces" - } - } - } -} -{ - "method": "PATCH", - "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", - "body": {} -} +>>> print_requests.py --oneline //serving-endpoints +{"method": "POST", "path": "/api/2.0/serving-endpoints", "body": {"config": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Small"}]}, "name": "test-endpoint", "telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}, "table_names": {"logs_table": "main.default.test_logs", "traces_table": "main.default.test_traces"}}}} +{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs", "traces_table": "main.default.test_traces"}}}} +{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs_v2", "traces_table": "main.default.test_traces"}}}} +{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index 5d55d346a3b..d396ca93b3f 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -8,7 +8,7 @@ trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, title "Editing sampling_fraction is a real change: it round trips through GET" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' title "Deploy applies it via the telemetry-config API" trace $CLI bundle deploy @@ -18,7 +18,7 @@ trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, title "Editing table_names is a real change even though GET never echoes it" update_file.py databricks.yml "main.default.test_logs" "main.default.test_logs_v2" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' trace $CLI bundle deploy title "Plan converges after the tables changed" @@ -26,11 +26,11 @@ trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, title "Removing telemetry_config from the bundle removes it remotely" grep -v TO_REMOVE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' trace $CLI bundle deploy MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config title "Plan converges with telemetry_config removed" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -trace print_requests.py //serving-endpoints +trace print_requests.py --oneline //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt index 88735a92023..d7ca4c61572 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt @@ -10,16 +10,6 @@ Deployment complete! === A remote-only telemetry_config is a backend default, not drift >>> [CLI] bundle plan --output json { - "description": { - "action": "skip", - "reason": "empty", - "remote": "" - }, - "route_optimized": { - "action": "skip", - "reason": "empty", - "remote": false - }, "telemetry_config": { "action": "skip", "reason": "backend_default", @@ -42,15 +32,5 @@ Deployment complete! } === The out-of-band call above is the only telemetry-config PATCH ->>> print_requests.py --method PATCH //serving-endpoints -{ - "method": "PATCH", - "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", - "body": { - "telemetry_config": { - "table_names": { - "logs_table": "main.default.other_logs" - } - } - } -} +>>> print_requests.py --method PATCH --oneline //serving-endpoints +{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index e80552e8a37..fbe2924fb46 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -7,11 +7,11 @@ title "Telemetry gets configured out of band, as on an adopted endpoint" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "A remote-only telemetry_config is a backend default, not drift" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' title "Deploying does not remove it" trace $CLI bundle deploy MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config title "The out-of-band call above is the only telemetry-config PATCH" -trace print_requests.py --method PATCH //serving-endpoints +trace print_requests.py --method PATCH --oneline //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt index b60d31a3711..1c88f5d3b6f 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -31,87 +31,10 @@ Deployment complete! Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The config update is followed by a poll for NOT_UPDATING, then the telemetry patch ->>> print_requests.py --get //serving-endpoints -{ - "method": "POST", - "path": "/api/2.0/serving-endpoints", - "body": { - "config": { - "served_entities": [ - { - "entity_name": "main.default.test_model", - "entity_version": "1", - "name": "prod", - "scale_to_zero_enabled": true, - "workload_size": "Small" - } - ] - }, - "name": "test-endpoint", - "telemetry_config": { - "inference_table_config": { - "sampling_fraction": 0.5 - }, - "table_names": { - "logs_table": "main.default.test_logs" - } - } - } -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} -{ - "method": "PUT", - "path": "/api/2.0/serving-endpoints/test-endpoint/config", - "body": { - "served_entities": [ - { - "entity_name": "main.default.test_model", - "entity_version": "1", - "name": "prod", - "scale_to_zero_enabled": true, - "workload_size": "Medium" - } - ] - } -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} -{ - "method": "PATCH", - "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", - "body": { - "telemetry_config": { - "inference_table_config": { - "sampling_fraction": 0.9 - }, - "table_names": { - "logs_table": "main.default.test_logs" - } - } - } -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} -{ - "method": "GET", - "path": "/api/2.0/serving-endpoints/test-endpoint" -} +>>> print_requests.py --get --oneline //serving-endpoints +1 {"method": "POST", "path": "/api/2.0/serving-endpoints", "body": {"config": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Small"}]}, "name": "test-endpoint", "telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}, "table_names": {"logs_table": "main.default.test_logs"}}}} +3 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} +1 {"method": "PUT", "path": "/api/2.0/serving-endpoints/test-endpoint/config", "body": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Medium"}]}} +2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} +1 {"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs"}}}} +2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script index abb93db4778..ba40d7cee33 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script @@ -15,4 +15,4 @@ title "Plan converges" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The config update is followed by a poll for NOT_UPDATING, then the telemetry patch" -trace print_requests.py --get //serving-endpoints +trace print_requests.py --get --oneline //serving-endpoints | uniq -c | sed 's/^ *//' From 3a61bb710703742f19dbf1d8b0d34cb4eb9e04c4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 6 Aug 2026 14:47:54 +0000 Subject: [PATCH 20/28] Trim the comments that restate their code endpointUpdating had a doc comment repeating its name and its one-line body. The file already documents only the helpers with a reason to state, so it goes. The applyTelemetryConfig doc and the two test.toml notes said in three lines what fits in two, and the one on telemetry_config_with_config_update repeated the 409 explanation the comment in DoUpdate already carries. --- .../drift/telemetry_config_unmanaged/test.toml | 5 ++--- .../drift/telemetry_config_with_config_update/test.toml | 5 ++--- libs/testserver/serving_endpoints.go | 6 ++---- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml index aaf5e57b677..2604635ec70 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -1,6 +1,5 @@ RecordRequests = true -# Direct engine only, where the classification lives. Guards the backend_defaults rule on -# the telemetry_config subtree: without it the remote-only config is planned as an update -# and DoUpdate patches it away. +# Direct engine only, where the classification lives. Guards the backend_defaults rule on the +# telemetry_config subtree: without it DoUpdate patches away a config the bundle never declared. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml index a89d9b03a11..f034a569c2d 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml @@ -1,6 +1,5 @@ RecordRequests = true -# Direct engine only, as with the sibling drift tests. Guards the wait in DoUpdate: the -# telemetry API rejects an endpoint still applying an earlier update, so without the wait -# this deploy fails on the telemetry patch with a 409. +# Direct engine only, as with the sibling drift tests. Guards the wait in DoUpdate: without +# it this deploy fails on the telemetry patch with a 409. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index fb4e0f23fe6..eacafb35e40 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -36,9 +36,8 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se } // applyTelemetryConfig mirrors the backend: table_names is consumed to create a profile and -// never echoed back, so a GET returns the profile ID plus the inference_table_config the -// caller supplied. A config naming neither table_names nor telemetry_profile_id identifies -// no profile, and the backend reports success while applying nothing. +// never echoed back, so GET returns the profile ID and the caller's inference_table_config. +// A config naming neither table_names nor telemetry_profile_id is discarded, reporting success. func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil @@ -297,7 +296,6 @@ func (s *FakeWorkspace) ServingEndpointGet(name string) Response { return Response{Body: endpoint} } -// endpointUpdating reports whether the endpoint is still applying a config update. func endpointUpdating(endpoint serving.ServingEndpointDetailed) bool { return endpoint.State != nil && endpoint.State.ConfigUpdate == serving.EndpointStateConfigUpdateInProgress } From 4c79f79d817782ca7f2d7d75aa2a72bd80986935 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 08:16:25 +0000 Subject: [PATCH 21/28] Drop leftover noise from the telemetry_config drift tests traffic_config on the unmanaged fixture, the unused discard branch in the fake, and the wait test's plan-keys dump were not exercising anything this change cares about. --- .../telemetry_config_unmanaged/databricks.yml | 4 ---- .../output.txt | 17 ++--------------- .../telemetry_config_with_config_update/script | 5 +---- libs/testserver/serving_endpoints.go | 10 +++------- 4 files changed, 6 insertions(+), 30 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml index 1ccacd2fee0..1291612ba80 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/databricks.yml @@ -12,7 +12,3 @@ resources: entity_version: "1" workload_size: Small scale_to_zero_enabled: true - traffic_config: - routes: - - served_model_name: prod - traffic_percentage: 100 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt index 1c88f5d3b6f..442a25bb5fa 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -6,20 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Change config and telemetry_config in the same deploy ->>> [CLI] bundle plan --output json -[ - "config.served_entities[0].workload_size", - "config.traffic_config", - "description", - "route_optimized", - "telemetry_config.inference_table_config.name", - "telemetry_config.inference_table_config.sampling_fraction", - "telemetry_config.table_names", - "telemetry_config.telemetry_profile_id" -] - -=== Deploy waits for the config update to settle before patching telemetry +=== Change config and telemetry_config; deploy waits for the config update to settle before patching telemetry >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... @@ -33,7 +20,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The config update is followed by a poll for NOT_UPDATING, then the telemetry patch >>> print_requests.py --get --oneline //serving-endpoints 1 {"method": "POST", "path": "/api/2.0/serving-endpoints", "body": {"config": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Small"}]}, "name": "test-endpoint", "telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}, "table_names": {"logs_table": "main.default.test_logs"}}}} -3 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} +2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} 1 {"method": "PUT", "path": "/api/2.0/serving-endpoints/test-endpoint/config", "body": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Medium"}]}} 2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} 1 {"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs"}}}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script index ba40d7cee33..31229dccf3c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script @@ -3,12 +3,9 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Change config and telemetry_config in the same deploy" +title "Change config and telemetry_config; deploy waits for the config update to settle before patching telemetry" update_file.py databricks.yml "workload_size: Small" "workload_size: Medium" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" -trace $CLI bundle plan --output json | jq '.plan[].changes | keys' - -title "Deploy waits for the config update to settle before patching telemetry" trace $CLI bundle deploy title "Plan converges" diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index eacafb35e40..8576aed4368 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -37,14 +37,10 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se // applyTelemetryConfig mirrors the backend: table_names is consumed to create a profile and // never echoed back, so GET returns the profile ID and the caller's inference_table_config. -// A config naming neither table_names nor telemetry_profile_id is discarded, reporting success. -func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { +func applyTelemetryConfig(config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil } - if config.TableNames == nil && config.TelemetryProfileId == "" { - return previous - } applied := serving.TelemetryConfig{TelemetryProfileId: config.TelemetryProfileId} if applied.TelemetryProfileId == "" { @@ -253,7 +249,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { PermissionLevel: serving.ServingEndpointDetailedPermissionLevelCanManage, RouteOptimized: createReq.RouteOptimized, Tags: createReq.Tags, - TelemetryConfig: applyTelemetryConfig(nil, createReq.TelemetryConfig), + TelemetryConfig: applyTelemetryConfig(createReq.TelemetryConfig), State: &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, Ready: serving.EndpointStateReadyNotReady, @@ -462,7 +458,7 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } // An omitted telemetry_config removes the configuration from the endpoint. - endpoint.TelemetryConfig = applyTelemetryConfig(endpoint.TelemetryConfig, patchReq.TelemetryConfig) + endpoint.TelemetryConfig = applyTelemetryConfig(patchReq.TelemetryConfig) endpoint.LastUpdatedTimestamp = nowMilli() s.ServingEndpoints[name] = endpoint From 14fbfee54e74acb910872e5d1538198128dac59a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 08:44:40 +0000 Subject: [PATCH 22/28] Restore telemetry fake fidelity dropped in the last cleanup The discard branch and endpoint-type rejection match the serving API: a config naming no profile is a no-op, create drops telemetry it cannot apply, and the telemetry PATCH rejects NO_CONFIG / EXTERNAL_MODELS. Clarify the notable changelog that create can still succeed on the first deploy of those endpoint types. --- ...-serving-telemetry-unsupported-endpoint.md | 2 +- libs/testserver/serving_endpoints.go | 42 +++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md b/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md index 33c661ff0b7..8ff7fefbea0 100644 --- a/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md +++ b/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md @@ -1 +1 @@ -direct: A `telemetry_config` on a model serving endpoint that serves nothing or only external models now fails `bundle deploy` with the error the serving endpoints telemetry API returns for it. Previously the field was silently dropped when the endpoint was created and never applied afterwards, so the deploy succeeded and the endpoint received no telemetry ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: A `telemetry_config` on a model serving endpoint that serves nothing or only external models now fails on a deploy that applies the field via the serving endpoints telemetry API, with the error that API returns. Create still drops the field rather than failing, so the first deploy of a new endpoint can succeed; a later deploy that plans an update then fails. Previously `DoUpdate` never called the telemetry API, so every deploy succeeded and the endpoint received no telemetry ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 8576aed4368..690b4ed3c74 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -37,14 +37,19 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se // applyTelemetryConfig mirrors the backend: table_names is consumed to create a profile and // never echoed back, so GET returns the profile ID and the caller's inference_table_config. -func applyTelemetryConfig(config *serving.TelemetryConfig) *serving.TelemetryConfig { +// A config naming neither table_names nor telemetry_profile_id is discarded, reporting success. +func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil } + if config.TableNames == nil && config.TelemetryProfileId == "" { + return previous + } applied := serving.TelemetryConfig{TelemetryProfileId: config.TelemetryProfileId} if applied.TelemetryProfileId == "" { - // New tables mean a new profile, not the endpoint's current one. + // table_names in the request provisions a profile; assign a fresh ID when the + // caller did not supply one (bundle configs never do). applied.TelemetryProfileId = nextUUID() } if config.InferenceTableConfig != nil { @@ -59,6 +64,20 @@ func applyTelemetryConfig(config *serving.TelemetryConfig) *serving.TelemetryCon return &applied } +// telemetrySupported mirrors the backend, which supports telemetry only on custom served +// models, and returns the endpoint type the backend names when it rejects one. +func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) { + if endpoint.Config == nil || len(endpoint.Config.ServedEntities) == 0 { + return "NO_CONFIG", false + } + for _, entity := range endpoint.Config.ServedEntities { + if entity.ExternalModel == nil && entity.EntityName != "" { + return "", true + } + } + return "EXTERNAL_MODELS", false +} + // clearExternalModelSecrets mirrors the backend, which persists the *_plaintext // API keys as secrets and never returns them on GET. func clearExternalModelSecrets(em *serving.ExternalModel) *serving.ExternalModel { @@ -249,7 +268,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { PermissionLevel: serving.ServingEndpointDetailedPermissionLevelCanManage, RouteOptimized: createReq.RouteOptimized, Tags: createReq.Tags, - TelemetryConfig: applyTelemetryConfig(createReq.TelemetryConfig), + TelemetryConfig: applyTelemetryConfig(nil, createReq.TelemetryConfig), State: &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, Ready: serving.EndpointStateReadyNotReady, @@ -259,6 +278,11 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { ForceSendFields: append(createReq.ForceSendFields, "PermissionLevel", "RouteOptimized", "Description"), } + // Unlike the telemetry API, create drops telemetry it cannot apply instead of failing. + if _, ok := telemetrySupported(endpoint); !ok { + endpoint.TelemetryConfig = nil + } + s.ServingEndpoints[createReq.Name] = endpoint return Response{ @@ -446,6 +470,16 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } } + if endpointType, ok := telemetrySupported(endpoint); !ok { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": fmt.Sprintf("Telemetry configuration is not supported for endpoint type '%s'. This API only supports endpoints with custom served models.", endpointType), + }, + } + } + // The telemetry API refuses to run while an earlier update is still applying. if endpointUpdating(endpoint) { return Response{ @@ -458,7 +492,7 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } // An omitted telemetry_config removes the configuration from the endpoint. - endpoint.TelemetryConfig = applyTelemetryConfig(patchReq.TelemetryConfig) + endpoint.TelemetryConfig = applyTelemetryConfig(endpoint.TelemetryConfig, patchReq.TelemetryConfig) endpoint.LastUpdatedTimestamp = nowMilli() s.ServingEndpoints[name] = endpoint From 37ba094b688217debe0caebbd036103cf1c67cfc Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 08:49:22 +0000 Subject: [PATCH 23/28] Tighten telemetry_config comments added by this change Keep the non-obvious API reasons; drop restatements of the surrounding code and trim the acceptance step titles that had grown into mini-explanations. --- .../drift/telemetry_config/output.txt | 12 +++++------ .../drift/telemetry_config/script | 12 +++++------ .../telemetry_config_unmanaged/output.txt | 10 +++++----- .../drift/telemetry_config_unmanaged/script | 10 +++++----- .../telemetry_config_unmanaged/test.toml | 3 +-- .../output.txt | 4 ++-- .../script | 4 ++-- .../test.toml | 3 +-- .../dresources/model_serving_endpoint.go | 5 ++--- bundle/direct/dresources/resources.yml | 7 ++----- libs/testserver/serving_endpoints.go | 20 ++++++++----------- 11 files changed, 40 insertions(+), 50 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index cd2d39004f3..e1677c4dc12 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -6,11 +6,11 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Plan converges: the profile and inference table the backend named are not drift +=== Plan converges >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== Editing sampling_fraction is a real change: it round trips through GET +=== Editing sampling_fraction plans an update >>> [CLI] bundle plan --output json { "telemetry_config.inference_table_config.name": { @@ -43,7 +43,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } -=== Deploy applies it via the telemetry-config API +=== Deploy applies the edit >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... @@ -54,7 +54,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== Editing table_names is a real change even though GET never echoes it +=== Editing table_names plans an update >>> [CLI] bundle plan --output json { "telemetry_config.inference_table_config.name": { @@ -91,11 +91,11 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Plan converges after the tables changed +=== Plan converges after table_names edit >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== Removing telemetry_config from the bundle removes it remotely +=== Removing telemetry_config removes it remotely >>> [CLI] bundle plan --output json { "telemetry_config": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index d396ca93b3f..47dba25bf8f 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -3,28 +3,28 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Plan converges: the profile and inference table the backend named are not drift" +title "Plan converges" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "Editing sampling_fraction is a real change: it round trips through GET" +title "Editing sampling_fraction plans an update" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' -title "Deploy applies it via the telemetry-config API" +title "Deploy applies the edit" trace $CLI bundle deploy title "Plan converges again" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "Editing table_names is a real change even though GET never echoes it" +title "Editing table_names plans an update" update_file.py databricks.yml "main.default.test_logs" "main.default.test_logs_v2" trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' trace $CLI bundle deploy -title "Plan converges after the tables changed" +title "Plan converges after table_names edit" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "Removing telemetry_config from the bundle removes it remotely" +title "Removing telemetry_config removes it remotely" grep -v TO_REMOVE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt index d7ca4c61572..f6924643628 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt @@ -1,13 +1,13 @@ -=== Deploy an endpoint that does not declare telemetry_config +=== Deploy without telemetry_config >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... Deployment complete! -=== Telemetry gets configured out of band, as on an adopted endpoint -=== A remote-only telemetry_config is a backend default, not drift +=== Configure telemetry out of band +=== Remote-only telemetry_config is not drift >>> [CLI] bundle plan --output json { "telemetry_config": { @@ -19,7 +19,7 @@ Deployment complete! } } -=== Deploying does not remove it +=== Deploy does not remove it >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... @@ -31,6 +31,6 @@ Deployment complete! "telemetry_profile_id": "[UUID]" } -=== The out-of-band call above is the only telemetry-config PATCH +=== Only the out-of-band telemetry PATCH was sent >>> print_requests.py --method PATCH --oneline //serving-endpoints {"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index fbe2924fb46..5e30339c21f 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -1,17 +1,17 @@ echo "*" > .gitignore -title "Deploy an endpoint that does not declare telemetry_config" +title "Deploy without telemetry_config" trace $CLI bundle deploy -title "Telemetry gets configured out of band, as on an adopted endpoint" +title "Configure telemetry out of band" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null -title "A remote-only telemetry_config is a backend default, not drift" +title "Remote-only telemetry_config is not drift" trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' -title "Deploying does not remove it" +title "Deploy does not remove it" trace $CLI bundle deploy MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config -title "The out-of-band call above is the only telemetry-config PATCH" +title "Only the out-of-band telemetry PATCH was sent" trace print_requests.py --method PATCH --oneline //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml index 2604635ec70..62f6a4c7a8b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -1,5 +1,4 @@ RecordRequests = true -# Direct engine only, where the classification lives. Guards the backend_defaults rule on the -# telemetry_config subtree: without it DoUpdate patches away a config the bundle never declared. +# Direct engine only, where the classification lives. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt index 442a25bb5fa..79940b58584 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Change config and telemetry_config; deploy waits for the config update to settle before patching telemetry +=== Deploy changing config and telemetry_config waits before the telemetry patch >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... @@ -17,7 +17,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== The config update is followed by a poll for NOT_UPDATING, then the telemetry patch +=== Request order: config PUT, poll, telemetry PATCH >>> print_requests.py --get --oneline //serving-endpoints 1 {"method": "POST", "path": "/api/2.0/serving-endpoints", "body": {"config": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Small"}]}, "name": "test-endpoint", "telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}, "table_names": {"logs_table": "main.default.test_logs"}}}} 2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script index 31229dccf3c..6b345736dfa 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script @@ -3,7 +3,7 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Change config and telemetry_config; deploy waits for the config update to settle before patching telemetry" +title "Deploy changing config and telemetry_config waits before the telemetry patch" update_file.py databricks.yml "workload_size: Small" "workload_size: Medium" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle deploy @@ -11,5 +11,5 @@ trace $CLI bundle deploy title "Plan converges" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "The config update is followed by a poll for NOT_UPDATING, then the telemetry patch" +title "Request order: config PUT, poll, telemetry PATCH" trace print_requests.py --get --oneline //serving-endpoints | uniq -c | sed 's/^ *//' diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml index f034a569c2d..b11179c5a12 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml @@ -1,5 +1,4 @@ RecordRequests = true -# Direct engine only, as with the sibling drift tests. Guards the wait in DoUpdate: without -# it this deploy fails on the telemetry patch with a 409. +# Direct engine only, where the DoUpdate wait lives. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 935899d7d64..9804054e655 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -358,9 +358,8 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, } if entry.Changes.HasChange(pathTelemetryConfig) { - // The telemetry API rejects an endpoint that is still applying an update, which the - // calls above start and an interrupted deploy can leave behind. WaitAfterUpdate only - // runs once DoUpdate has returned, so it cannot order these two calls. + // The telemetry API rejects an endpoint still applying an update, and + // WaitAfterUpdate runs too late to order the earlier calls with this patch. _, err = r.waitForEndpointReady(ctx, id) if err != nil { return nil, err diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 37f6ebf8890..00117077d1c 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,8 +285,7 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # Consumed to create a telemetry profile and never echoed back; GET returns only the - # resulting telemetry_profile_id. inference_table_config is not listed: it round trips. + # Consumed to build a profile and never returned; inference_table_config round-trips so it is not listed. - field: telemetry_config.table_names reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -320,9 +319,7 @@ resources: - field: rate_limits reason: not_implemented backend_defaults: - # telemetry_profile_id is assigned by the backend when it creates the profile. - # Declared on the subtree rather than that one field: a telemetry_config the bundle - # never set arrives as a single change on the parent path, which a child rule misses. + # Subtree, not only telemetry_profile_id: a remote-only config is one change on the parent path. - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 690b4ed3c74..88b22c6cc98 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,9 +35,8 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend: table_names is consumed to create a profile and -// never echoed back, so GET returns the profile ID and the caller's inference_table_config. -// A config naming neither table_names nor telemetry_profile_id is discarded, reporting success. +// applyTelemetryConfig mirrors the backend: table_names is consumed and not echoed; +// naming neither table_names nor telemetry_profile_id discards the config. func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil @@ -48,8 +47,7 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te applied := serving.TelemetryConfig{TelemetryProfileId: config.TelemetryProfileId} if applied.TelemetryProfileId == "" { - // table_names in the request provisions a profile; assign a fresh ID when the - // caller did not supply one (bundle configs never do). + // Do not reuse previous: table_names provisions a new profile. applied.TelemetryProfileId = nextUUID() } if config.InferenceTableConfig != nil { @@ -64,8 +62,8 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te return &applied } -// telemetrySupported mirrors the backend, which supports telemetry only on custom served -// models, and returns the endpoint type the backend names when it rejects one. +// telemetrySupported is true only for custom served models; otherwise it returns the +// endpoint type the backend names when rejecting (NO_CONFIG / EXTERNAL_MODELS). func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) { if endpoint.Config == nil || len(endpoint.Config.ServedEntities) == 0 { return "NO_CONFIG", false @@ -290,8 +288,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { } } -// ServingEndpointGet reports an in-progress config update once and then settles the -// endpoint, so a caller that polls converges and one that does not sees it in flight. +// ServingEndpointGet reports an in-progress update once, then settles so a poller converges. func (s *FakeWorkspace) ServingEndpointGet(name string) Response { defer s.LockUnlock()() @@ -304,7 +301,7 @@ func (s *FakeWorkspace) ServingEndpointGet(name string) Response { } if endpointUpdating(endpoint) { - // Settle the stored copy for the next read; this response still reports IN_PROGRESS. + // This response stays IN_PROGRESS; settle the stored copy for the next read. settled := endpoint settled.State = &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, @@ -367,8 +364,7 @@ func (s *FakeWorkspace) ServingEndpointUpdate(req Request, name string) Response endpoint.Config = config endpoint.LastUpdatedTimestamp = nowMilli() - // A config update is asynchronous: until it settles the endpoint reports - // IN_PROGRESS, which is what makes a telemetry patch in the same pass fail. + // Leave IN_PROGRESS until the next GET settles it; a same-pass telemetry PATCH must see that. endpoint.State = &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateInProgress, Ready: serving.EndpointStateReadyNotReady, From 0d9ea65364b120af748f471823c93807a775af54 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 11:45:18 +0000 Subject: [PATCH 24/28] Cover telemetry_config with an invariant test instead of a manual one The convergence case the manual drift test asserted is exactly what no_drift checks, so move telemetry_config into the invariant configs and drop the hand-written test. Keep the two cases invariant cannot express: out-of-band telemetry left in place, and the wait ordering a combined config update needs. Also fold the unsupported-endpoint regression into the bundles changelog entry rather than carrying a second fragment under notable-changes. --- ...odel-serving-telemetry-config-converges.md | 2 +- ...-serving-telemetry-unsupported-endpoint.md | 1 - .../model_serving_endpoint_telemetry.yml.tmpl | 20 +++ .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 4 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 5 + .../drift/telemetry_config/databricks.yml | 20 --- .../drift/telemetry_config/out.test.toml | 3 - .../drift/telemetry_config/output.txt | 139 ------------------ .../drift/telemetry_config/script | 36 ----- .../drift/telemetry_config/test.toml | 5 - 14 files changed, 34 insertions(+), 205 deletions(-) delete mode 100644 .nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md create mode 100644 acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry.yml.tmpl delete mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml delete mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml delete mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt delete mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script delete mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index af5dfea897c..8de756d958e 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id`, so every `bundle plan` after a deploy reported an update, and the deploy that followed made no telemetry call at all ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id`, so every `bundle plan` after a deploy reported an update, and the deploy that followed made no telemetry call at all. Wiring the telemetry API also means a `telemetry_config` on an endpoint that serves nothing or only external models now fails on a deploy that applies the field, with the error that API returns (create still drops the field) ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md b/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md deleted file mode 100644 index 8ff7fefbea0..00000000000 --- a/.nextchanges/notable-changes/model-serving-telemetry-unsupported-endpoint.md +++ /dev/null @@ -1 +0,0 @@ -direct: A `telemetry_config` on a model serving endpoint that serves nothing or only external models now fails on a deploy that applies the field via the serving endpoints telemetry API, with the error that API returns. Create still drops the field rather than failing, so the first deploy of a new endpoint can succeed; a later deploy that plans an update then fails. Previously `DoUpdate` never called the telemetry API, so every deploy succeeded and the endpoint received no telemetry ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry.yml.tmpl b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry.yml.tmpl new file mode 100644 index 00000000000..d8c0147e937 --- /dev/null +++ b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + model_serving_endpoints: + foo: + name: test-endpoint-$UNIQUE_NAME + config: + served_entities: + - name: prod + entity_name: main.default.test_model + entity_version: "1" + workload_size: Small + scale_to_zero_enabled: true + telemetry_config: + table_names: + logs_table: main.default.test_logs + traces_table: main.default.test_traces + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index ca367650f71..b06760dbfae 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -23,6 +23,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 9b8f7ea4435..04295a71d21 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 9b8f7ea4435..04295a71d21 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index bb2337b32aa..502866608f2 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -37,3 +37,7 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # migrate deploys via Terraform first, and the TF provider rejects an uppercase # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] + +# The TF provider types sampling_fraction as an integer, so the terraform seed +# deploy truncates 0.5 to 0 and the plan never converges. Covered by no_drift. +EnvMatrixExclude.no_model_serving_endpoint_telemetry = ["INPUT_CONFIG=model_serving_endpoint_telemetry.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 9b8f7ea4435..04295a71d21 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 32a0808dbac..f1bfb88047b 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -48,6 +48,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", @@ -116,6 +117,10 @@ no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_externa # it here to keep the cloud invariant runs from timing out. Still exercised locally. no_vector_search_index_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=vector_search_index.yml.tmpl"] +# Unlike model_serving_endpoint.yml.tmpl, this config serves a real model, which +# telemetry_config requires and the cloud test environment does not provision. +no_model_serving_endpoint_telemetry_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=model_serving_endpoint_telemetry.yml.tmpl"] + # Fake SQL endpoint for local tests [[Server]] Pattern = "POST /api/2.0/sql/statements/" diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml deleted file mode 100644 index 46818f14591..00000000000 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml +++ /dev/null @@ -1,20 +0,0 @@ -bundle: - name: test-bundle - -resources: - model_serving_endpoints: - endpoint1: - name: test-endpoint - config: - served_entities: - - name: prod - entity_name: main.default.test_model - entity_version: "1" - workload_size: Small - scale_to_zero_enabled: true - telemetry_config: # TO_REMOVE - table_names: # TO_REMOVE - logs_table: main.default.test_logs # TO_REMOVE - traces_table: main.default.test_traces # TO_REMOVE - inference_table_config: # TO_REMOVE - sampling_fraction: 0.5 # TO_REMOVE diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml deleted file mode 100644 index e90b6d5d1ba..00000000000 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt deleted file mode 100644 index e1677c4dc12..00000000000 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ /dev/null @@ -1,139 +0,0 @@ - -=== Initial deployment ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Plan converges ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Editing sampling_fraction plans an update ->>> [CLI] bundle plan --output json -{ - "telemetry_config.inference_table_config.name": { - "action": "skip", - "reason": "spec:output_only", - "remote": "main.default.test_logs_payload" - }, - "telemetry_config.inference_table_config.sampling_fraction": { - "action": "update", - "old": 0.5, - "new": 0.9, - "remote": 0.5 - }, - "telemetry_config.table_names": { - "action": "skip", - "reason": "input_only", - "old": { - "logs_table": "main.default.test_logs", - "traces_table": "main.default.test_traces" - }, - "new": { - "logs_table": "main.default.test_logs", - "traces_table": "main.default.test_traces" - } - }, - "telemetry_config.telemetry_profile_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[UUID]" - } -} - -=== Deploy applies the edit ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Plan converges again ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Editing table_names plans an update ->>> [CLI] bundle plan --output json -{ - "telemetry_config.inference_table_config.name": { - "action": "skip", - "reason": "spec:output_only", - "remote": "main.default.test_logs_payload" - }, - "telemetry_config.table_names": { - "action": "update", - "old": { - "logs_table": "main.default.test_logs", - "traces_table": "main.default.test_traces" - }, - "new": { - "logs_table": "main.default.test_logs_v2", - "traces_table": "main.default.test_traces" - } - }, - "telemetry_config.table_names.logs_table": { - "action": "update", - "old": "main.default.test_logs", - "new": "main.default.test_logs_v2" - }, - "telemetry_config.telemetry_profile_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[UUID]" - } -} - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Plan converges after table_names edit ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Removing telemetry_config removes it remotely ->>> [CLI] bundle plan --output json -{ - "telemetry_config": { - "action": "update", - "old": { - "inference_table_config": { - "sampling_fraction": 0.9 - }, - "table_names": { - "logs_table": "main.default.test_logs_v2", - "traces_table": "main.default.test_traces" - } - }, - "remote": { - "inference_table_config": { - "name": "main.default.test_logs_v2_payload", - "sampling_fraction": 0.9 - }, - "telemetry_profile_id": "[UUID]" - } - } -} - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> [CLI] api get /api/2.0/serving-endpoints/test-endpoint -null - -=== Plan converges with telemetry_config removed ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - ->>> print_requests.py --oneline //serving-endpoints -{"method": "POST", "path": "/api/2.0/serving-endpoints", "body": {"config": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Small"}]}, "name": "test-endpoint", "telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}, "table_names": {"logs_table": "main.default.test_logs", "traces_table": "main.default.test_traces"}}}} -{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs", "traces_table": "main.default.test_traces"}}}} -{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs_v2", "traces_table": "main.default.test_traces"}}}} -{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script deleted file mode 100644 index 47dba25bf8f..00000000000 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ /dev/null @@ -1,36 +0,0 @@ -echo "*" > .gitignore - -title "Initial deployment" -trace $CLI bundle deploy - -title "Plan converges" -trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" - -title "Editing sampling_fraction plans an update" -update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" -trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' - -title "Deploy applies the edit" -trace $CLI bundle deploy - -title "Plan converges again" -trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" - -title "Editing table_names plans an update" -update_file.py databricks.yml "main.default.test_logs" "main.default.test_logs_v2" -trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' -trace $CLI bundle deploy - -title "Plan converges after table_names edit" -trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" - -title "Removing telemetry_config removes it remotely" -grep -v TO_REMOVE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' -trace $CLI bundle deploy -MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config - -title "Plan converges with telemetry_config removed" -trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" - -trace print_requests.py --oneline //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml deleted file mode 100644 index 0d20ee379d2..00000000000 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -RecordRequests = true - -# Direct engine only: the terraform provider types sampling_fraction as an integer while -# the API takes a fraction in [0.0, 1.0], so a terraform deploy truncates 0.5 to 0. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 7f9534889bba7779a79ec2695bc9f6a0a965d05e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 12:00:29 +0000 Subject: [PATCH 25/28] Drop Local from the telemetry test snapshots The Local knob was removed in #6196; regenerate the two out.test.toml files this branch adds so they match the current snapshot format. --- .../drift/telemetry_config_unmanaged/out.test.toml | 1 - .../drift/telemetry_config_with_config_update/out.test.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml index e90b6d5d1ba..0938e678987 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml index e90b6d5d1ba..0938e678987 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 1cb9377f627d0e4d886e15e7977271d2ef8a3bc2 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 12:03:12 +0000 Subject: [PATCH 26/28] Drop the Local knob left behind by a concurrent merge #5587 was branched before #6196 removed Local, so it landed still setting it and now fails config loading with an undecoded-key error. --- .../drift/recreated_same_name/out.test.toml | 1 - .../model_serving_endpoints/drift/recreated_same_name/test.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml index fe4076cdf9b..8c52d40aa2d 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml @@ -1,4 +1,3 @@ -Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml index 83e36142b53..fbd03b35ba9 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml @@ -1,6 +1,5 @@ Badness = "After deleting and recreating a model serving endpoint remotely with the same name but a different endpoint_id, bundle plan/deploy ends up with a permanent update on permissions because the V1 permissions API does not delete ACLs immediately when the parent is gone." -Local = true Cloud = true RequiresUnityCatalog = true RecordRequests = false From 200afa8965255335e6e50b637eb65cda9cd24fd0 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 12:40:36 +0000 Subject: [PATCH 27/28] Cover removing telemetry_config Nothing asserted what happens when telemetry_config is dropped from the bundle. The removal relies on the SDK omitting a nil TelemetryConfig so the PATCH body is empty, which the serving API reads as "remove the configuration"; that was an assumption with no test behind it. The wait-ordering test now continues into a removal step and records the empty PATCH, the endpoint reading back null, and the plan converging afterwards. --- .../databricks.yml | 10 +++++----- .../output.txt | 20 ++++++++++++++++++- .../script | 13 +++++++++++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml index 0ab171df466..889f5172c6a 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/databricks.yml @@ -12,8 +12,8 @@ resources: entity_version: "1" workload_size: Small scale_to_zero_enabled: true - telemetry_config: - table_names: - logs_table: main.default.test_logs - inference_table_config: - sampling_fraction: 0.5 + telemetry_config: # TELEMETRY + table_names: # TELEMETRY + logs_table: main.default.test_logs # TELEMETRY + inference_table_config: # TELEMETRY + sampling_fraction: 0.5 # TELEMETRY diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt index 79940b58584..c0f25ba86f1 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Deploy changing config and telemetry_config waits before the telemetry patch +=== Update config and telemetry_config >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... @@ -25,3 +25,21 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged 2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} 1 {"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"inference_table_config": {"sampling_fraction": 0.9}, "table_names": {"logs_table": "main.default.test_logs"}}}} 2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} + +=== Remove telemetry_config +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] api get /api/2.0/serving-endpoints/test-endpoint +null + +=== Plan converges after removal +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Removal sends an empty telemetry PATCH +>>> print_requests.py --method PATCH --oneline //serving-endpoints +{"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script index 6b345736dfa..3df97d41e8c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/script @@ -3,7 +3,7 @@ echo "*" > .gitignore title "Initial deployment" trace $CLI bundle deploy -title "Deploy changing config and telemetry_config waits before the telemetry patch" +title "Update config and telemetry_config" update_file.py databricks.yml "workload_size: Small" "workload_size: Medium" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle deploy @@ -13,3 +13,14 @@ trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, title "Request order: config PUT, poll, telemetry PATCH" trace print_requests.py --get --oneline //serving-endpoints | uniq -c | sed 's/^ *//' + +title "Remove telemetry_config" +grep -v TELEMETRY databricks.yml > updated.yml && mv updated.yml databricks.yml +trace $CLI bundle deploy +MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.0/serving-endpoints/test-endpoint" | jq .telemetry_config + +title "Plan converges after removal" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Removal sends an empty telemetry PATCH" +trace print_requests.py --method PATCH --oneline //serving-endpoints From 6484f94daa712f755d7c706dd330459982ad3251 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 7 Aug 2026 12:40:49 +0000 Subject: [PATCH 28/28] Shorten the comments and changelog added by this change Several comments restated the code or carried the reasoning behind an earlier revision, which belongs in the commit messages. Each one now states the API behaviour or decision the code depends on and stops there. The changelog entry explained the mechanism at length; it now names the fix and keeps the regression for unsupported endpoint types. --- .../model-serving-telemetry-config-converges.md | 2 +- acceptance/bundle/invariant/migrate/test.toml | 3 +-- acceptance/bundle/invariant/test.toml | 3 +-- .../drift/telemetry_config_unmanaged/test.toml | 2 +- .../test.toml | 2 +- .../direct/dresources/model_serving_endpoint.go | 5 ++--- bundle/direct/dresources/resources.yml | 4 ++-- libs/testserver/serving_endpoints.go | 16 ++++++++-------- 8 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md index 8de756d958e..e06f76c7610 100644 --- a/.nextchanges/bundles/model-serving-telemetry-config-converges.md +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -1 +1 @@ -direct: Fixed `telemetry_config` on model serving endpoints never converging, and edits to it being planned and then silently dropped. The backend creates a telemetry profile from `table_names` and returns only the resulting `telemetry_profile_id`, so every `bundle plan` after a deploy reported an update, and the deploy that followed made no telemetry call at all. Wiring the telemetry API also means a `telemetry_config` on an endpoint that serves nothing or only external models now fails on a deploy that applies the field, with the error that API returns (create still drops the field) ([#6106](https://github.com/databricks/cli/pull/6106)). +direct: Fixed model serving `telemetry_config` drift and applied planned telemetry updates. Unsupported endpoint types now fail when telemetry is applied; create may still succeed because it drops the field ([#6106](https://github.com/databricks/cli/pull/6106)). diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 502866608f2..9d79e11bef9 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -38,6 +38,5 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] -# The TF provider types sampling_fraction as an integer, so the terraform seed -# deploy truncates 0.5 to 0 and the plan never converges. Covered by no_drift. +# Terraform types sampling_fraction as an integer and truncates 0.5; covered by no_drift. EnvMatrixExclude.no_model_serving_endpoint_telemetry = ["INPUT_CONFIG=model_serving_endpoint_telemetry.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index f1bfb88047b..4d0777fdea3 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -117,8 +117,7 @@ no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_externa # it here to keep the cloud invariant runs from timing out. Still exercised locally. no_vector_search_index_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=vector_search_index.yml.tmpl"] -# Unlike model_serving_endpoint.yml.tmpl, this config serves a real model, which -# telemetry_config requires and the cloud test environment does not provision. +# Telemetry requires a real model, which cloud invariant tests do not provision. no_model_serving_endpoint_telemetry_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=model_serving_endpoint_telemetry.yml.tmpl"] # Fake SQL endpoint for local tests diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml index 62f6a4c7a8b..ec0e9c5ca86 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/test.toml @@ -1,4 +1,4 @@ RecordRequests = true -# Direct engine only, where the classification lives. +# Telemetry classification is direct-engine-specific. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml index b11179c5a12..49a9063424b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/test.toml @@ -1,4 +1,4 @@ RecordRequests = true -# Direct engine only, where the DoUpdate wait lives. +# Telemetry updates are direct-engine-specific. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 9804054e655..c0cc8f82598 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -247,7 +247,7 @@ func (r *ResourceModelServingEndpoint) updateNotifications(ctx context.Context, return nil } -// updateTelemetryConfig applies telemetry_config; a nil config removes it. +// updateTelemetryConfig removes telemetry when telemetryConfig is nil. func (r *ResourceModelServingEndpoint) updateTelemetryConfig(ctx context.Context, id string, telemetryConfig *serving.TelemetryConfig) error { req := serving.PatchTelemetryConfigRequest{ Name: id, @@ -358,8 +358,7 @@ func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, } if entry.Changes.HasChange(pathTelemetryConfig) { - // The telemetry API rejects an endpoint still applying an update, and - // WaitAfterUpdate runs too late to order the earlier calls with this patch. + // The telemetry API rejects endpoints with an update in progress. _, err = r.waitForEndpointReady(ctx, id) if err != nil { return nil, err diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 00117077d1c..0c9c9bf5319 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -285,7 +285,7 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # Consumed to build a profile and never returned; inference_table_config round-trips so it is not listed. + # table_names creates a profile and is not returned; inference_table_config round-trips. - field: telemetry_config.table_names reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -319,7 +319,7 @@ resources: - field: rate_limits reason: not_implemented backend_defaults: - # Subtree, not only telemetry_profile_id: a remote-only config is one change on the parent path. + # Remote-only telemetry is reported as a change at this parent path. - field: telemetry_config # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index 88b22c6cc98..efa10d47463 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -35,8 +35,7 @@ func servedEntitiesInputToOutput(input []serving.ServedEntityInput) []serving.Se return entities } -// applyTelemetryConfig mirrors the backend: table_names is consumed and not echoed; -// naming neither table_names nor telemetry_profile_id discards the config. +// applyTelemetryConfig consumes table_names and discards configs that identify no profile. func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.TelemetryConfig { if config == nil { return nil @@ -62,8 +61,7 @@ func applyTelemetryConfig(previous, config *serving.TelemetryConfig) *serving.Te return &applied } -// telemetrySupported is true only for custom served models; otherwise it returns the -// endpoint type the backend names when rejecting (NO_CONFIG / EXTERNAL_MODELS). +// telemetrySupported returns the unsupported endpoint type when telemetry cannot be applied. func telemetrySupported(endpoint serving.ServingEndpointDetailed) (string, bool) { if endpoint.Config == nil || len(endpoint.Config.ServedEntities) == 0 { return "NO_CONFIG", false @@ -276,7 +274,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { ForceSendFields: append(createReq.ForceSendFields, "PermissionLevel", "RouteOptimized", "Description"), } - // Unlike the telemetry API, create drops telemetry it cannot apply instead of failing. + // Create drops unsupported telemetry, while the telemetry API rejects it. if _, ok := telemetrySupported(endpoint); !ok { endpoint.TelemetryConfig = nil } @@ -288,7 +286,7 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { } } -// ServingEndpointGet reports an in-progress update once, then settles so a poller converges. +// ServingEndpointGet reports an in-progress update once before settling it. func (s *FakeWorkspace) ServingEndpointGet(name string) Response { defer s.LockUnlock()() @@ -313,6 +311,7 @@ func (s *FakeWorkspace) ServingEndpointGet(name string) Response { return Response{Body: endpoint} } +// endpointUpdating reports whether a config update is in progress. func endpointUpdating(endpoint serving.ServingEndpointDetailed) bool { return endpoint.State != nil && endpoint.State.ConfigUpdate == serving.EndpointStateConfigUpdateInProgress } @@ -364,7 +363,7 @@ func (s *FakeWorkspace) ServingEndpointUpdate(req Request, name string) Response endpoint.Config = config endpoint.LastUpdatedTimestamp = nowMilli() - // Leave IN_PROGRESS until the next GET settles it; a same-pass telemetry PATCH must see that. + // Keep the update in progress until GET observes it. endpoint.State = &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateInProgress, Ready: serving.EndpointStateReadyNotReady, @@ -446,6 +445,7 @@ func (s *FakeWorkspace) ServingEndpointUpdateNotifications(req Request, name str } } +// ServingEndpointPatchTelemetryConfig applies telemetry after validating endpoint state. func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name string) Response { defer s.LockUnlock()() @@ -476,7 +476,7 @@ func (s *FakeWorkspace) ServingEndpointPatchTelemetryConfig(req Request, name st } } - // The telemetry API refuses to run while an earlier update is still applying. + // The telemetry API returns 409 while another update is in progress. if endpointUpdating(endpoint) { return Response{ StatusCode: 409,