diff --git a/README.md b/README.md index 94a9d76..70e259b 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ See [example/README.md](example/README.md) for details. deployml destroy ``` -Deletes all Cloud Run services, Cloud SQL, the GCS bucket, and the BigQuery dataset, and also removes the Artifact Registry repo and the Cloud Build staging bucket that `build-images` created, so a destroyed project leaves no billing residue. Does not delete the GCP project itself. +Deletes all Cloud Run services, Cloud SQL, the GCS bucket, and the BigQuery dataset, and also removes the Artifact Registry repo and the Cloud Build staging bucket that `build-images` created, so a destroyed project leaves no billing residue. Pass `--keep-images` if other workspaces in the same project share those images. Does not delete the GCP project itself. ## Full Tutorial diff --git a/docs/api/cli-commands.md b/docs/api/cli-commands.md index 5789780..786bb50 100644 --- a/docs/api/cli-commands.md +++ b/docs/api/cli-commands.md @@ -98,7 +98,7 @@ deployml get-urls --show-secrets ## `deployml destroy` -Tear down all infrastructure for a given config. Also removes the Artifact Registry repo and the Cloud Build staging bucket created by build-images, so a destroyed project leaves no billing residue. +Tear down all infrastructure for a given config. Also removes the Artifact Registry repo and the Cloud Build staging bucket created by build-images, so a destroyed project leaves no billing residue. Pass `--keep-images` when other workspaces in the same project share those images. ```bash deployml destroy --yes @@ -107,8 +107,10 @@ deployml destroy --yes **Options:** - `--config-path`, `-c`: Path to config YAML. Default `config.yaml`. - `--clean-workspace`: Remove the local `.deployml/` workspace folder after destroy. -- `--yes`, `-y`: Skip both the destroy confirmation and the Terraform state cleanup prompt. +- `--yes`, `-y`: Skip both the destroy confirmation and the Terraform state cleanup prompt. Also auto-confirms image cleanup. - `--workspace`: Override the workspace name from config. +- `--keep-images`: Keep the Artifact Registry repo and the Cloud Build staging bucket. Use when other workspaces in the same project still run on those images. +- `--repository`: Artifact Registry repo to delete. Default `mlops-images`. Match the `--repository` you passed to build-images. On partial failure, Terraform state is preserved and the command prints recovery instructions including `gcloud asset search-all-resources` for finding residual resources. diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index 26ed893..ffe4d6d 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -1640,6 +1640,16 @@ def destroy( yes: bool = typer.Option( False, "--yes", "-y", help="Skip confirmation prompts and destroy" ), + keep_images: bool = typer.Option( + False, "--keep-images", + help="Keep the Artifact Registry repo and Cloud Build staging bucket. " + "Use when other workspaces in the same project share the images.", + ), + repository: str = typer.Option( + "mlops-images", "--repository", + help="Artifact Registry repository to delete after destroy. " + "Match the --repository you passed to build-images.", + ), ): """ Destroy infrastructure and optionally clean up workspace and Terraform state files. @@ -1751,27 +1761,40 @@ def destroy( if result.returncode == 0: typer.echo(" Infrastructure destroyed successfully!") - # Clean up the Artifact Registry repo created by build-images. - # Terraform does not manage it, so without this it lingers and bills. + # Clean up the Artifact Registry repo created by build-images and the + # Cloud Build staging bucket. Terraform does not manage either, so + # without this they linger and bill. The repo may be shared by other + # workspaces deployed to the same project, so this is skippable via + # --keep-images and confirmed interactively when not --yes. region = config.get("provider", {}).get("region", "us-central1") - ar_repo = "mlops-images" - typer.echo(f" Removing Artifact Registry repo {ar_repo}...") - run_tool( - "gcloud", ["artifacts", "repositories", "delete", ar_repo, - "--location", region, "--project", project_id, "--quiet"], - capture_output=True, - ) - - # Clean up the Cloud Build staging bucket that `gcloud builds submit` - # auto-creates during build-images. It is not Terraform-managed and - # accumulates source tarballs across cycles. Best-effort; Cloud Build - # recreates it on the next build if needed. cb_bucket = f"gs://{project_id}_cloudbuild" - typer.echo(f" Removing Cloud Build staging bucket {cb_bucket}...") - run_tool( - "gcloud", ["storage", "rm", "--recursive", cb_bucket, "--quiet"], - capture_output=True, - ) + if keep_images: + typer.echo( + f" Keeping Artifact Registry repo {repository} and {cb_bucket} (--keep-images)." + ) + elif yes or typer.confirm( + f"Also delete Artifact Registry repo '{repository}' and Cloud Build " + f"staging bucket {cb_bucket}? Other workspaces in project " + f"'{project_id}' may still use these images.", + default=True, + ): + typer.echo(f" Removing Artifact Registry repo {repository}...") + run_tool( + "gcloud", ["artifacts", "repositories", "delete", repository, + "--location", region, "--project", project_id, "--quiet"], + capture_output=True, + ) + + # The staging bucket auto-created by `gcloud builds submit` accumulates + # source tarballs across cycles. Best-effort; Cloud Build recreates it + # on the next build if needed. + typer.echo(f" Removing Cloud Build staging bucket {cb_bucket}...") + run_tool( + "gcloud", ["storage", "rm", "--recursive", cb_bucket, "--quiet"], + capture_output=True, + ) + else: + typer.echo(f" Keeping Artifact Registry repo {repository} and {cb_bucket}.") if clean_workspace: typer.echo(" Cleaning workspace...") diff --git a/src/deployml/templates/gcp/cloud_run/main.tf.j2 b/src/deployml/templates/gcp/cloud_run/main.tf.j2 index 7ff3a7b..735e666 100644 --- a/src/deployml/templates/gcp/cloud_run/main.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/main.tf.j2 @@ -124,7 +124,8 @@ module "{{ stage_name }}_{{ tool.name }}" { bucket_exists = {% if create_artifact_bucket and tool.params.get("artifact_bucket") %}true{% else %}false{% endif %} {% if flags.needs_postgres %} - backend_store_uri = module.cloud_sql_postgres.connection_string + # Socket DSN. Public IP is blocked; Cloud SQL Auth Proxy via annotation only. + backend_store_uri = module.cloud_sql_postgres.connection_string_cloud_sql cloudsql_instance_annotation = module.cloud_sql_postgres.instance_connection_name {% else %} backend_store_uri = "{{ tool.params.get('backend_store_uri', 'sqlite:///' + tool.name + '.db') }}" @@ -153,7 +154,7 @@ module "{{ stage_name }}_{{ tool.name }}" { # Database connection configuration {% if flags.needs_postgres %} - db_connection_string = module.cloud_sql_postgres.connection_string + db_connection_string = module.cloud_sql_postgres.connection_string_cloud_sql {% else %} db_connection_string = "" {% endif %} @@ -176,7 +177,8 @@ module "{{ stage_name }}_{{ tool.name }}" { # Feast-specific parameters {% if tool.name == "feast" and flags.needs_postgres %} backend_store_uri = module.cloud_sql_postgres.feast_connection_string_cloud_sql - postgres_host = module.cloud_sql_postgres.db_public_ip + # Socket dir, not the public IP: direct TCP to the instance is blocked. + postgres_host = module.cloud_sql_postgres.postgres_host postgres_port = "5432" postgres_database = "feast" postgres_user = module.cloud_sql_postgres.db_user @@ -270,8 +272,11 @@ module "{{ stage_name }}_{{ tool.name }}" { # Database configuration {% if flags.needs_postgres %} - database_url = module.cloud_sql_postgres.connection_string - feast_online_store_host = module.cloud_sql_postgres.db_public_ip + # Socket-based access only: the instance has no authorized networks, so jobs + # must mount the /cloudsql volume (cloud_sql_connection_name) and use socket DSNs. + cloud_sql_connection_name = module.cloud_sql_postgres.instance_connection_name + database_url = module.cloud_sql_postgres.connection_string_cloud_sql + feast_online_store_host = module.cloud_sql_postgres.postgres_host feast_online_store_port = "5432" feast_online_store_database = "feast" feast_online_store_user = module.cloud_sql_postgres.db_user diff --git a/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 b/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 index b69147d..ab4de38 100644 --- a/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 @@ -242,7 +242,8 @@ module "{{ stage_name }}_{{ tool.name }}" { feast_online_store_database = module.cloud_sql_postgres.postgres_database feast_online_store_user = module.cloud_sql_postgres.postgres_user feast_online_store_password = module.cloud_sql_postgres.postgres_password - feast_registry_path = module.cloud_sql_postgres.feast_connection_string + # Socket DSN: the public-IP variant no longer works with authorized networks removed. + feast_registry_path = module.cloud_sql_postgres.feast_connection_string_cloud_sql {% endif %} # Grafana configuration for drift monitoring - check if grafana exists in stack diff --git a/src/deployml/terraform/modules/cloud_sql_postgres/outputs.tf b/src/deployml/terraform/modules/cloud_sql_postgres/outputs.tf index 32ab971..772e31c 100644 --- a/src/deployml/terraform/modules/cloud_sql_postgres/outputs.tf +++ b/src/deployml/terraform/modules/cloud_sql_postgres/outputs.tf @@ -71,7 +71,8 @@ output "grafana_connection_string_cloud_sql" { } output "postgres_host" { - value = google_sql_database_instance.postgres.public_ip_address + description = "Cloud SQL Auth Proxy socket directory, usable as a psycopg/libpq host=. Direct TCP to the public IP is blocked (no authorized networks), so consumers must connect through the /cloudsql socket mounted via the Cloud Run annotation/volume." + value = "/cloudsql/${google_sql_database_instance.postgres.connection_name}" } output "postgres_port" {