diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf index 88cf6a64..dc8bb4d2 100644 --- a/infra/dcp/main.tf +++ b/infra/dcp/main.tf @@ -12,6 +12,10 @@ terraform { source = "hashicorp/random" version = ">= 3.0" } + time = { + source = "hashicorp/time" + version = ">= 0.7.0" + } } } @@ -56,6 +60,14 @@ resource "google_project_service" "apis" { disable_on_destroy = false } +resource "time_sleep" "wait_for_foundation" { + create_duration = "90s" + + depends_on = [ + google_project_service.apis + ] +} + locals { global_config = { project_id = var.project_id @@ -149,5 +161,5 @@ module "stack" { redis_config = local.redis_config ingestion_config = local.ingestion_config - depends_on = [google_project_service.apis] + foundation_dependency = time_sleep.wait_for_foundation.id } diff --git a/infra/dcp/modules/datacommons_services/main.tf b/infra/dcp/modules/datacommons_services/main.tf index 58955cc1..65ebd390 100644 --- a/infra/dcp/modules/datacommons_services/main.tf +++ b/infra/dcp/modules/datacommons_services/main.tf @@ -5,26 +5,12 @@ locals { resource "google_service_account" "serving_sa" { account_id = "${local.name_prefix}dc-srvs-sa" display_name = "Data Commons Serving Service Account" -} -resource "google_project_iam_member" "serving_sa_roles" { - for_each = setsubtract(toset([ - "roles/compute.networkViewer", - "roles/redis.editor", - "roles/storage.objectViewer", - "roles/vpcaccess.user", - # TODO: Review this overly broad permission. - "roles/iam.serviceAccountUser", - "roles/secretmanager.secretAccessor", - "roles/spanner.databaseUser", - "roles/workflows.invoker" - ]), var.use_spanner ? [] : ["roles/spanner.databaseUser"]) - - project = var.project_id - member = "serviceAccount:${google_service_account.serving_sa.email}" - role = each.value + depends_on = [var.foundation_dependency] } + + resource "google_cloud_run_v2_service" "dc_web_service" { name = "${local.name_prefix}dc-datacommons-service" location = var.region @@ -66,6 +52,8 @@ resource "google_cloud_run_v2_service" "dc_web_service" { value = var.project_id } + + dynamic "env" { for_each = var.secret_env_vars content { diff --git a/infra/dcp/modules/datacommons_services/variables.tf b/infra/dcp/modules/datacommons_services/variables.tf index 32a4b795..71c39dad 100644 --- a/infra/dcp/modules/datacommons_services/variables.tf +++ b/infra/dcp/modules/datacommons_services/variables.tf @@ -101,3 +101,11 @@ variable "secret_env_vars" { version = string })) } + +variable "foundation_dependency" { + description = "An artificial dependency to delay resource creation until APIs are ready." + type = any + default = null +} + + diff --git a/infra/dcp/modules/ingestion/dataflow/main.tf b/infra/dcp/modules/ingestion/dataflow/main.tf index 8ea3e86a..56266d2b 100644 --- a/infra/dcp/modules/ingestion/dataflow/main.tf +++ b/infra/dcp/modules/ingestion/dataflow/main.tf @@ -7,23 +7,15 @@ resource "google_service_account" "dataflow_sa" { count = var.deploy ? 1 : 0 account_id = "${local.name_prefix}dc-ing-df-sa" display_name = "Data Commons Ingestion Dataflow SA" -} -resource "google_project_iam_member" "ingestion_spanner_user" { - count = var.deploy && var.use_spanner ? 1 : 0 - project = var.project_id - role = "roles/spanner.databaseUser" - member = "serviceAccount:${google_service_account.dataflow_sa[0].email}" + depends_on = [var.foundation_dependency] } -resource "google_project_iam_member" "dataflow_worker" { - count = var.deploy ? 1 : 0 - project = var.project_id - role = "roles/dataflow.worker" - member = "serviceAccount:${google_service_account.dataflow_sa[0].email}" -} + + + # This is only needed to trigger the services restart to pick up the GCS embeddings change diff --git a/infra/dcp/modules/ingestion/dataflow/variables.tf b/infra/dcp/modules/ingestion/dataflow/variables.tf index e0cd1066..33edfe37 100644 --- a/infra/dcp/modules/ingestion/dataflow/variables.tf +++ b/infra/dcp/modules/ingestion/dataflow/variables.tf @@ -19,3 +19,9 @@ variable "use_spanner" { default = true } +variable "foundation_dependency" { + description = "An artificial dependency to delay resource creation until APIs are ready." + type = any + default = null +} + diff --git a/infra/dcp/modules/ingestion/helper_service/main.tf b/infra/dcp/modules/ingestion/helper_service/main.tf index 731ac35d..282343f0 100644 --- a/infra/dcp/modules/ingestion/helper_service/main.tf +++ b/infra/dcp/modules/ingestion/helper_service/main.tf @@ -82,12 +82,7 @@ resource "google_cloud_run_v2_service" "ingestion_helper" { } } -resource "google_project_iam_member" "helper_spanner_user" { - count = var.deploy && var.use_spanner ? 1 : 0 - project = var.project_id - role = "roles/spanner.databaseUser" - member = "serviceAccount:${google_service_account.helper_sa[0].email}" -} + resource "google_storage_bucket_iam_member" "helper_bucket_access" { @@ -97,15 +92,7 @@ resource "google_storage_bucket_iam_member" "helper_bucket_access" { member = "serviceAccount:${google_service_account.helper_sa[0].email}" } -resource "google_project_iam_member" "helper_bq_roles" { - for_each = toset(var.deploy && var.enable_bigquery_postprocessing ? [ - "roles/bigquery.dataEditor", - "roles/bigquery.jobUser" - ] : []) - project = var.project_id - role = each.value - member = "serviceAccount:${google_service_account.helper_sa[0].email}" -} + resource "google_bigquery_connection_iam_member" "helper_connection_user" { count = var.deploy && var.enable_bigquery_connection ? 1 : 0 diff --git a/infra/dcp/modules/ingestion/preprocessing_job/main.tf b/infra/dcp/modules/ingestion/preprocessing_job/main.tf index 860e1f52..0ed428ef 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/main.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/main.tf @@ -5,6 +5,8 @@ locals { resource "google_service_account" "preprocessing_sa" { account_id = "${local.name_prefix}dc-ing-pre-sa" display_name = "Data Commons Ingestion Preprocessing SA" + + depends_on = [var.foundation_dependency] } resource "google_cloud_run_v2_job" "dc_data_job" { @@ -44,6 +46,8 @@ resource "google_cloud_run_v2_job" "dc_data_job" { } } + + env { name = "GCS_BUCKET" value = var.bucket_name @@ -118,10 +122,6 @@ resource "google_secret_manager_secret_iam_member" "preprocessing_maps_key_acces member = "serviceAccount:${google_service_account.preprocessing_sa.email}" } -resource "google_project_iam_member" "preprocessing_workflow_invoker" { - project = var.project_id - role = "roles/workflows.invoker" - member = "serviceAccount:${google_service_account.preprocessing_sa.email}" -} + diff --git a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf index 07be6f2b..0bdb55ab 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf @@ -42,3 +42,11 @@ variable "maps_api_key_secret_id" { description = "Secret ID for Maps API key" default = "" } + +variable "foundation_dependency" { + description = "An artificial dependency to delay resource creation until APIs are ready." + type = any + default = null +} + + diff --git a/infra/dcp/modules/ingestion/workflow/main.tf b/infra/dcp/modules/ingestion/workflow/main.tf index 648de4d1..a0df53dc 100644 --- a/infra/dcp/modules/ingestion/workflow/main.tf +++ b/infra/dcp/modules/ingestion/workflow/main.tf @@ -23,6 +23,7 @@ resource "google_workflows_workflow" "ingestion_orchestrator" { - init: assign: - project_id: '${var.project_id}' + - workflow_id: '$${sys.get_env("GOOGLE_CLOUD_WORKFLOW_EXECUTION_ID")}' - version: '$${"version-" + string(int(sys.now()))}' - bucket_name: '$${text.split(input.tempLocation, "/")[2]}' @@ -214,9 +215,4 @@ resource "google_cloud_run_v2_service_iam_member" "helper_invoker" { member = "serviceAccount:${google_service_account.workflow_sa[0].email}" } -resource "google_project_iam_member" "workflow_run_viewer" { - count = var.deploy ? 1 : 0 - project = var.project_id - role = "roles/run.viewer" - member = "serviceAccount:${google_service_account.workflow_sa[0].email}" -} + diff --git a/infra/dcp/modules/ingestion/workflow/variables.tf b/infra/dcp/modules/ingestion/workflow/variables.tf index 019c7f44..0f8bfc39 100644 --- a/infra/dcp/modules/ingestion/workflow/variables.tf +++ b/infra/dcp/modules/ingestion/workflow/variables.tf @@ -53,3 +53,5 @@ variable "enable_redis_cache_clearing" { description = "Flag to enable Redis cache clearing" default = false } + + diff --git a/infra/dcp/modules/spanner/main.tf b/infra/dcp/modules/spanner/main.tf index 85889e8f..bc332bb9 100644 --- a/infra/dcp/modules/spanner/main.tf +++ b/infra/dcp/modules/spanner/main.tf @@ -12,6 +12,8 @@ resource "google_spanner_instance" "main" { processing_units = var.processing_units force_destroy = !var.stateful_deletion_protection edition = "ENTERPRISE" + + depends_on = [var.foundation_dependency] } resource "google_spanner_database" "database" { @@ -51,12 +53,7 @@ resource "google_spanner_database_iam_member" "spanner_reader" { member = "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}" } -resource "google_project_iam_member" "bq_sa_spanner_viewer" { - count = var.enable_bigquery_connection ? 1 : 0 - project = var.project_id - role = "roles/spanner.viewer" - member = "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}" -} + # Create the BigQuery Reservation for Federation queries diff --git a/infra/dcp/modules/spanner/variables.tf b/infra/dcp/modules/spanner/variables.tf index df79b3f1..2979f761 100644 --- a/infra/dcp/modules/spanner/variables.tf +++ b/infra/dcp/modules/spanner/variables.tf @@ -75,3 +75,9 @@ variable "bigquery_reservation_max_slots" { description = "Max slots for BigQuery reservation autoscale" default = 400 } + +variable "foundation_dependency" { + description = "An artificial dependency to delay resource creation until APIs are ready." + type = any + default = null +} diff --git a/infra/dcp/modules/stack/main.tf b/infra/dcp/modules/stack/main.tf index 9ab80903..70a3a3ce 100644 --- a/infra/dcp/modules/stack/main.tf +++ b/infra/dcp/modules/stack/main.tf @@ -99,6 +99,7 @@ module "spanner" { create_bigquery_reservation = var.spanner_config.create_bigquery_reservation bigquery_reservation_slot_capacity = var.spanner_config.bigquery_reservation_slot_capacity bigquery_reservation_max_slots = var.spanner_config.bigquery_reservation_max_slots + foundation_dependency = var.foundation_dependency } @@ -114,6 +115,8 @@ module "storage" { # Shared vars project_id = var.global.project_id namespace = var.global.namespace + + foundation_dependency = var.foundation_dependency } module "ingestion_preprocessing_job" { @@ -138,6 +141,7 @@ module "ingestion_preprocessing_job" { secret_env_vars = local.datacommons_services_secrets dc_api_key_secret_id = module.auth.dc_api_key_secret_id maps_api_key_secret_id = module.auth.maps_api_key_secret_id + foundation_dependency = var.foundation_dependency depends_on = [module.auth] } @@ -150,6 +154,7 @@ module "ingestion_dataflow" { namespace = var.global.namespace ingestion_bucket_name = module.storage.artifacts_bucket_name use_spanner = var.spanner_config.enable + foundation_dependency = var.foundation_dependency } @@ -249,6 +254,7 @@ module "datacommons_services" { use_spanner = var.spanner_config.enable env_vars = local.cloud_run_shared_env_variables secret_env_vars = local.datacommons_services_secrets + foundation_dependency = var.foundation_dependency depends_on = [module.ingestion_preprocessing_job] } @@ -352,6 +358,81 @@ resource "google_cloud_run_v2_service_iam_member" "workflow_serving_developer" { member = "serviceAccount:${module.ingestion_workflow.service_account_email}" } +data "google_bigquery_default_service_account" "bq_sa" { + project = var.global.project_id +} + +resource "google_project_iam_member" "bq_sa_spanner_viewer" { + count = var.spanner_config.enable && var.spanner_config.enable_bigquery_connection ? 1 : 0 + project = var.global.project_id + role = "roles/spanner.viewer" + member = "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}" +} + +resource "google_project_iam_member" "serving_sa_roles" { + for_each = var.datacommons_services_config.enable ? setsubtract(toset([ + "roles/compute.networkViewer", + "roles/redis.editor", + "roles/storage.objectViewer", + "roles/vpcaccess.user", + "roles/iam.serviceAccountUser", + "roles/secretmanager.secretAccessor", + "roles/spanner.databaseUser", + "roles/workflows.invoker" + ]), var.spanner_config.enable ? [] : ["roles/spanner.databaseUser"]) : [] + + project = var.global.project_id + member = "serviceAccount:${module.datacommons_services[0].service_account_email}" + role = each.value +} + +resource "google_project_iam_member" "helper_spanner_user" { + count = var.ingestion_config.enable_ingestion && var.spanner_config.enable ? 1 : 0 + project = var.global.project_id + role = "roles/spanner.databaseUser" + member = "serviceAccount:${module.ingestion_helper_service.service_account_email}" +} + +resource "google_project_iam_member" "helper_bq_roles" { + for_each = toset(var.ingestion_config.enable_ingestion && var.ingestion_config.workflow_enable_bigquery_postprocessing ? [ + "roles/bigquery.dataEditor", + "roles/bigquery.jobUser" + ] : []) + project = var.global.project_id + role = each.value + member = "serviceAccount:${module.ingestion_helper_service.service_account_email}" +} + +resource "google_project_iam_member" "workflow_run_viewer" { + count = var.ingestion_config.enable_ingestion ? 1 : 0 + project = var.global.project_id + role = "roles/run.viewer" + member = "serviceAccount:${module.ingestion_workflow.service_account_email}" +} + +resource "google_project_iam_member" "preprocessing_workflow_invoker" { + count = var.ingestion_config.enable_ingestion ? 1 : 0 + project = var.global.project_id + role = "roles/workflows.invoker" + member = "serviceAccount:${module.ingestion_preprocessing_job[0].service_account_email}" +} + +resource "google_project_iam_member" "ingestion_spanner_user" { + count = var.ingestion_config.enable_ingestion && var.spanner_config.enable ? 1 : 0 + project = var.global.project_id + role = "roles/spanner.databaseUser" + member = "serviceAccount:${module.ingestion_dataflow.service_account_email}" +} + +resource "google_project_iam_member" "dataflow_worker" { + count = var.ingestion_config.enable_ingestion ? 1 : 0 + project = var.global.project_id + role = "roles/dataflow.worker" + member = "serviceAccount:${module.ingestion_dataflow.service_account_email}" +} + + + diff --git a/infra/dcp/modules/stack/variables.tf b/infra/dcp/modules/stack/variables.tf index 1e75aa8f..43a13677 100644 --- a/infra/dcp/modules/stack/variables.tf +++ b/infra/dcp/modules/stack/variables.tf @@ -98,3 +98,9 @@ variable "ingestion_config" { helper_service_image = string }) } + +variable "foundation_dependency" { + description = "An artificial dependency to delay resource creation until APIs are ready." + type = any + default = null +} diff --git a/infra/dcp/modules/storage/main.tf b/infra/dcp/modules/storage/main.tf index 6f2afde2..45f13ed5 100644 --- a/infra/dcp/modules/storage/main.tf +++ b/infra/dcp/modules/storage/main.tf @@ -9,5 +9,7 @@ resource "google_storage_bucket" "artifacts_bucket" { location = var.region uniform_bucket_level_access = true force_destroy = !var.stateful_deletion_protection + + depends_on = [var.foundation_dependency] } diff --git a/infra/dcp/modules/storage/variables.tf b/infra/dcp/modules/storage/variables.tf index a3512823..08047630 100644 --- a/infra/dcp/modules/storage/variables.tf +++ b/infra/dcp/modules/storage/variables.tf @@ -24,3 +24,9 @@ variable "stateful_deletion_protection" { description = "Enable deletion protection for stateful resources (GCS) to prevent data loss." } +variable "foundation_dependency" { + description = "An artificial dependency to delay resource creation until APIs are ready." + type = any + default = null +} +