Moving UC Resources to account level (#89)

* Moving UC to account level
Changes:
- Moving UC resources to account level
- Bumping `databricks` provider version
- Adjusting dependencies

* Adjusting versions and admin group definition for module

Author: flaviomalavazi

examples/aws-workspace-uc-simple/main.tf

@@ -13,8 +13,7 @@ module "aws_base" {
 
 module "databricks_workspace" {
   providers = {
-    databricks.mws       = databricks.mws
-    databricks.workspace = databricks.workspace
+    databricks = databricks.mws
   }
   source                 = "../../modules/aws-databricks-workspace"
   prefix                 = local.prefix
@@ -26,26 +25,24 @@ module "databricks_workspace" {
   root_storage_bucket    = module.aws_base.root_bucket
   cross_account_role_arn = module.aws_base.cross_account_role_arn
   tags                   = local.tags
+
   depends_on = [
     module.aws_base
   ]
+
 }
 
 module "unity_catalog" {
   source = "../../modules/aws-databricks-unity-catalog"
   providers = {
-    databricks.mws       = databricks.mws
-    databricks.workspace = databricks.workspace
+    databricks = databricks.mws
   }
   prefix                   = local.prefix
   region                   = var.region
   databricks_account_id    = var.databricks_account_id
   aws_account_id           = local.aws_account_id
-  unity_metastore_owner    = local.unity_admin_group
+  unity_metastore_owner    = databricks_group.admin_group.display_name
   databricks_workspace_ids = [module.databricks_workspace.databricks_workspace_id]
   tags                     = local.tags
-  depends_on = [
-    module.databricks_workspace,
-    resource.databricks_group.admin_group
-  ]
+
 }

examples/aws-workspace-uc-simple/versions.tf

@@ -17,7 +17,7 @@ terraform {
 
     databricks = {
       source  = "databricks/databricks"
-      version = "=1.17.0"
+      version = ">= 1.2.0, < 2.0.0"
     }
 
   }

modules/aws-databricks-base-infra/outputs.tf

@@ -31,4 +31,5 @@ output "root_bucket" {
 output "cross_account_role_arn" {
   value       = aws_iam_role.cross_account_role.arn
   description = "AWS Cross account role arn"
+  depends_on  = [resource.aws_iam_role_policy.this]
 }

modules/aws-databricks-base-infra/versions.tf

@@ -7,7 +7,7 @@ terraform {
 
     databricks = {
       source                = "databricks/databricks"
-      version               = ">=1.17.0"
+      version               = ">=1.24.1"
       configuration_aliases = [databricks.mws]
     }
   }

modules/aws-databricks-unity-catalog/main.tf

@@ -1,13 +1,12 @@
 resource "databricks_metastore" "this" {
-  provider      = databricks.workspace
   name          = local.metastore_name
+  region        = var.region
   owner         = var.unity_metastore_owner
   storage_root  = "s3://${aws_s3_bucket.metastore.id}/metastore"
   force_destroy = true
 }
 
 resource "databricks_metastore_data_access" "this" {
-  provider     = databricks.workspace
   metastore_id = databricks_metastore.this.id
   name         = aws_iam_role.metastore_data_access.name
   aws_iam_role {
@@ -29,7 +28,6 @@ resource "time_sleep" "wait_role_creation" {
 }
 
 resource "databricks_metastore_assignment" "default_metastore" {
-  provider             = databricks.workspace
   count                = length(var.databricks_workspace_ids)
   workspace_id         = var.databricks_workspace_ids[count.index]
   metastore_id         = databricks_metastore.this.id

modules/aws-databricks-unity-catalog/s3.tf

@@ -13,6 +13,16 @@ resource "aws_s3_bucket_versioning" "versioning_example" {
   }
 }
 
+resource "aws_s3_bucket_server_side_encryption_configuration" "root_storage_bucket" {
+  bucket = aws_s3_bucket.metastore.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
 resource "aws_s3_bucket_public_access_block" "metastore" {
   bucket                  = aws_s3_bucket.metastore.id
   block_public_acls       = true

modules/aws-databricks-unity-catalog/versions.tf

@@ -6,9 +6,8 @@ terraform {
     }
 
     databricks = {
-      source                = "databricks/databricks"
-      version               = ">=1.17.0"
-      configuration_aliases = [databricks.mws, databricks.workspace]
+      source  = "databricks/databricks"
+      version = ">=1.24.1"
     }
 
   }

modules/aws-databricks-workspace/main.tf

@@ -1,12 +1,10 @@
 resource "databricks_mws_credentials" "this" {
-  provider         = databricks.mws
   account_id       = var.databricks_account_id
   role_arn         = var.cross_account_role_arn
   credentials_name = "${var.prefix}-creds"
 }
 
 resource "databricks_mws_networks" "this" {
-  provider           = databricks.mws
   account_id         = var.databricks_account_id
   network_name       = "${var.prefix}-network"
   security_group_ids = var.security_group_ids
@@ -15,14 +13,12 @@ resource "databricks_mws_networks" "this" {
 }
 
 resource "databricks_mws_storage_configurations" "this" {
-  provider                   = databricks.mws
   account_id                 = var.databricks_account_id
   bucket_name                = var.root_storage_bucket
   storage_configuration_name = "${var.prefix}-storage"
 }
 
 resource "databricks_mws_workspaces" "this" {
-  provider       = databricks.mws
   account_id     = var.databricks_account_id
   aws_region     = var.region
   workspace_name = var.prefix

modules/aws-databricks-workspace/versions.tf

@@ -1,14 +1,8 @@
 terraform {
   required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "=4.57.0"
-    }
-
     databricks = {
-      source                = "databricks/databricks"
-      version               = ">=1.17.0"
-      configuration_aliases = [databricks.mws, databricks.workspace]
+      source  = "databricks/databricks"
+      version = ">=1.24.1"
     }
 
   }