[Fix] Invert creation of storage credential and metastore (#143)

* Invert creation of storage credential and metastore

* Update example

* small tweak

* add sleep

Author: mgyucht

modules/aws-databricks-unity-catalog/README.md

@@ -0,0 +1,10 @@
+# Module aws-databricks-unity-catalog
+
+## Description
+
+This module creates a Databricks Unity Catalog Metastore and Storage Credential for said metastore. It automatically creates an IAM role suitable for Unity Catalog and attaches it to the newly created metastore.
+
+## Required Providers
+
+- `databricks` - only account-level providers are supported.
+- `aws`

modules/aws-databricks-unity-catalog/main.tf

@@ -1,3 +1,8 @@
+locals {
+  iam_role_name = "${var.prefix}-unity-catalog-metastore-access"
+  iam_role_arn = "arn:aws:iam::${var.aws_account_id}:role/${local.iam_role_name}"
+}
+
 resource "databricks_metastore" "this" {
   name          = local.metastore_name
   region        = var.region
@@ -8,23 +13,11 @@ resource "databricks_metastore" "this" {
 
 resource "databricks_metastore_data_access" "this" {
   metastore_id = databricks_metastore.this.id
-  name         = aws_iam_role.metastore_data_access.name
+  name         = local.iam_role_name
   aws_iam_role {
-    role_arn = aws_iam_role.metastore_data_access.arn
+    role_arn = local.iam_role_arn
   }
   is_default = true
-  depends_on = [
-    resource.time_sleep.wait_role_creation
-  ]
-}
-
-# Sleeping for 20s to wait for the workspace to enable identity federation
-resource "time_sleep" "wait_role_creation" {
-  depends_on = [
-    resource.aws_iam_role.metastore_data_access,
-    resource.databricks_metastore.this
-  ]
-  create_duration = "20s"
 }
 
 resource "databricks_metastore_assignment" "default_metastore" {

modules/aws-databricks-unity-catalog/uc_cross_account_role.tf

@@ -11,9 +11,10 @@ data "aws_iam_policy_document" "passrole_for_uc" {
     condition {
       test     = "StringEquals"
       variable = "sts:ExternalId"
-      values   = [var.databricks_account_id]
+      values   = [databricks_metastore_data_access.this.aws_iam_role.0.external_id]
     }
   }
+
   statement {
     sid     = "ExplicitSelfRoleAssumption"
     effect  = "Allow"
@@ -25,7 +26,7 @@ data "aws_iam_policy_document" "passrole_for_uc" {
     condition {
       test     = "ArnLike"
       variable = "aws:PrincipalArn"
-      values   = ["arn:aws:iam::${var.aws_account_id}:role/${var.prefix}-uc-access"]
+      values   = [local.iam_role_arn]
     }
   }
 }
@@ -54,15 +55,13 @@ resource "aws_iam_policy" "unity_metastore" {
         "Action" : [
           "sts:AssumeRole"
         ],
-        "Resource" : [
-          "arn:aws:iam::${var.aws_account_id}:role/${var.prefix}-uc-access"
-        ],
+        "Resource" : [local.iam_role_arn],
         "Effect" : "Allow"
       }
     ]
   })
   tags = merge(var.tags, {
-    Name = "${var.prefix}-unity-catalog IAM policy"
+    Name = "${local.iam_role_name} IAM policy"
   })
 }
 
@@ -95,10 +94,16 @@ resource "aws_iam_policy" "sample_data" {
 }
 
 resource "aws_iam_role" "metastore_data_access" {
-  name                = "${var.prefix}-uc-access"
+  name                = local.iam_role_name
   assume_role_policy  = data.aws_iam_policy_document.passrole_for_uc.json
   managed_policy_arns = [aws_iam_policy.unity_metastore.arn, aws_iam_policy.sample_data.arn]
   tags = merge(var.tags, {
     Name = "${var.prefix}-unity-catalog IAM role"
   })
 }
+
+# Sleeping for 20s to wait for the workspace to enable identity federation
+resource "time_sleep" "wait_role_creation" {
+  depends_on = [aws_iam_role.metastore_data_access]
+  create_duration = "20s"
+}

modules/aws-databricks-unity-catalog/variables.tf

@@ -6,7 +6,7 @@ variable "tags" {
 
 variable "prefix" {
   type        = string
-  description = "(Optional) Prefix to name the resources created by this module"
+  description = "(Required) Prefix to name the resources created by this module"
 }
 
 variable "region" {