Skip to content

Commit 543f337

Browse files
owlleg6owlleg6
authored
Merge pull request #17 from data-platform-hq/fix_refactor
feat: refactoring
2 parents 2bd44d9 + 9a72e4e commit 543f337

File tree

8 files changed

+280
-139
lines changed

8 files changed

+280
-139
lines changed

‎README.md

+20-19
Large diffs are not rendered by default.

‎cluster.tf

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
resource "databricks_cluster" "cluster" {
2+
for_each = { for cluster in var.clusters : cluster.cluster_name => cluster }
3+
4+
cluster_name = each.value.cluster_name
5+
spark_version = each.value.spark_version
6+
spark_conf = each.value.spark_conf
7+
spark_env_vars = each.value.spark_env_vars
8+
data_security_mode = each.value.data_security_mode
9+
node_type_id = each.value.node_type_id
10+
autotermination_minutes = each.value.autotermination_minutes
11+
12+
autoscale {
13+
min_workers = each.value.min_workers
14+
max_workers = each.value.max_workers
15+
}
16+
17+
azure_attributes {
18+
availability = each.value.availability
19+
first_on_demand = each.value.first_on_demand
20+
spot_bid_max_price = each.value.spot_bid_max_price
21+
}
22+
23+
dynamic "cluster_log_conf" {
24+
for_each = each.value.cluster_log_conf_destination != null ? [each.value.cluster_log_conf_destination] : []
25+
content {
26+
dbfs {
27+
destination = cluster_log_conf.value
28+
}
29+
}
30+
}
31+
32+
lifecycle {
33+
ignore_changes = [
34+
state
35+
]
36+
}
37+
}

‎main.tf

+23-8
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,21 @@ locals {
33
suffix = length(var.suffix) == 0 ? "" : "-${var.suffix}"
44
}
55

6+
data "azurerm_key_vault_secret" "sp_client_id" {
7+
name = var.sp_client_id_secret_name
8+
key_vault_id = var.key_vault_id
9+
}
10+
11+
data "azurerm_key_vault_secret" "sp_key" {
12+
name = var.sp_key_secret_name
13+
key_vault_id = var.key_vault_id
14+
}
15+
16+
data "azurerm_key_vault_secret" "tenant_id" {
17+
name = var.tenant_id_secret_name
18+
key_vault_id = var.key_vault_id
19+
}
20+
621
resource "databricks_workspace_conf" "this" {
722
count = local.ip_rules == null ? 0 : 1
823

@@ -32,14 +47,14 @@ resource "databricks_sql_endpoint" "this" {
3247
for_each = { for endpoint in var.sql_endpoint : (endpoint.name) => endpoint }
3348

3449
name = "${each.key}${local.suffix}"
35-
cluster_size = coalesce(each.value.cluster_size, "2X-Small")
36-
min_num_clusters = coalesce(each.value.min_num_clusters, 0)
37-
max_num_clusters = coalesce(each.value.max_num_clusters, 1)
38-
auto_stop_mins = coalesce(each.value.auto_stop_mins, "30")
39-
enable_photon = coalesce(each.value.enable_photon, false)
40-
enable_serverless_compute = coalesce(each.value.enable_serverless_compute, false)
41-
spot_instance_policy = coalesce(each.value.spot_instance_policy, "COST_OPTIMIZED")
42-
warehouse_type = coalesce(each.value.warehouse_type, "PRO")
50+
cluster_size = each.value.cluster_size
51+
min_num_clusters = each.value.min_num_clusters
52+
max_num_clusters = each.value.max_num_clusters
53+
auto_stop_mins = each.value.auto_stop_mins
54+
enable_photon = each.value.enable_photon
55+
enable_serverless_compute = each.value.enable_serverless_compute
56+
spot_instance_policy = each.value.spot_instance_policy
57+
warehouse_type = each.value.warehouse_type
4358

4459
lifecycle {
4560
ignore_changes = [state, num_clusters]

‎mount.tf

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
resource "databricks_mount" "adls" {
2+
for_each = var.mountpoints
3+
4+
name = each.key
5+
uri = "abfss://${each.value["container_name"]}@${each.value["storage_account_name"]}.dfs.core.windows.net"
6+
extra_configs = {
7+
"fs.azure.account.auth.type" : "OAuth",
8+
"fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
9+
"fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
10+
"fs.azure.account.oauth2.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key.name].config_reference,
11+
"fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${data.azurerm_key_vault_secret.tenant_id.value}/oauth2/token",
12+
"fs.azure.createRemoteFileSystemDuringInitialization" : "false",
13+
"spark.databricks.sqldw.jdbc.service.principal.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
14+
"spark.databricks.sqldw.jdbc.service.principal.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key.name].config_reference
15+
}
16+
}

‎permissions.tf

+13-31
Original file line numberDiff line numberDiff line change
@@ -6,44 +6,26 @@ locals {
66
])
77
}
88

9-
resource "databricks_permissions" "default_cluster" {
10-
for_each = coalesce(flatten([values(var.iam)[*].default_cluster_permission, "none"])...) != "none" ? var.default_cluster_id : {}
11-
12-
cluster_id = each.value
13-
14-
dynamic "access_control" {
15-
for_each = { for k, v in var.iam : k => v.default_cluster_permission if v.default_cluster_permission != null }
16-
content {
17-
group_name = databricks_group.this[access_control.key].display_name
18-
permission_level = access_control.value
19-
}
20-
}
21-
}
22-
23-
resource "databricks_permissions" "cluster_policy" {
9+
resource "databricks_cluster_policy" "this" {
2410
for_each = {
25-
for policy in var.cluster_policies_object : (policy.name) => policy
26-
if policy.can_use != null
11+
for param in var.custom_cluster_policies : (param.name) => param.definition
12+
if param.definition != null
2713
}
2814

29-
cluster_policy_id = each.value.id
30-
31-
dynamic "access_control" {
32-
for_each = each.value.can_use
33-
content {
34-
group_name = databricks_group.this[access_control.value].display_name
35-
permission_level = "CAN_USE"
36-
}
37-
}
15+
name = each.key
16+
definition = jsonencode(each.value)
3817
}
3918

40-
resource "databricks_permissions" "unity_cluster" {
41-
count = var.unity_cluster_config.permissions != null && var.unity_cluster_enabled ? 1 : 0
19+
resource "databricks_permissions" "clusters" {
20+
for_each = {
21+
for v in var.clusters : (v.cluster_name) => v
22+
if length(v.permissions) != 0
23+
}
4224

43-
cluster_id = databricks_cluster.this[0].id
25+
cluster_id = databricks_cluster.cluster[each.key].id
4426

4527
dynamic "access_control" {
46-
for_each = var.unity_cluster_config.permissions
28+
for_each = each.value.permissions
4729
content {
4830
group_name = databricks_group.this[access_control.value.group_name].display_name
4931
permission_level = access_control.value.permission_level
@@ -54,7 +36,7 @@ resource "databricks_permissions" "unity_cluster" {
5436
resource "databricks_permissions" "sql_endpoint" {
5537
for_each = {
5638
for endpoint in var.sql_endpoint : (endpoint.name) => endpoint
57-
if endpoint.permissions != null
39+
if length(endpoint.permissions) != 0
5840
}
5941

6042
sql_endpoint_id = databricks_sql_endpoint.this[each.key].id

‎secrets.tf

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
locals {
2+
sp_secrets = {
3+
(var.sp_client_id_secret_name) = { value = data.azurerm_key_vault_secret.sp_client_id.value }
4+
(var.sp_key_secret_name) = { value = data.azurerm_key_vault_secret.sp_key.value }
5+
}
6+
7+
secrets_objects_list = flatten([for param in var.secret_scope : [
8+
for secret in param.secrets : {
9+
scope_name = param.scope_name, key = secret.key, string_value = secret.string_value
10+
}] if param.secrets != null
11+
])
12+
}
13+
14+
# Secret Scope with SP secrets for mounting Azure Data Lake Storage
15+
resource "databricks_secret_scope" "main" {
16+
name = "main"
17+
initial_manage_principal = null
18+
}
19+
20+
resource "databricks_secret" "main" {
21+
for_each = local.sp_secrets
22+
23+
key = each.key
24+
string_value = each.value["value"]
25+
scope = databricks_secret_scope.main.id
26+
}
27+
28+
# Custom additional Databricks Secret Scope
29+
resource "databricks_secret_scope" "this" {
30+
for_each = {
31+
for param in var.secret_scope : (param.scope_name) => param
32+
if param.scope_name != null
33+
}
34+
35+
name = each.key
36+
initial_manage_principal = null
37+
}
38+
39+
resource "databricks_secret" "this" {
40+
for_each = { for entry in local.secrets_objects_list : "${entry.scope_name}.${entry.key}" => entry }
41+
42+
key = each.value.key
43+
string_value = each.value.string_value
44+
scope = databricks_secret_scope.this[each.value.scope_name].id
45+
}
46+
47+
# At the nearest future, Azure will allow acquiring AAD tokens by service principals,
48+
# thus providing an ability to create Azure backed Key Vault with Terraform
49+
# https://github.com/databricks/terraform-provider-databricks/pull/1965
50+
51+
## Azure Key Vault-backed Scope
52+
#resource "azurerm_key_vault_access_policy" "databricks" {
53+
# count = var.key_vault_secret_scope.key_vault_id != null ? 1 : 0
54+
55+
# key_vault_id = var.key_vault_secret_scope.key_vault_id
56+
# object_id = "9b38785a-6e08-4087-a0c4-20634343f21f" # Global 'AzureDatabricks' SP object id
57+
# tenant_id = data.azurerm_key_vault_secret.tenant_id.value
58+
#
59+
# secret_permissions = [
60+
# "Get",
61+
# "List",
62+
# ]
63+
#}
64+
#
65+
#resource "databricks_secret_scope" "external" {
66+
# count = var.key_vault_secret_scope.key_vault_id != null ? 1 : 0
67+
#
68+
# name = "external"
69+
# keyvault_metadata {
70+
# resource_id = var.key_vault_secret_scope.key_vault_id
71+
# dns_name = var.key_vault_secret_scope.dns_name
72+
# }
73+
# depends_on = [azurerm_key_vault_access_policy.databricks]
74+
#}

‎unity.tf

-38
Original file line numberDiff line numberDiff line change
@@ -133,41 +133,3 @@ resource "databricks_grants" "schema" {
133133
privileges = each.value.permission
134134
}
135135
}
136-
137-
resource "databricks_cluster" "this" {
138-
count = var.unity_cluster_enabled ? 1 : 0
139-
140-
cluster_name = var.unity_cluster_config.cluster_name
141-
spark_version = var.unity_cluster_config.spark_version
142-
spark_conf = var.unity_cluster_config.spark_conf
143-
spark_env_vars = var.unity_cluster_config.spark_env_vars
144-
data_security_mode = var.unity_cluster_config.data_security_mode
145-
node_type_id = var.unity_cluster_config.node_type_id
146-
autotermination_minutes = var.unity_cluster_config.autotermination_minutes
147-
148-
autoscale {
149-
min_workers = var.unity_cluster_config.min_workers
150-
max_workers = var.unity_cluster_config.max_workers
151-
}
152-
153-
azure_attributes {
154-
availability = var.unity_cluster_config.availability
155-
first_on_demand = var.unity_cluster_config.first_on_demand
156-
spot_bid_max_price = var.unity_cluster_config.spot_bid_max_price
157-
}
158-
159-
dynamic "cluster_log_conf" {
160-
for_each = var.unity_cluster_config.cluster_log_conf_destination != null ? [var.unity_cluster_config.cluster_log_conf_destination] : []
161-
content {
162-
dbfs {
163-
destination = cluster_log_conf.value
164-
}
165-
}
166-
}
167-
168-
lifecycle {
169-
ignore_changes = [
170-
state
171-
]
172-
}
173-
}

0 commit comments

Comments
 (0)