Skip to content

Commit 152ebcc

Browse files
nerdynicknerdynick
committed
Initial sweep of Security Group support changes.
1 parent e67d1c4 commit 152ebcc

26 files changed

+541
-42
lines changed

‎README.md

+19-10
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,24 @@
11
# terraform-aws-confluent-platform
22

3-
Terraform Module for Deploying the Confluent Platform to AWS.
3+
Terraform Module(s) for Deploying the Confluent Platform within AWS.
44

55
# Features
66

7-
| CP Component | EC2 Instance | Route53 DNS | Security Groups |
8-
|:--------------- |:------------:|:-----------:|:---------------:|
9-
| Zookeeper | X | X | |
10-
| Kafka Broker | X | X | |
11-
| Kafka Connect | X | X | |
12-
| ksqlDB | X | X | |
13-
| Rest Proxy | X | X | |
14-
| Schema Registry | X | X | |
15-
| Control Center | X | X | |
7+
## Feature Metric
8+
9+
| CP Component | EC2 Instance | Route53 DNS | Security Groups | Load Balancers | Multi AZ |
10+
|:--------------- |:------------:|:-----------:|:---------------:|:--------------:|:--------:|
11+
| Zookeeper | X | X | X | N/A | |
12+
| Kafka Broker | X | X | X | | |
13+
| Kafka Connect | X | X | X | | |
14+
| ksqlDB | X | X | X | | |
15+
| Rest Proxy | X | X | X | | |
16+
| Schema Registry | X | X | X | | |
17+
| Control Center | X | X | X | N/A | |
18+
19+
## Feature Limitations
20+
21+
1. Out of the box all nodes and security groups are required to live in the same VPC
22+
23+
NOTE: If you leverage the individual component modules, some of these limitations can be worked around.
24+
These limitation just haven't be able to be baked into a single unified parent module yet, or may not be possible to at all.

‎main.tf

+41
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@ module "cp-aws-zookeeper" {
1313
dns_ttl = var.zookeeper_dns_ttl
1414
name_template = var.zookeeper_name_template
1515
dns_template = var.zookeeper_dns_template
16+
sg_name = var.zookeeper_sg_name
17+
kafka_broker_sg_id = module.cp-aws-kafka_broker.security_group.id
18+
1619
extra_template_vars = var.extra_template_vars
20+
vpc_id = var.vpc_id
21+
enable_sg_creation = var.enable_sg_creation
1722
}
1823

1924
module "cp-aws-kafka_broker" {
@@ -31,7 +36,16 @@ module "cp-aws-kafka_broker" {
3136
dns_ttl = var.kafka_broker_dns_ttl
3237
name_template = var.kafka_broker_name_template
3338
dns_template = var.kafka_broker_dns_template
39+
sg_name = var.kafka_broker_sg_name
40+
kafka_connect_sg_ids = [module.cp-aws-kafka_connect.security_group.id]
41+
ksql_sg_ids = [module.cp-aws-ksql.security_group.id]
42+
rest_proxy_sg_id = module.cp-aws-rest_proxy.security_group.id
43+
schema_registry_sg_id = module.cp-aws-schema_registry.security_group.id
44+
control_center_sg_id = module.cp-aws-control_center.security_group.id
45+
3446
extra_template_vars = var.extra_template_vars
47+
vpc_id = var.vpc_id
48+
enable_sg_creation = var.enable_sg_creation
3549
}
3650

3751
module "cp-aws-kafka_connect" {
@@ -49,7 +63,12 @@ module "cp-aws-kafka_connect" {
4963
dns_ttl = var.kafka_connect_dns_ttl
5064
name_template = var.kafka_connect_name_template
5165
dns_template = var.kafka_connect_dns_template
66+
sg_name = var.kafka_connect_sg_name
67+
control_center_sg_id = module.cp-aws-control_center.security_group.id
68+
5269
extra_template_vars = var.extra_template_vars
70+
vpc_id = var.vpc_id
71+
enable_sg_creation = var.enable_sg_creation
5372
}
5473

5574
module "cp-aws-control_center" {
@@ -67,7 +86,11 @@ module "cp-aws-control_center" {
6786
dns_ttl = var.control_center_dns_ttl
6887
name_template = var.control_center_name_template
6988
dns_template = var.control_center_dns_template
89+
sg_name = var.control_center_sg_name
90+
7091
extra_template_vars = var.extra_template_vars
92+
vpc_id = var.vpc_id
93+
enable_sg_creation = var.enable_sg_creation
7194
}
7295

7396
module "cp-aws-ksql" {
@@ -85,7 +108,12 @@ module "cp-aws-ksql" {
85108
dns_ttl = var.ksql_dns_ttl
86109
name_template = var.ksql_name_template
87110
dns_template = var.ksql_dns_template
111+
sg_name = var.ksql_sg_name
112+
control_center_sg_id = module.cp-aws-control_center.security_group.id
113+
88114
extra_template_vars = var.extra_template_vars
115+
vpc_id = var.vpc_id
116+
enable_sg_creation = var.enable_sg_creation
89117
}
90118

91119
module "cp-aws-rest_proxy" {
@@ -103,7 +131,11 @@ module "cp-aws-rest_proxy" {
103131
dns_ttl = var.rest_proxy_dns_ttl
104132
name_template = var.rest_proxy_name_template
105133
dns_template = var.rest_proxy_dns_template
134+
sg_name = var.rest_proxy_sg_name
135+
106136
extra_template_vars = var.extra_template_vars
137+
vpc_id = var.vpc_id
138+
enable_sg_creation = var.enable_sg_creation
107139
}
108140

109141
module "cp-aws-schema_registry" {
@@ -121,5 +153,14 @@ module "cp-aws-schema_registry" {
121153
dns_ttl = var.schema_registry_dns_ttl
122154
name_template = var.schema_registry_name_template
123155
dns_template = var.schema_registry_dns_template
156+
sg_name = var.schema_registry_sg_name
157+
kafka_connect_sg_ids = [module.cp-aws-kafka_connect.security_group.id]
158+
ksql_sg_ids = [module.cp-aws-ksql.security_group.id]
159+
rest_proxy_sg_id = module.cp-aws-rest_proxy.security_group.id
160+
schema_registry_sg_id = module.cp-aws-schema_registry.security_group.id
161+
control_center_sg_id = module.cp-aws-control_center.security_group.id
162+
124163
extra_template_vars = var.extra_template_vars
164+
vpc_id = var.vpc_id
165+
enable_sg_creation = var.enable_sg_creation
125166
}

‎modules/base_node/main.tf

+9-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,15 @@ resource "aws_instance" "instance" {
2323
vpc_security_group_ids = var.security_groups_ids
2424

2525

26-
tags = merge(var.tags, {"name"=data.template_file.node_name[count.index].rendered, "Name"=data.template_file.node_name[count.index].rendered})
27-
volume_tags = merge(var.tags, {"name"=data.template_file.node_name[count.index].rendered, "Name"=data.template_file.node_name[count.index].rendered})
26+
tags = merge(var.tags, {
27+
"name"=data.template_file.node_name[count.index].rendered,
28+
"Name"=data.template_file.node_name[count.index].rendered
29+
})
30+
31+
volume_tags = merge(var.tags, {
32+
"name"=data.template_file.node_name[count.index].rendered,
33+
"Name"=data.template_file.node_name[count.index].rendered
34+
})
2835

2936
root_block_device {
3037
volume_size = var.root_volume_size

‎modules/control_center/main.tf

+20-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,22 @@
1-
module "cp-aws-control_center" {
1+
resource "aws_security_group" "my_security_group" {
2+
count = var.enable_sg_creation ? 1 : 0
3+
name = var.sg_name
4+
description = "Confluent Platform - Control Center"
5+
vpc_id = var.vpc_id
6+
7+
tags = var.tags
8+
9+
#Control Center Related
10+
ingress {
11+
description = "C3 - REST Interface - Internal"
12+
from_port = 9021
13+
to_port = 9021
14+
protocol = "tcp"
15+
self = true
16+
}
17+
}
18+
19+
module "my_instance" {
220
source = "../base_node"
321

422
extra_template_vars = var.extra_template_vars
@@ -10,7 +28,7 @@ module "cp-aws-control_center" {
1028
key_pair = var.key_pair
1129
tags = var.tags
1230
subnet_id = var.subnet_id
13-
security_groups_ids = var.security_groups_ids
31+
security_groups_ids = combine(var.security_groups_ids, [aws_security_group.my_security_group.id])
1432
dns_zone_id = var.dns_zone_id
1533
dns_ttl = var.dns_ttl
1634
name_template = var.name_template

‎modules/control_center/outputs.tf

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
output "instances" {
2-
value = module.cp-aws-control_center.instances
2+
value = module.my_instance.instances
33
}
44

55
output "dns_records" {
6-
value = module.cp-aws-control_center.dns_records
6+
value = module.my_instance.dns_records
7+
}
8+
9+
output "security_group" {
10+
value = aws_security_group.my_security_group
711
}

‎modules/control_center/variables.tf

+13
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,17 @@ variable "dns_template" {
5555
variable "extra_template_vars" {
5656
type = map
5757
default = {}
58+
}
59+
60+
#SG Related Vars
61+
variable "vpc_id" {
62+
type = string
63+
}
64+
variable "enable_sg_creation" {
65+
type = boolean
66+
default = true
67+
}
68+
variable "sg_name" {
69+
type = string
70+
default = "CP_Control_Center"
5871
}

‎modules/kafka_broker/main.tf

+52-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,54 @@
1-
module "cp-aws-kafka_broker" {
1+
resource "aws_security_group" "my_security_group" {
2+
count = var.enable_sg_creation ? 1 : 0
3+
name = var.sg_name
4+
description = "Confluent Platform - Kafka Brokers/Confluent Servers"
5+
vpc_id = var.vpc_id
6+
7+
tags = var.tags
8+
9+
#Kafka/Confluent Server Related
10+
ingress {
11+
description = "Kafka - Listeners - Internal Access"
12+
from_port = 9091
13+
to_port = 9093
14+
protocol = "tcp"
15+
self = true
16+
}
17+
18+
ingress {
19+
description = "Kafka - Listeners - External Access"
20+
from_port = 9092
21+
to_port = 9093
22+
protocol = "tcp"
23+
security_groups = combine([
24+
var.rest_proxy_sg_id,
25+
var.schema_registry_sg_id,
26+
var.control_center_sg_id
27+
], var.kafka_connect_sg_ids, var.ksql_sg_ids)
28+
}
29+
30+
ingress {
31+
description = "Kafka - MDS Listeners - Internal Access"
32+
from_port = 8090
33+
to_port = 8091
34+
protocol = "tcp"
35+
self = true
36+
}
37+
38+
ingress {
39+
description = "Kafka - MDS Listeners - External Access"
40+
from_port = 8090
41+
to_port = 8091
42+
protocol = "tcp"
43+
security_groups = combine([
44+
var.rest_proxy_sg_id,
45+
var.schema_registry_sg_id,
46+
var.control_center_sg_id
47+
], var.kafka_connect_sg_ids, var.ksql_sg_ids)
48+
}
49+
}
50+
51+
module "my_instance" {
252
source = "../base_node"
353

454
extra_template_vars = var.extra_template_vars
@@ -10,7 +60,7 @@ module "cp-aws-kafka_broker" {
1060
key_pair = var.key_pair
1161
tags = var.tags
1262
subnet_id = var.subnet_id
13-
security_groups_ids = var.security_groups_ids
63+
security_groups_ids = combine(var.security_groups_ids, [aws_security_group.my_security_group.id])
1464
dns_zone_id = var.dns_zone_id
1565
dns_ttl = var.dns_ttl
1666
name_template = var.name_template

‎modules/kafka_broker/outputs.tf

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
output "instances" {
2-
value = module.cp-aws-kafka_broker.instances
2+
value = module.my_instance.instances
33
}
44

55
output "dns_records" {
6-
value = module.cp-aws-kafka_broker.dns_records
6+
value = module.my_instance.dns_records
7+
}
8+
9+
output "security_group" {
10+
value = aws_security_group.my_security_group
711
}

‎modules/kafka_broker/variables.tf

+33
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,37 @@ variable "dns_template" {
5555
variable "extra_template_vars" {
5656
type = map
5757
default = {}
58+
}
59+
60+
#SG Related Vars
61+
variable "vpc_id" {
62+
type = string
63+
}
64+
variable "enable_sg_creation" {
65+
type = boolean
66+
default = true
67+
}
68+
variable "sg_name" {
69+
type = string
70+
default = "CP_Kafka_Broker"
71+
}
72+
variable "kafka_connect_sg_ids" {
73+
type = list
74+
default = []
75+
}
76+
variable "ksql_sg_ids" {
77+
type = list
78+
default = []
79+
}
80+
variable "rest_proxy_sg_id" {
81+
type = string
82+
default = ""
83+
}
84+
variable "schema_registry_sg_id" {
85+
type = string
86+
default = ""
87+
}
88+
variable "control_center_sg_id" {
89+
type = string
90+
default = ""
5891
}

‎modules/kafka_connect/main.tf

+28-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,31 @@
1-
###########################
2-
# Connect Resources
3-
###########################
4-
module "cp-aws-kafka_connect" {
1+
resource "aws_security_group" "my_security_group" {
2+
count = var.enable_sg_creation ? 1 : 0
3+
name = var.sg_name
4+
description = "Confluent Platform - Kafka Connect"
5+
vpc_id = var.vpc_id
6+
7+
tags = var.tags
8+
9+
#Kafka Connect Related
10+
ingress {
11+
description = "Connect - REST Interface - Internal"
12+
from_port = 8083
13+
to_port = 8083
14+
protocol = "tcp"
15+
self = true
16+
}
17+
ingress {
18+
description = "Connect - REST Interface - External"
19+
from_port = 8083
20+
to_port = 8083
21+
protocol = "tcp"
22+
security_groups = [
23+
var.control_center_sg_id
24+
]
25+
}
26+
}
27+
28+
module "my_instance" {
529
source = "../base_node"
630

731
extra_template_vars = var.extra_template_vars

‎modules/kafka_connect/output.tf

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
output "instances" {
2-
value = module.cp-aws-kafka_connect.instances
2+
value = module.my_instance.instances
33
}
44

55
output "dns_records" {
6-
value = module.cp-aws-kafka_connect.dns_records
6+
value = module.my_instance.dns_records
7+
}
8+
9+
output "security_group" {
10+
value = aws_security_group.my_security_group
711
}

0 commit comments

Comments
 (0)