From 5f6b4edda49ff2c537e6345241696527fc28e620 Mon Sep 17 00:00:00 2001
From: Justin Bradfield <justin@materialize.com>
Date: Mon, 14 Apr 2025 22:14:22 -0500
Subject: [PATCH] add karpenter for auto-scaling

---
 examples/simple/main.tf  |  76 ++++++++++++++++-
 main.tf                  |   4 +
 modules/eks/karpenter.tf | 109 ++++++++++++++++++++++++
 modules/eks/main.tf      | 177 +++++++++++++++++++++++++++++++++++++++
 modules/eks/variables.tf |  60 ++++++++++++-
 modules/eks/versions.tf  |   4 +
 variables.tf             |  17 ++++
 versions.tf              |   4 +
 8 files changed, 447 insertions(+), 4 deletions(-)
 create mode 100644 modules/eks/karpenter.tf

diff --git a/examples/simple/main.tf b/examples/simple/main.tf
index b4318c5..e55430f 100644
--- a/examples/simple/main.tf
+++ b/examples/simple/main.tf
@@ -26,6 +26,51 @@ provider "helm" {
   }
 }
 
+provider "kubectl" {
+  host                   = module.materialize_infrastructure.eks_cluster_endpoint
+  cluster_ca_certificate = base64decode(module.materialize_infrastructure.cluster_certificate_authority_data)
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    args        = ["eks", "get-token", "--cluster-name", module.materialize_infrastructure.eks_cluster_name]
+  }
+  load_config_file = false
+}
+
+
+locals {
+  with_karpenter_installed = {
+    "clusterd" = {
+      nodeSelector = {
+        workload = "materialize-instance-karpenter"
+      }
+    }
+    "operator" = {
+      nodeSelector = {
+        workload = "materialize-instance"
+      }
+    }
+    "environmentd" = {
+      nodeSelector = {
+        workload = "materialize-instance-karpenter"
+      }
+    }
+    "console" = {
+      nodeSelector = {
+        workload = "materialize-instance"
+        # "karpenter.sh/registered" = "m7g.medium"
+      }
+    }
+    "balancerd" = {
+      nodeSelector = {
+        workload = "materialize-instance"
+        # "karpenter.sh/registered" = "m7g.medium"
+      }
+    }
+  }
+  node_selectors = var.install_karpenter ? local.with_karpenter_installed : {}
+}
+
 module "materialize_infrastructure" {
   # To pull this from GitHub, use the following:
   # source = "git::https://github.com/MaterializeInc/terraform-aws-materialize.git"
@@ -52,11 +97,15 @@ module "materialize_infrastructure" {
   # EKS Configuration
   cluster_version                          = "1.32"
   node_group_instance_types                = ["r7gd.2xlarge"]
-  node_group_desired_size                  = 1
-  node_group_min_size                      = 1
+  node_group_desired_size                  = 2
+  node_group_min_size                      = 2
   node_group_max_size                      = 2
   node_group_capacity_type                 = "ON_DEMAND"
   enable_cluster_creator_admin_permissions = true
+  install_karpenter                        = var.install_karpenter
+  karpenter_instance_sizes                 = var.karpenter_instance_sizes
+
+  enable_disk_support = true
 
   # Storage Configuration
   bucket_force_destroy = true
@@ -87,7 +136,7 @@ module "materialize_infrastructure" {
   install_materialize_operator = true
   operator_version             = var.operator_version
   orchestratord_version        = var.orchestratord_version
-  helm_values                  = var.helm_values
+  helm_values                  = merge(local.node_selectors, var.helm_values)
 
   # Once the operator is installed, you can define your Materialize instances here.
   materialize_instances = var.materialize_instances
@@ -172,6 +221,27 @@ variable "use_self_signed_cluster_issuer" {
   default     = true
 }
 
+variable "install_karpenter" {
+  description = "Whether to install karpenter: https://karpenter.sh"
+  type        = bool
+  default     = false
+}
+
+variable "karpenter_instance_sizes" {
+  description = "Additional settings for Karpenter Helm chart"
+  type        = list(string)
+  default = [
+    # Optionally,  console and balancer don't
+    # need disk we should be able to throw them on their own nodes
+    # "m7g.medium",
+    # Recommended clusters sizes
+    "r7gd.xlarge",
+    "r7gd.2xlarge",
+    "r7gd.4xlarge",
+    "r7gd.8xlarge",
+    "r7gd.16xlarge",
+  ]
+}
 # Outputs
 output "vpc_id" {
   description = "VPC ID"
diff --git a/main.tf b/main.tf
index 9d95d41..1f13e2f 100644
--- a/main.tf
+++ b/main.tf
@@ -22,9 +22,11 @@ module "eks" {
   # e.g. ${namespace}-${environment}-eks
   namespace   = var.namespace
   environment = var.environment
+  region      = data.aws_region.current.name
 
   cluster_version                          = var.cluster_version
   vpc_id                                   = local.network_id
+  vpc_cidr                                 = var.vpc_cidr
   private_subnet_ids                       = local.network_private_subnet_ids
   node_group_desired_size                  = var.node_group_desired_size
   node_group_min_size                      = var.node_group_min_size
@@ -35,6 +37,8 @@ module "eks" {
   node_group_capacity_type                 = var.node_group_capacity_type
   enable_cluster_creator_admin_permissions = var.enable_cluster_creator_admin_permissions
 
+  install_karpenter = var.install_karpenter
+
   install_openebs   = local.disk_config.install_openebs
   enable_disk_setup = local.disk_config.run_disk_setup_script
   openebs_namespace = local.disk_config.openebs_namespace
diff --git a/modules/eks/karpenter.tf b/modules/eks/karpenter.tf
new file mode 100644
index 0000000..9ec6f95
--- /dev/null
+++ b/modules/eks/karpenter.tf
@@ -0,0 +1,109 @@
+resource "kubectl_manifest" "karpenter_node_class" {
+  count           = var.install_karpenter ? 1 : 0
+  force_conflicts = true
+
+
+  yaml_body = yamlencode({
+    apiVersion = "karpenter.k8s.aws/v1"
+    kind       = "EC2NodeClass"
+    metadata = {
+      name = "${local.name_prefix}-node-class2"
+    }
+    spec = {
+      amiSelectorTerms = [{
+        alias = "al2023@latest"
+      }]
+      role = module.eks.eks_managed_node_groups["${local.name_prefix}-mz"].iam_role_name
+      subnetSelectorTerms = [
+        for sn_id in var.private_subnet_ids :
+        {
+          id = sn_id
+        }
+      ]
+      securityGroupSelectorTerms = [
+        {
+          id = module.eks.node_security_group_id
+        }
+      ]
+      kubelet = {
+        clusterDNS = [cidrhost(module.eks.cluster_service_cidr, 10)]
+      }
+      blockDeviceMappings = [
+        {
+          deviceName = "/dev/xvda"
+          ebs = {
+            deleteOnTermination = true
+            volumeSize          = "20Gi"
+            volumeType          = "gp3"
+            encrypted           = true
+          }
+        }
+      ]
+
+      userData = var.enable_disk_setup ? local.disk_setup_script : ""
+      tags = merge(var.tags, {
+        Name = "${local.name_prefix}-karpenter"
+      })
+      metadataOptions = {
+        httpEndpoint            = "enabled"
+        httpProtocolIPv6        = "enabled"
+        httpPutResponseHopLimit = 3
+        httpTokens              = "required"
+      }
+    }
+  })
+
+  depends_on = [helm_release.karpenter]
+}
+
+resource "kubectl_manifest" "karpenter_node_pool" {
+  count           = var.install_karpenter ? 1 : 0
+  force_conflicts = true
+
+  yaml_body = yamlencode({
+    apiVersion = "karpenter.sh/v1"
+    kind       = "NodePool"
+    metadata = {
+      name = "${local.name_prefix}-node-pool2"
+    }
+    spec = {
+      template = {
+        metadata = {
+          labels = {
+            "Environment"            = var.environment
+            "Name"                   = "${local.name_prefix}-karpenter_node_pool"
+            "materialize.cloud/disk" = var.enable_disk_setup ? "true" : "false"
+            "workload"               = "materialize-instance-karpenter"
+          }
+        }
+        spec = {
+          expireAfter = "Never"
+          nodeClassRef = {
+            group = "karpenter.k8s.aws"
+            kind  = "EC2NodeClass"
+            name  = "${local.name_prefix}-node-class2"
+          }
+          requirements = [
+            {
+              key      = "node.kubernetes.io/instance-type"
+              operator = "In"
+              values   = var.karpenter_instance_sizes
+            },
+            {
+              key      = "karpenter.sh/capacity-type"
+              operator = "In"
+              values   = ["on-demand", "reserved"]
+            }
+          ]
+        }
+      }
+      disruption = {
+        consolidationPolicy = "WhenEmpty"
+        consolidateAfter    = "15s"
+
+      }
+    }
+  })
+
+  depends_on = [kubectl_manifest.karpenter_node_class]
+}
diff --git a/modules/eks/main.tf b/modules/eks/main.tf
index 2020095..aff813b 100644
--- a/modules/eks/main.tf
+++ b/modules/eks/main.tf
@@ -21,6 +21,7 @@ module "eks" {
 
   eks_managed_node_groups = {
     "${local.name_prefix}-mz" = {
+      # Desired node count is is ignored after first run
       desired_size = var.node_group_desired_size
       min_size     = var.node_group_min_size
       max_size     = var.node_group_max_size
@@ -37,6 +38,9 @@ module "eks" {
         "materialize.cloud/disk" = var.enable_disk_setup ? "true" : "false"
         "workload"               = "materialize-instance"
       }
+      iam_role_additional_policies = {
+        AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+      }
 
       cloudinit_pre_nodeadm = var.enable_disk_setup ? [
         {
@@ -110,3 +114,176 @@ resource "helm_release" "openebs" {
 
   depends_on = [kubernetes_namespace.openebs]
 }
+
+# Karpenter Namespace
+resource "aws_security_group" "karpenter" {
+  count       = var.install_karpenter ? 1 : 0
+  name_prefix = "${local.name_prefix}-sg-"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    security_groups = [module.eks.node_security_group_id, module.eks.cluster_security_group_id]
+    description     = "Allow access from the eks security group"
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = merge(var.tags, {
+    Name = "${local.name_prefix}-sg"
+  })
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "kubernetes_namespace" "karpenter" {
+  count = var.install_karpenter ? 1 : 0
+
+  metadata {
+    name = var.karpenter_namespace
+  }
+}
+
+# Karpenter IAM Role
+resource "aws_iam_role" "karpenter" {
+  count = var.install_karpenter ? 1 : 0
+
+  name = "${local.name_prefix}-karpenter"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRoleWithWebIdentity"
+        Effect = "Allow"
+        Principal = {
+          Federated = module.eks.oidc_provider_arn
+        }
+        Condition = {
+          StringEquals = {
+            "${module.eks.oidc_provider}:sub" = "system:serviceaccount:${var.karpenter_namespace}:${var.karpenter_service_account}"
+          }
+        }
+      }
+    ]
+  })
+
+  tags = var.tags
+}
+
+# Karpenter IAM Policy
+resource "aws_iam_role_policy" "karpenter" {
+  count = var.install_karpenter ? 1 : 0
+
+  name = "${local.name_prefix}-karpenter"
+  role = aws_iam_role.karpenter[0].id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "ec2:CreateLaunchTemplate",
+          "ec2:CreateFleet",
+          "ec2:RunInstances",
+          "ec2:CreateTags",
+          "iam:PassRole",
+          "ec2:TerminateInstances",
+          "ec2:DeleteLaunchTemplate",
+          "ec2:DescribeLaunchTemplates",
+          "ec2:DescribeInstances",
+          "ec2:DescribeSecurityGroups",
+          "ec2:DescribeSubnets",
+          "ec2:DescribeInstanceTypes",
+          "ec2:DescribeInstanceTypeOfferings",
+          "ec2:DescribeAvailabilityZones",
+          "ec2:DescribeImages",
+          "ec2:DescribeImages",
+          "ec2:DescribeSpotPriceHistory",
+          "ssm:GetParameter",
+          "iam:GetInstanceProfile",
+          "iam:CreateInstanceProfile",
+          "iam:TagInstanceProfile",
+          "iam:AddRoleToInstanceProfile",
+          "eks:DescribeCluster",
+          "pricing:GetProducts"
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+# Karpenter Service Account
+resource "kubernetes_service_account" "karpenter" {
+  count = var.install_karpenter ? 1 : 0
+
+  metadata {
+    name      = var.karpenter_service_account
+    namespace = var.karpenter_namespace
+    annotations = {
+      "eks.amazonaws.com/role-arn" = aws_iam_role.karpenter[0].arn
+    }
+  }
+
+  depends_on = [kubernetes_namespace.karpenter]
+}
+
+# Karpenter Helm Chart
+resource "helm_release" "karpenter" {
+  count = var.install_karpenter ? 1 : 0
+
+  name       = "karpenter"
+  namespace  = var.karpenter_namespace
+  repository = "oci://public.ecr.aws/karpenter"
+  chart      = "karpenter"
+  version    = var.karpenter_version
+
+  set {
+    name  = "serviceAccount.name"
+    value = var.karpenter_service_account
+  }
+
+  set {
+    name  = "serviceAccount.create"
+    value = false
+  }
+
+  set {
+    name  = "settings.clusterName"
+    value = module.eks.cluster_name
+  }
+
+  set {
+    name  = "settings.clusterEndpoint"
+    value = module.eks.cluster_endpoint
+  }
+
+  set {
+    name  = "settings.defaultInstanceProfile"
+    value = module.eks.eks_managed_node_groups["${local.name_prefix}-mz"].iam_role_name
+  }
+
+  dynamic "set" {
+    for_each = var.karpenter_settings
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+
+  depends_on = [
+    kubernetes_service_account.karpenter,
+    aws_iam_role_policy.karpenter
+  ]
+}
diff --git a/modules/eks/variables.tf b/modules/eks/variables.tf
index fd468b1..d2b4882 100644
--- a/modules/eks/variables.tf
+++ b/modules/eks/variables.tf
@@ -46,7 +46,7 @@ variable "node_group_instance_types" {
 variable "node_group_ami_type" {
   description = "AMI type for the node group"
   type        = string
-  default     = "AL2023_x86_64_STANDARD"
+  default     = "AL2023_ARM_64_STANDARD"
 }
 
 variable "cluster_enabled_log_types" {
@@ -97,3 +97,61 @@ variable "enable_disk_setup" {
   type        = bool
   default     = true
 }
+
+variable "vpc_cidr" {
+  description = "CIDR of eks vpc"
+  type        = string
+}
+
+variable "cluster_service_ipv4_cidr" {
+  description = "CIDR block to assign Kubernetes service IP addresses from"
+  type        = string
+  default     = "10.100.0.0/16"
+}
+
+# Karpenter configuration
+variable "install_karpenter" {
+  description = "Whether to install Karpenter"
+  type        = bool
+  default     = false
+}
+
+variable "karpenter_version" {
+  description = "Version of the Karpenter Helm chart to install"
+  type        = string
+  default     = "1.3.3"
+}
+
+variable "karpenter_namespace" {
+  description = "Namespace for Karpenter"
+  type        = string
+  default     = "karpenter"
+}
+
+variable "karpenter_service_account" {
+  description = "Name of the Karpenter service account"
+  type        = string
+  default     = "karpenter"
+}
+
+variable "karpenter_settings" {
+  description = "Additional settings for Karpenter Helm chart"
+  type        = map(string)
+  default     = {}
+}
+
+variable "karpenter_instance_sizes" {
+  description = "Additional settings for Karpenter Helm chart"
+  type        = list(string)
+  default = [
+    "r7gd.xlarge",
+    "r7gd.2xlarge",
+    "r7gd.4xlarge",
+    "r7gd.8xlarge",
+  ]
+}
+
+variable "region" {
+  description = "AWS region"
+  type        = string
+}
diff --git a/modules/eks/versions.tf b/modules/eks/versions.tf
index 6f04d7c..d486cb4 100644
--- a/modules/eks/versions.tf
+++ b/modules/eks/versions.tf
@@ -14,5 +14,9 @@ terraform {
       source  = "hashicorp/helm"
       version = "~> 2.0"
     }
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = ">= 1.18.0"
+    }
   }
 }
diff --git a/variables.tf b/variables.tf
index aea75bd..7aa9d73 100644
--- a/variables.tf
+++ b/variables.tf
@@ -279,6 +279,23 @@ variable "install_cert_manager" {
   default     = true
 }
 
+variable "install_karpenter" {
+  description = "Whether to instal-karpenter."
+  type        = bool
+  default     = false
+}
+
+variable "karpenter_instance_sizes" {
+  description = "Additional settings for Karpenter Helm chart"
+  type        = list(string)
+  default = [
+    "r7gd.xlarge",
+    "r7gd.2xlarge",
+    "r7gd.4xlarge",
+    "r7gd.8xlarge",
+  ]
+}
+
 variable "use_self_signed_cluster_issuer" {
   description = "Whether to install and use a self-signed ClusterIssuer for TLS. To work around limitations in Terraform, this will be treated as `false` if no materialize instances are defined."
   type        = bool
diff --git a/versions.tf b/versions.tf
index 54bd60e..b6253d6 100644
--- a/versions.tf
+++ b/versions.tf
@@ -18,5 +18,9 @@ terraform {
       source  = "hashicorp/random"
       version = "~> 3.0"
     }
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = ">= 1.18.0"
+    }
   }
 }