Netflix · nschmeller · May 8, 2025
diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py
@@ -370,6 +370,8 @@
 )
 # Image pull policy for container images
 KUBERNETES_IMAGE_PULL_POLICY = from_conf("KUBERNETES_IMAGE_PULL_POLICY", None)
+# Image pull secrets for container images
+KUBERNETES_IMAGE_PULL_SECRETS = from_conf("KUBERNETES_IMAGE_PULL_SECRETS", "")
 # Default container registry for K8S
 KUBERNETES_CONTAINER_REGISTRY = from_conf(
     "KUBERNETES_CONTAINER_REGISTRY", DEFAULT_CONTAINER_REGISTRY

diff --git a/metaflow/plugins/argo/argo_workflows.py b/metaflow/plugins/argo/argo_workflows.py
@@ -2007,6 +2007,7 @@ def _container_templates(self):
                     namespace=resources["namespace"],
                     image=resources["image"],
                     image_pull_policy=resources["image_pull_policy"],
+                    image_pull_secrets=resources["image_pull_secrets"],
                     service_account=resources["service_account"],
                     secrets=(
                         [
@@ -2193,6 +2194,8 @@ def _container_templates(self):
                     .node_selectors(resources.get("node_selector"))
                     # Set tolerations
                     .tolerations(resources.get("tolerations"))
+                    # Set image pull secrets
+                    .image_pull_secrets(resources.get("image_pull_secrets"))
                     # Set container
                     .container(
                         # TODO: Unify the logic with kubernetes.py
@@ -3744,6 +3747,10 @@ def tolerations(self, tolerations):
         self.payload["tolerations"] = tolerations
         return self
 
+    def image_pull_secrets(self, image_pull_secrets):
+        self.payload["image_pull_secrets"] = image_pull_secrets
+        return self
+
     def to_json(self):
         return self.payload
 

diff --git a/metaflow/plugins/kubernetes/kubernetes.py b/metaflow/plugins/kubernetes/kubernetes.py
@@ -170,6 +170,7 @@ def create_jobset(
         code_package_ds,
         docker_image,
         docker_image_pull_policy,
+        image_pull_secrets=None,
         step_cli=None,
         service_account=None,
         secrets=None,
@@ -206,6 +207,7 @@ def create_jobset(
                 node_selector=node_selector,
                 image=docker_image,
                 image_pull_policy=docker_image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 cpu=cpu,
                 memory=memory,
                 disk=disk,
@@ -467,6 +469,7 @@ def create_job_object(
         step_cli,
         docker_image,
         docker_image_pull_policy,
+        image_pull_secrets=None,
         service_account=None,
         secrets=None,
         node_selector=None,
@@ -513,6 +516,7 @@ def create_job_object(
                 ),
                 image=docker_image,
                 image_pull_policy=docker_image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 cpu=cpu,
                 memory=memory,
                 disk=disk,

diff --git a/metaflow/plugins/kubernetes/kubernetes_cli.py b/metaflow/plugins/kubernetes/kubernetes_cli.py
@@ -53,6 +53,12 @@ def kubernetes():
     default=None,
     help="Optional Docker Image Pull Policy for Kubernetes pod.",
 )
+@click.option(
+    "--image-pull-secrets",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
 @click.option(
     "--service-account",
     help="IRSA requirement for Kubernetes pod.",
@@ -160,6 +166,7 @@ def step(
     executable=None,
     image=None,
     image_pull_policy=None,
+    image_pull_secrets=None,
     service_account=None,
     secrets=None,
     node_selector=None,
@@ -303,6 +310,7 @@ def _sync_metadata():
                 step_cli=step_cli,
                 docker_image=image,
                 docker_image_pull_policy=image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 service_account=service_account,
                 secrets=secrets,
                 node_selector=node_selector,

diff --git a/metaflow/plugins/kubernetes/kubernetes_decorator.py b/metaflow/plugins/kubernetes/kubernetes_decorator.py
@@ -18,6 +18,7 @@
     KUBERNETES_FETCH_EC2_METADATA,
     KUBERNETES_GPU_VENDOR,
     KUBERNETES_IMAGE_PULL_POLICY,
+    KUBERNETES_IMAGE_PULL_SECRETS,
     KUBERNETES_MEMORY,
     KUBERNETES_LABELS,
     KUBERNETES_ANNOTATIONS,
@@ -72,6 +73,10 @@ class KubernetesDecorator(StepDecorator):
         not, a default Docker image mapping to the current version of Python is used.
     image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
         If given, the imagePullPolicy to be applied to the Docker image of the step.
+    image_pull_secrets: List[str], default []
+        The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
+        Kubernetes image pull secrets to use when pulling container images
+        in Kubernetes.
     service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
         Kubernetes service account to use when launching pod in Kubernetes.
     secrets : List[str], optional, default None
@@ -139,6 +144,7 @@ class KubernetesDecorator(StepDecorator):
         "disk": "10240",
         "image": None,
         "image_pull_policy": None,
+        "image_pull_secrets": None,  # e.g., ["regcred"]
         "service_account": None,
         "secrets": None,  # e.g., mysecret
         "node_selector": None,  # e.g., kubernetes.io/os=linux
@@ -192,6 +198,10 @@ def init(self):
             )
         if not self.attributes["image_pull_policy"] and KUBERNETES_IMAGE_PULL_POLICY:
             self.attributes["image_pull_policy"] = KUBERNETES_IMAGE_PULL_POLICY
+        if not self.attributes["image_pull_secrets"] and KUBERNETES_IMAGE_PULL_SECRETS:
+            self.attributes["image_pull_secrets"] = json.loads(
+                KUBERNETES_IMAGE_PULL_SECRETS
+            )
 
         if isinstance(self.attributes["node_selector"], str):
             self.attributes["node_selector"] = parse_kube_keyvalue_list(
@@ -479,6 +489,7 @@ def runtime_step_cli(
                         for key, val in v.items()
                     ]
                 elif k in [
+                    "image_pull_secrets",
                     "tolerations",
                     "persistent_volume_claims",
                     "labels",

diff --git a/metaflow/plugins/kubernetes/kubernetes_job.py b/metaflow/plugins/kubernetes/kubernetes_job.py
@@ -214,8 +214,10 @@ def create_job_spec(self):
                         )
                     ],
                     node_selector=self._kwargs.get("node_selector"),
-                    # TODO (savin): Support image_pull_secrets
-                    # image_pull_secrets=?,
+                    image_pull_secrets=[
+                        client.V1LocalObjectReference(secret)
+                        for secret in self._kwargs.get("image_pull_secrets") or []
+                    ],
                     # TODO (savin): Support preemption policies
                     # preemption_policy=?,
                     #

diff --git a/metaflow/plugins/kubernetes/kubernetes_jobsets.py b/metaflow/plugins/kubernetes/kubernetes_jobsets.py
@@ -718,8 +718,11 @@ def dump(self):
                                     )
                                 ],
                                 node_selector=self._kwargs.get("node_selector"),
-                                # TODO (savin): Support image_pull_secrets
-                                # image_pull_secrets=?,
+                                image_pull_secrets=[
+                                    client.V1LocalObjectReference(secret)
+                                    for secret in self._kwargs.get("image_pull_secrets")
+                                    or []
+                                ],
                                 # TODO (savin): Support preemption policies
                                 # preemption_policy=?,
                                 #