diff --git a/R/R/decorators-aws.R b/R/R/decorators-aws.R index 6c06bde39ee..24679b78cd3 100644 --- a/R/R/decorators-aws.R +++ b/R/R/decorators-aws.R @@ -14,7 +14,7 @@ #' #' @param cpu Integer number of CPUs required for this step. Defaults to `1`. #' @param gpu Integer number of GPUs required for this step. Defaults to `0`. -#' @param memory Integer memory size (in MB) required for this step. Defaults to +#' @param memory Integer memory size (in MiB) required for this step. Defaults to #' `4096`. #' @param image Character. Specifies the image to use when launching on AWS #' Batch. If not specified, an appropriate diff --git a/R/man/batch.Rd b/R/man/batch.Rd index 5286e63645f..7723b4d8a70 100644 --- a/R/man/batch.Rd +++ b/R/man/batch.Rd @@ -25,7 +25,7 @@ resources(cpu = 1L, gpu = 0L, memory = 4096L, shared_memory = NULL) \item{gpu}{Integer number of GPUs required for this step. Defaults to \code{0}.} -\item{memory}{Integer memory size (in MB) required for this step. Defaults to +\item{memory}{Integer memory size (in MiB) required for this step. Defaults to \code{4096}.} \item{image}{Character. Specifies the image to use when launching on AWS diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 5c61d3f8f03..68c65ef5e6e 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -40,7 +40,7 @@ class BatchDecorator(StepDecorator): Number of GPUs required for this step. If `@resources` is also present, the maximum value from all decorators is used. memory : int, default 4096 - Memory size (in MB) required for this step. If + Memory size (in MiB) required for this step. If `@resources` is also present, the maximum value from all decorators is used. image : str, optional, default None diff --git a/metaflow/plugins/kubernetes/kube_utils.py b/metaflow/plugins/kubernetes/kube_utils.py index d41f613be00..e9d56b59543 100644 --- a/metaflow/plugins/kubernetes/kube_utils.py +++ b/metaflow/plugins/kubernetes/kube_utils.py @@ -1,7 +1,8 @@ import re from typing import Dict, List, Optional + from metaflow.exception import CommandException, MetaflowException -from metaflow.util import get_username, get_latest_run_id +from metaflow.util import get_latest_run_id, get_username # avoid circular import by having the exception class contained here @@ -43,8 +44,8 @@ def qos_requests_and_limits(qos: str, cpu: int, memory: int, storage: int): # Guaranteed - has both cpu/memory limits. requests not required, as these will be inferred. qos_limits = { "cpu": str(cpu), - "memory": "%sM" % str(memory), - "ephemeral-storage": "%sM" % str(storage), + "memory": "%sMi" % str(memory), + "ephemeral-storage": "%sMi" % str(storage), } # NOTE: Even though Kubernetes will produce matching requests for the specified limits, this happens late in the lifecycle. # We specify them explicitly here to make some K8S tooling happy, in case they rely on .resources.requests being present at time of submitting the job. @@ -53,8 +54,8 @@ def qos_requests_and_limits(qos: str, cpu: int, memory: int, storage: int): # Burstable - not Guaranteed, and has a memory/cpu limit or request qos_requests = { "cpu": str(cpu), - "memory": "%sM" % str(memory), - "ephemeral-storage": "%sM" % str(storage), + "memory": "%sMi" % str(memory), + "ephemeral-storage": "%sMi" % str(storage), } # TODO: Add support for BestEffort once there is a use case for it. # BestEffort - no limit or requests for cpu/memory diff --git a/metaflow/plugins/kubernetes/kubernetes_decorator.py b/metaflow/plugins/kubernetes/kubernetes_decorator.py index 697d5c055b1..f53bfc1d46d 100644 --- a/metaflow/plugins/kubernetes/kubernetes_decorator.py +++ b/metaflow/plugins/kubernetes/kubernetes_decorator.py @@ -11,6 +11,8 @@ from metaflow.metadata_provider.util import sync_local_metadata_to_datastore from metaflow.metaflow_config import ( DATASTORE_LOCAL_DIR, + KUBERNETES_ANNOTATIONS, + KUBERNETES_CONDA_ARCH, KUBERNETES_CONTAINER_IMAGE, KUBERNETES_CONTAINER_REGISTRY, KUBERNETES_CPU, @@ -18,18 +20,16 @@ KUBERNETES_FETCH_EC2_METADATA, KUBERNETES_GPU_VENDOR, KUBERNETES_IMAGE_PULL_POLICY, - KUBERNETES_MEMORY, KUBERNETES_LABELS, - KUBERNETES_ANNOTATIONS, + KUBERNETES_MEMORY, KUBERNETES_NAMESPACE, KUBERNETES_NODE_SELECTOR, KUBERNETES_PERSISTENT_VOLUME_CLAIMS, KUBERNETES_PORT, + KUBERNETES_QOS, KUBERNETES_SERVICE_ACCOUNT, KUBERNETES_SHARED_MEMORY, KUBERNETES_TOLERATIONS, - KUBERNETES_QOS, - KUBERNETES_CONDA_ARCH, ) from metaflow.plugins.resources_decorator import ResourcesDecorator from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task @@ -37,8 +37,8 @@ from metaflow.unbounded_foreach import UBF_CONTROL from ..aws.aws_utils import get_docker_registry, get_ec2_instance_metadata +from .kube_utils import parse_kube_keyvalue_list, validate_kube_labels from .kubernetes import KubernetesException -from .kube_utils import validate_kube_labels, parse_kube_keyvalue_list try: unicode @@ -59,11 +59,11 @@ class KubernetesDecorator(StepDecorator): Number of CPUs required for this step. If `@resources` is also present, the maximum value from all decorators is used. memory : int, default 4096 - Memory size (in MB) required for this step. If + Memory size (in MiB) required for this step. If `@resources` is also present, the maximum value from all decorators is used. disk : int, default 10240 - Disk size (in MB) required for this step. If + Disk size (in MiB) required for this step. If `@resources` is also present, the maximum value from all decorators is used. image : str, optional, default None diff --git a/metaflow/plugins/resources_decorator.py b/metaflow/plugins/resources_decorator.py index d1e04a4f89d..dcf17073ecc 100644 --- a/metaflow/plugins/resources_decorator.py +++ b/metaflow/plugins/resources_decorator.py @@ -26,9 +26,9 @@ class ResourcesDecorator(StepDecorator): gpu : int, optional, default None Number of GPUs required for this step. disk : int, optional, default None - Disk size (in MB) required for this step. Only applies on Kubernetes. + Disk size (in MiB) required for this step. Only applies on Kubernetes. memory : int, default 4096 - Memory size (in MB) required for this step. + Memory size (in MiB) required for this step. shared_memory : int, optional, default None The value for the size (in MiB) of the /dev/shm volume for this step. This parameter maps to the `--shm-size` option in Docker.