From 0fe707007bb1d8f0e896dad883534902aaa844c5 Mon Sep 17 00:00:00 2001 From: Joe Kurokawa Date: Sat, 3 May 2025 18:47:04 +0000 Subject: [PATCH] Implement per interface IMDS For requesting IMDS, ec2-net-utils only makes calls through the primray ENI which can lead to throttling by IMDS. We split these calls over the secondary ENIs in order to avoid making too many requests throught the primary interface. If setup_interface cannot use its own interface to call IMDS, use the current primary route, failing to do that use the lowest available secondary. --- .../patches/update-networkd-priorities.patch | 12 +-- lib/lib.sh | 76 ++++++++++++++++--- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/debian/patches/update-networkd-priorities.patch b/debian/patches/update-networkd-priorities.patch index a0e27be..d8951fa 100644 --- a/debian/patches/update-networkd-priorities.patch +++ b/debian/patches/update-networkd-priorities.patch @@ -1,6 +1,6 @@ -From 2761694987b588be2f6cc63e704a421e8a088b81 Mon Sep 17 00:00:00 2001 +From 5e1071bec2f025a9a90be250239eca5ef1dedb66 Mon Sep 17 00:00:00 2001 From: Joe Kurokawa -Date: Tue, 6 May 2025 21:31:36 +0000 +Date: Thu, 8 May 2025 00:00:55 +0000 Subject: [PATCH] change the priority of the networkd configs ensure they're order before netplan @@ -25,10 +25,10 @@ index a79fd09..9cb623b 100755 ;; stop|cleanup) diff --git a/lib/lib.sh b/lib/lib.sh -index 981f643..858dc86 100644 +index 90cad29..936e986 100644 --- a/lib/lib.sh +++ b/lib/lib.sh -@@ -149,7 +149,7 @@ create_ipv4_aliases() { +@@ -206,7 +206,7 @@ create_ipv4_aliases() { info "No addresses found for ${iface}" return 0 fi @@ -37,7 +37,7 @@ index 981f643..858dc86 100644 mkdir -p "$drop_in_dir" local file="$drop_in_dir/ec2net_alias.conf" local work="${file}.new" -@@ -208,7 +208,7 @@ create_rules() { +@@ -265,7 +265,7 @@ create_rules() { local family=$4 local addrs prefixes local local_addr_key subnet_pd_key @@ -46,7 +46,7 @@ index 981f643..858dc86 100644 mkdir -p "$drop_in_dir" local -i ruleid=$((device_number+rule_base+100*network_card)) -@@ -376,7 +376,7 @@ create_interface_config() { +@@ -433,7 +433,7 @@ create_interface_config() { local -i retval=0 diff --git a/lib/lib.sh b/lib/lib.sh index 981f643..e9ccd16 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -24,35 +24,84 @@ declare -r imds_token_path="api/token" declare -r syslog_facility="user" declare -r syslog_tag="ec2net" declare -i -r rule_base=10000 +declare -r primary_route="PRIMARY" # Systemd installs routes with a metric of 1024 by default. We # override to a lower metric to ensure that our fully configured # interfaces are preferred over those in the process of being # configured. declare -i -r metric_base=512 -declare imds_endpoint imds_token +declare imds_endpoint="" +declare imds_token="" +declare imds_interface="" +declare self_iface_name="" + +make_token_request() { + local ep=${1:-""} + local interface=${2:-""} + local -a curl_opts=(--max-time 5 --connect-timeout 0.15 -s --fail -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60") + if [ -n "$interface" ]; then + curl_opts+=(--interface "$interface") + fi + curl "${curl_opts[@]}" "${ep}/${imds_token_path}" +} + +get_lowest_secondary_interface() { + # This is the best effort guess at what the lowest secondary interface would be. + # It will return empty if there are no secondary interfaces. + local iface + local -a interfaces=() + + for iface in /sys/class/net/*; do + iface=${iface##*/} + [[ $iface =~ ^(lo|docker|veth|dummy|vlan) ]] && continue + interfaces+=("$iface") + done + printf "%s\n" "${interfaces[@]}" | sort -V | sed -n '2p' +} get_token() { - # try getting a token early, using each endpoint in - # turn. Whichever endpoint responds will be used for the rest of - # the IMDS API calls. On initial interface setup, we'll retry + # Try getting a token early, using each endpoint in + # turn. Whichever interface and endpoint responds will be used for all the IMDS calls + # used to setup the interface. For IMDS interface we will + # try to call the IMDS from its self first, failing that the + # primary eni, and failing that the best guess at the + # lowest secondary eni if available. + # On initial interface setup, we'll retry # this operation for up to 30 seconds, but on subsequent # invocations we avoid retrying local deadline + local intf=$self_iface_name deadline=$(date -d "now+30 seconds" +%s) local old_opts=$- + while [ "$(date +%s)" -lt $deadline ]; do for ep in "${imds_endpoints[@]}"; do set +e - imds_token=$(curl --max-time 5 --connect-timeout 0.15 -s --fail \ - -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" ${ep}/${imds_token_path}) + imds_token=$(make_token_request "$ep" "$intf") + + if [ -z "$imds_token" ]; then + imds_token=$(make_token_request "$ep") + intf="$primary_route" + fi + + if [ -z "$imds_token" ]; then + lowest_secondary=$(get_lowest_secondary_interface) + if [ -n "$lowest_secondary" ]; then + imds_token=$(make_token_request "$ep" "$intf") + intf="$lowest_secondary" + fi + fi [[ $old_opts = *e* ]] && set -e + if [ -n "$imds_token" ]; then - debug "Got IMDSv2 token from ${ep}" + debug "Got IMDSv2 token from ${ep} via ${intf}" imds_endpoint=$ep + imds_interface=$intf return fi done + if [ ! -v EC2_IF_INITIAL_SETUP ]; then break fi @@ -85,11 +134,19 @@ get_meta() { debug "[get_meta] Querying IMDS for ${key}" get_token - + if [[ -z $imds_endpoint || -z $imds_token || -z $imds_interface ]]; then + error "[get_meta] Unable to obtain IMDS token, endpoint, or interface" + return 1 + fi local url="${imds_endpoint}/meta-data/${key}" local meta rc + local curl_opts=(-s --max-time 5 -H "X-aws-ec2-metadata-token:${imds_token}" -f) + if [[ "$imds_interface" != "$primary_route" ]]; then + curl_opts+=(--interface "$imds_interface") + fi + while [ $attempts -lt $max_tries ]; do - meta=$(curl -s --max-time 5 -H "X-aws-ec2-metadata-token:${imds_token}" -f "$url") + meta=$(curl "${curl_opts[@]}" "$url") rc=$? if [ $rc -eq 0 ]; then echo "$meta" @@ -472,6 +529,7 @@ setup_interface() { local -i device_number network_card rc iface=$1 ether=$2 + self_iface_name=$1 network_card=$(_get_network_card "$iface" "$ether") device_number=$(_get_device_number "$iface" "$ether" "$network_card")