Implement per interface IMDS

debian/patches/update-networkd-priorities.patch

@@ -1,6 +1,6 @@
-From 2761694987b588be2f6cc63e704a421e8a088b81 Mon Sep 17 00:00:00 2001
+From 5e1071bec2f025a9a90be250239eca5ef1dedb66 Mon Sep 17 00:00:00 2001
 From: Joe Kurokawa <[email protected]>
-Date: Tue, 6 May 2025 21:31:36 +0000
+Date: Thu, 8 May 2025 00:00:55 +0000
 Subject: [PATCH] change the priority of the networkd configs
 
 ensure they're order before netplan
@@ -25,10 +25,10 @@ index a79fd09..9cb623b 100755
      ;;
  stop|cleanup)
 diff --git a/lib/lib.sh b/lib/lib.sh
-index 981f643..858dc86 100644
+index 90cad29..936e986 100644
 --- a/lib/lib.sh
 +++ b/lib/lib.sh
-@@ -149,7 +149,7 @@ create_ipv4_aliases() {
+@@ -206,7 +206,7 @@ create_ipv4_aliases() {
          info "No addresses found for ${iface}"
          return 0
      fi
@@ -37,7 +37,7 @@ index 981f643..858dc86 100644
      mkdir -p "$drop_in_dir"
      local file="$drop_in_dir/ec2net_alias.conf"
      local work="${file}.new"
-@@ -208,7 +208,7 @@ create_rules() {
+@@ -265,7 +265,7 @@ create_rules() {
      local family=$4
      local addrs prefixes
      local local_addr_key subnet_pd_key
@@ -46,7 +46,7 @@ index 981f643..858dc86 100644
      mkdir -p "$drop_in_dir"
 
      local -i ruleid=$((device_number+rule_base+100*network_card))
-@@ -376,7 +376,7 @@ create_interface_config() {
+@@ -433,7 +433,7 @@ create_interface_config() {
 
      local -i retval=0
 

lib/lib.sh

@@ -24,35 +24,84 @@ declare -r imds_token_path="api/token"
 declare -r syslog_facility="user"
 declare -r syslog_tag="ec2net"
 declare -i -r rule_base=10000
+declare -r primary_route="PRIMARY"
 
 # Systemd installs routes with a metric of 1024 by default.  We
 # override to a lower metric to ensure that our fully configured
 # interfaces are preferred over those in the process of being
 # configured.
 declare -i -r metric_base=512
-declare imds_endpoint imds_token
+declare imds_endpoint=""
+declare imds_token=""
+declare imds_interface=""
+declare self_iface_name=""
+
+make_token_request() {
+    local ep=${1:-""}
+    local interface=${2:-""}
+    local -a curl_opts=(--max-time 5 --connect-timeout 0.15 -s --fail -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60")
+    if [ -n "$interface" ]; then
+        curl_opts+=(--interface "$interface")
+    fi
+    curl "${curl_opts[@]}" "${ep}/${imds_token_path}"
+}
+
+get_lowest_secondary_interface() {
+    # This is the best effort guess at what the lowest secondary interface would be.
+    # It will return empty if there are no secondary interfaces.
+    local iface
+    local -a interfaces=()
+
+    for iface in /sys/class/net/*; do
+        iface=${iface##*/}
+        [[ $iface =~ ^(lo|docker|veth|dummy|vlan) ]] && continue
+        interfaces+=("$iface")
+    done
+    printf "%s\n" "${interfaces[@]}" | sort -V | sed -n '2p'
+}
 
 get_token() {
-    # try getting a token early, using each endpoint in
-    # turn. Whichever endpoint responds will be used for the rest of
-    # the IMDS API calls.  On initial interface setup, we'll retry
+    # Try getting a token early, using each endpoint in
+    # turn. Whichever interface and endpoint responds will be used for all the IMDS calls
+    # used to setup the interface. For IMDS interface we will
+    # try to call the IMDS from its self first, failing that the
+    # primary eni, and failing that the best guess at the 
+    # lowest secondary eni if available.
+    # On initial interface setup, we'll retry
     # this operation for up to 30 seconds, but on subsequent
     # invocations we avoid retrying
     local deadline
+    local intf=$self_iface_name
     deadline=$(date -d "now+30 seconds" +%s)
     local old_opts=$-
+
     while [ "$(date +%s)" -lt $deadline ]; do
         for ep in "${imds_endpoints[@]}"; do
             set +e
-            imds_token=$(curl --max-time 5 --connect-timeout 0.15 -s --fail \
-                              -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" ${ep}/${imds_token_path})
+            imds_token=$(make_token_request "$ep" "$intf")
+
+            if [ -z "$imds_token" ]; then
+                imds_token=$(make_token_request "$ep")
+                intf="$primary_route"
+            fi
+
+            if [ -z "$imds_token" ]; then
+                lowest_secondary=$(get_lowest_secondary_interface)
+                if [ -n "$lowest_secondary" ]; then
+                    imds_token=$(make_token_request "$ep" "$intf")
+                    intf="$lowest_secondary"
+                fi
+            fi
             [[ $old_opts = *e* ]] && set -e
+
             if [ -n "$imds_token" ]; then
-                debug "Got IMDSv2 token from ${ep}"
+                debug "Got IMDSv2 token from ${ep} via ${intf}"
                 imds_endpoint=$ep
+                imds_interface=$intf
                 return
             fi
         done
+
         if [ ! -v EC2_IF_INITIAL_SETUP ]; then
             break
         fi
@@ -85,11 +134,19 @@ get_meta() {
     debug "[get_meta] Querying IMDS for ${key}"
 
     get_token
-
+    if [[ -z $imds_endpoint || -z $imds_token || -z $imds_interface ]]; then
+        error "[get_meta] Unable to obtain IMDS token, endpoint, or interface"
+        return 1
+    fi
     local url="${imds_endpoint}/meta-data/${key}"
     local meta rc
+    local curl_opts=(-s --max-time 5 -H "X-aws-ec2-metadata-token:${imds_token}" -f)
+    if [[ "$imds_interface" != "$primary_route" ]]; then
+        curl_opts+=(--interface "$imds_interface")
+    fi
+
     while [ $attempts -lt $max_tries ]; do
-        meta=$(curl -s --max-time 5 -H "X-aws-ec2-metadata-token:${imds_token}" -f "$url")
+        meta=$(curl "${curl_opts[@]}" "$url")
         rc=$?
         if [ $rc -eq 0 ]; then
             echo "$meta"
@@ -472,6 +529,7 @@ setup_interface() {
     local -i device_number network_card rc
     iface=$1
     ether=$2
+    self_iface_name=$1
 
     network_card=$(_get_network_card "$iface" "$ether")
     device_number=$(_get_device_number "$iface" "$ether" "$network_card")