RPW for OPTE v2p Mappings (#5568)

TODO
---
- [x] Extend db view to include probe v2p mappings
- [x] Update sagas to trigger rpw activation instead of directly
configuring v2p mappings
- [x] Test that the `delete` functionality cleans up v2p mappings

Related
---
Resolves #5214 
Resolves #4259 
Resolves #3107

- [x] Depends on oxidecomputer/opte#494
- [x] Depends on oxidecomputer/meta#409
- [x] Depends on oxidecomputer/maghemite#244

---------

Co-authored-by: Levon Tarver <[email protected]>

Author: internet-diglett

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.28"
+#: target = "lab-2.0-opte-0.29"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",

Cargo.lock

@@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "4cc823b50d3e4a629cdfaab2b3d3382514174ba9", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "4cc823b50d3e4a629cdfaab2b3d3382514174ba9" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }

Cargo.toml

@@ -35,7 +35,8 @@ progenitor::generate_api!(
         PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
-        OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] }
+        VirtualNetworkInterfaceHost = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
+        OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] },
     },
     //TODO trade the manual transformations later in this file for the
     //     replace directives below?

clients/sled-agent-client/src/lib.rs

@@ -114,6 +114,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -225,6 +229,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -323,6 +331,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/env.out

@@ -291,6 +291,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -471,6 +475,13 @@ task: "switch_port_config_manager"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "switch_port_config_manager" (don't know how to interpret details: Object {})
 
+task: "v2p_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/

dev-tools/omdb/tests/successes.out

@@ -47,7 +47,7 @@ struct FilterArgs {
     #[arg(short, long)]
     archived: bool,
 
-    // Print only the extra log files
+    /// Print only the extra log files
     #[arg(short, long)]
     extra: bool,
 

dev-tools/oxlog/src/bin/oxlog.rs

@@ -31,26 +31,16 @@ pub struct VpcFirewallRule {
 }
 
 /// A mapping from a virtual NIC to a physical host
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct SetVirtualNetworkInterfaceHost {
+#[derive(
+    Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct VirtualNetworkInterfaceHost {
     pub virtual_ip: IpAddr,
     pub virtual_mac: external::MacAddr,
     pub physical_host_ip: Ipv6Addr,
     pub vni: external::Vni,
 }
 
-/// The data needed to identify a virtual IP for which a sled maintains an OPTE
-/// virtual-to-physical mapping such that that mapping can be deleted.
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct DeleteVirtualNetworkInterfaceHost {
-    /// The virtual IP whose mapping should be deleted.
-    pub virtual_ip: IpAddr,
-
-    /// The VNI for the network containing the virtual IP whose mapping should
-    /// be deleted.
-    pub vni: external::Vni,
-}
-
 /// DHCP configuration for a port
 ///
 /// Not present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we

illumos-utils/src/opte/params.rs

@@ -5,8 +5,7 @@
 //! Manager for all OPTE ports on a Helios system
 
 use crate::opte::opte_firewall_rules;
-use crate::opte::params::DeleteVirtualNetworkInterfaceHost;
-use crate::opte::params::SetVirtualNetworkInterfaceHost;
+use crate::opte::params::VirtualNetworkInterfaceHost;
 use crate::opte::params::VpcFirewallRule;
 use crate::opte::Error;
 use crate::opte::Gateway;
@@ -570,10 +569,62 @@ impl PortManager {
         Ok(())
     }
 
+    #[cfg(target_os = "illumos")]
+    pub fn list_virtual_nics(
+        &self,
+    ) -> Result<Vec<VirtualNetworkInterfaceHost>, Error> {
+        use macaddr::MacAddr6;
+        use opte_ioctl::OpteHdl;
+
+        let hdl = OpteHdl::open(OpteHdl::XDE_CTL)?;
+        let v2p =
+            hdl.dump_v2p(&oxide_vpc::api::DumpVirt2PhysReq { unused: 99 })?;
+        let mut mappings: Vec<_> = vec![];
+
+        for mapping in v2p.mappings {
+            let vni = mapping
+                .vni
+                .as_u32()
+                .try_into()
+                .expect("opte VNI should be 24 bits");
+
+            for entry in mapping.ip4 {
+                mappings.push(VirtualNetworkInterfaceHost {
+                    virtual_ip: IpAddr::V4(entry.0.into()),
+                    virtual_mac: MacAddr6::from(entry.1.ether.bytes()).into(),
+                    physical_host_ip: entry.1.ip.into(),
+                    vni,
+                });
+            }
+
+            for entry in mapping.ip6 {
+                mappings.push(VirtualNetworkInterfaceHost {
+                    virtual_ip: IpAddr::V6(entry.0.into()),
+                    virtual_mac: MacAddr6::from(entry.1.ether.bytes()).into(),
+                    physical_host_ip: entry.1.ip.into(),
+                    vni,
+                });
+            }
+        }
+
+        Ok(mappings)
+    }
+
+    #[cfg(not(target_os = "illumos"))]
+    pub fn list_virtual_nics(
+        &self,
+    ) -> Result<Vec<VirtualNetworkInterfaceHost>, Error> {
+        info!(
+            self.inner.log,
+            "Listing virtual nics (ignored)";
+        );
+        Ok(vec![])
+    }
+
     #[cfg(target_os = "illumos")]
     pub fn set_virtual_nic_host(
         &self,
-        mapping: &SetVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
         use opte_ioctl::OpteHdl;
 
@@ -600,7 +651,7 @@ impl PortManager {
     #[cfg(not(target_os = "illumos"))]
     pub fn set_virtual_nic_host(
         &self,
-        mapping: &SetVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
         info!(
             self.inner.log,
@@ -613,20 +664,41 @@ impl PortManager {
     #[cfg(target_os = "illumos")]
     pub fn unset_virtual_nic_host(
         &self,
-        _mapping: &DeleteVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
-        // TODO requires https://github.com/oxidecomputer/opte/issues/332
+        use opte_ioctl::OpteHdl;
+
+        info!(
+            self.inner.log,
+            "Clearing mapping of virtual NIC to physical host";
+            "mapping" => ?&mapping,
+        );
+
+        let hdl = OpteHdl::open(OpteHdl::XDE_CTL)?;
+        hdl.clear_v2p(&oxide_vpc::api::ClearVirt2PhysReq {
+            vip: mapping.virtual_ip.into(),
+            phys: oxide_vpc::api::PhysNet {
+                ether: oxide_vpc::api::MacAddr::from(
+                    (*mapping.virtual_mac).into_array(),
+                ),
+                ip: mapping.physical_host_ip.into(),
+                vni: Vni::new(mapping.vni).unwrap(),
+            },
+        })?;
 
-        slog::warn!(self.inner.log, "unset_virtual_nic_host unimplmented");
         Ok(())
     }
 
     #[cfg(not(target_os = "illumos"))]
     pub fn unset_virtual_nic_host(
         &self,
-        _mapping: &DeleteVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
-        info!(self.inner.log, "Ignoring unset of virtual NIC mapping");
+        info!(
+            self.inner.log,
+            "Ignoring unset of virtual NIC mapping";
+            "mapping" => ?&mapping,
+        );
         Ok(())
     }
 }

illumos-utils/src/opte/port_manager.rs

@@ -379,6 +379,8 @@ pub struct BackgroundTaskConfig {
     pub instance_watcher: InstanceWatcherConfig,
     /// configuration for service VPC firewall propagation task
     pub service_firewall_propagation: ServiceFirewallPropagationConfig,
+    /// configuration for v2p mapping propagation task
+    pub v2p_mapping_propagation: V2PMappingPropagationConfig,
 }
 
 #[serde_as]
@@ -539,6 +541,14 @@ pub struct ServiceFirewallPropagationConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct V2PMappingPropagationConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 /// Configuration for a nexus server
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct PackageConfig {
@@ -777,6 +787,7 @@ mod test {
             region_replacement.period_secs = 30
             instance_watcher.period_secs = 30
             service_firewall_propagation.period_secs = 300
+            v2p_mapping_propagation.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -911,7 +922,10 @@ mod test {
                         service_firewall_propagation:
                             ServiceFirewallPropagationConfig {
                                 period_secs: Duration::from_secs(300),
-                            }
+                            },
+                        v2p_mapping_propagation: V2PMappingPropagationConfig {
+                            period_secs: Duration::from_secs(30)
+                        },
                     },
                     default_region_allocation_strategy:
                         crate::nexus_config::RegionAllocationStrategy::Random {
@@ -980,6 +994,7 @@ mod test {
             region_replacement.period_secs = 30
             instance_watcher.period_secs = 30
             service_firewall_propagation.period_secs = 300
+            v2p_mapping_propagation.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
             "##,

nexus-config/src/nexus_config.rs

@@ -55,6 +55,7 @@ mod project;
 mod semver_version;
 mod switch_interface;
 mod switch_port;
+mod v2p_mapping;
 // These actually represent subqueries, not real table.
 // However, they must be defined in the same crate as our tables
 // for join-based marker trait generation.
@@ -188,6 +189,7 @@ pub use typed_uuid::to_db_typed_uuid;
 pub use upstairs_repair::*;
 pub use user_builtin::*;
 pub use utilization::*;
+pub use v2p_mapping::*;
 pub use virtual_provisioning_collection::*;
 pub use virtual_provisioning_resource::*;
 pub use vmm::*;