diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index fa7ec0b1c9f..629d3f6ecc8 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -385,7 +385,9 @@ CREATE TABLE omicron.public.Zpool ( CREATE TYPE omicron.public.dataset_kind AS ENUM ( 'crucible', 'cockroach', - 'clickhouse' + 'clickhouse', + 'external_dns', + 'internal_dns' ); /* diff --git a/nexus/db-model/src/dataset_kind.rs b/nexus/db-model/src/dataset_kind.rs index e2c0510ab3d..f4c6a5eee66 100644 --- a/nexus/db-model/src/dataset_kind.rs +++ b/nexus/db-model/src/dataset_kind.rs @@ -19,6 +19,8 @@ impl_enum_type!( Crucible => b"crucible" Cockroach => b"cockroach" Clickhouse => b"clickhouse" + ExternalDns => b"external_dns" + InternalDns => b"internal_dns" ); impl From for DatasetKind { @@ -33,6 +35,12 @@ impl From for DatasetKind { internal_api::params::DatasetKind::Clickhouse => { DatasetKind::Clickhouse } + internal_api::params::DatasetKind::ExternalDns => { + DatasetKind::ExternalDns + } + internal_api::params::DatasetKind::InternalDns => { + DatasetKind::InternalDns + } } } } diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index dd45e16a1c5..a062b5a9870 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -17,7 +17,6 @@ use std::fmt; use std::net::IpAddr; use std::net::SocketAddr; use std::net::SocketAddrV6; -use std::str::FromStr; use uuid::Uuid; /// Describes the role of the sled within the rack. @@ -134,6 +133,8 @@ pub enum DatasetKind { Crucible, Cockroach, Clickhouse, + ExternalDns, + InternalDns, } impl fmt::Display for DatasetKind { @@ -143,27 +144,13 @@ impl fmt::Display for DatasetKind { Crucible => "crucible", Cockroach => "cockroach", Clickhouse => "clickhouse", + ExternalDns => "external_dns", + InternalDns => "internal_dns", }; write!(f, "{}", s) } } -impl FromStr for DatasetKind { - type Err = omicron_common::api::external::Error; - - fn from_str(s: &str) -> Result { - use DatasetKind::*; - match s { - "crucible" => Ok(Crucible), - "cockroach" => Ok(Cockroach), - "clickhouse" => Ok(Clickhouse), - _ => Err(Self::Err::InternalError { - internal_message: format!("Unknown dataset kind: {}", s), - }), - } - } -} - /// Describes a dataset within a pool. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct DatasetPutRequest { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 656c876bf97..67155bff979 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -931,7 +931,9 @@ "enum": [ "crucible", "cockroach", - "clickhouse" + "clickhouse", + "external_dns", + "internal_dns" ] }, "DatasetPutRequest": { diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 8f45de2e6c5..f7e89787499 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -617,6 +617,12 @@ "dataset_kind": { "$ref": "#/components/schemas/DatasetKind" }, + "gz_address": { + "nullable": true, + "default": null, + "type": "string", + "format": "ipv6" + }, "id": { "type": "string", "format": "uuid" @@ -677,6 +683,63 @@ "required": [ "type" ] + }, + { + "type": "object", + "properties": { + "dns_address": { + "description": "The address at which the external DNS server is reachable.", + "type": "string" + }, + "http_address": { + "description": "The address at which the external DNS server API is reachable.", + "type": "string" + }, + "nic": { + "description": "The service vNIC providing external connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/components/schemas/NetworkInterface" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "external_dns" + ] + } + }, + "required": [ + "dns_address", + "http_address", + "nic", + "type" + ] + }, + { + "type": "object", + "properties": { + "dns_address": { + "description": "The address at which the internal DNS server is reachable.", + "type": "string" + }, + "http_address": { + "description": "The address at which the internal DNS server API is reachable.", + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "internal_dns" + ] + } + }, + "required": [ + "dns_address", + "http_address", + "type" + ] } ] }, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 6d81d1b2d49..fd2b4b0ac20 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -124,6 +124,7 @@ async fn filesystem_put( body_args.zpool_id, body_args.dataset_kind, body_args.address, + body_args.gz_address, ) .await .map_err(|e| Error::from(e))?; diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index cfffd1fe823..868d293caa8 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -216,6 +216,20 @@ pub enum DatasetKind { CockroachDb, Crucible, Clickhouse, + ExternalDns { + /// The address at which the external DNS server API is reachable. + http_address: SocketAddrV6, + /// The address at which the external DNS server is reachable. + dns_address: SocketAddr, + /// The service vNIC providing external connectivity using OPTE. + nic: NetworkInterface, + }, + InternalDns { + /// The address at which the internal DNS server API is reachable. + http_address: SocketAddrV6, + /// The address at which the internal DNS server is reachable. + dns_address: SocketAddrV6, + }, } impl DatasetKind { @@ -225,6 +239,8 @@ impl DatasetKind { DatasetKind::CockroachDb => ZoneType::CockroachDb, DatasetKind::Crucible => ZoneType::Crucible, DatasetKind::Clickhouse => ZoneType::Clickhouse, + DatasetKind::ExternalDns { .. } => ZoneType::ExternalDns, + DatasetKind::InternalDns { .. } => ZoneType::InternalDns, } } @@ -234,10 +250,16 @@ impl DatasetKind { /// service in their zone. If that precondition is no longer true, this /// interface should be re-visited. pub fn service_type(&self) -> ServiceType { - match *self { + match self.clone() { DatasetKind::CockroachDb => ServiceType::CockroachDb, DatasetKind::Crucible => ServiceType::Crucible, DatasetKind::Clickhouse => ServiceType::Clickhouse, + DatasetKind::ExternalDns { http_address, dns_address, nic } => { + ServiceType::ExternalDns { http_address, dns_address, nic } + } + DatasetKind::InternalDns { http_address, dns_address } => { + ServiceType::InternalDns { http_address, dns_address } + } } } } @@ -249,6 +271,17 @@ impl From for sled_agent_client::types::DatasetKind { CockroachDb => Self::CockroachDb, Crucible => Self::Crucible, Clickhouse => Self::Clickhouse, + ExternalDns { http_address, dns_address, nic } => { + Self::ExternalDns { + http_address: http_address.to_string(), + dns_address: dns_address.to_string(), + nic: nic.into(), + } + } + InternalDns { http_address, dns_address } => Self::InternalDns { + http_address: http_address.to_string(), + dns_address: dns_address.to_string(), + }, } } } @@ -260,6 +293,8 @@ impl From for nexus_client::types::DatasetKind { CockroachDb { .. } => Self::Cockroach, Crucible => Self::Crucible, Clickhouse => Self::Clickhouse, + ExternalDns { .. } => Self::ExternalDns, + InternalDns { .. } => Self::InternalDns, } } } @@ -271,6 +306,8 @@ impl std::fmt::Display for DatasetKind { Crucible => "crucible", CockroachDb { .. } => "cockroachdb", Clickhouse => "clickhouse", + ExternalDns { .. } => "external_dns", + InternalDns { .. } => "internal_dns", }; write!(f, "{}", s) } @@ -290,6 +327,10 @@ pub struct DatasetEnsureBody { pub dataset_kind: DatasetKind, // The address on which the zone will listen for requests. pub address: SocketAddrV6, + // The addresses in the global zone which should be created, if necessary + // to route to the service. + #[serde(default)] + pub gz_address: Option, } impl From for sled_agent_client::types::DatasetEnsureBody { @@ -298,6 +339,7 @@ impl From for sled_agent_client::types::DatasetEnsureBody { zpool_id: p.zpool_id, dataset_kind: p.dataset_kind.into(), address: p.address.to_string(), + gz_address: p.gz_address, id: p.id, } } diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 7323d9f2d8f..f102d25c3a9 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -284,7 +284,8 @@ impl Plan { if idx < EXTERNAL_DNS_COUNT { let internal_ip = addr_alloc.next().expect("Not enough addrs"); let http_port = omicron_common::address::DNS_HTTP_PORT; - let dns_port = omicron_common::address::DNS_PORT; + let http_address = + SocketAddrV6::new(internal_ip, http_port, 0, 0); let id = Uuid::new_v4(); let zone = dns_builder.host_zone(id, internal_ip).unwrap(); dns_builder @@ -296,26 +297,41 @@ impl Plan { .unwrap(); let (nic, external_ip) = svc_port_builder.next_dns(id, &mut services_ip_pool)?; + let dns_port = omicron_common::address::DNS_PORT; + let dns_address = SocketAddr::new(external_ip, dns_port); + let dataset_kind = crate::params::DatasetKind::ExternalDns { + http_address, + dns_address, + nic: nic.clone(), + }; + + request.datasets.push(DatasetEnsureBody { + id, + zpool_id: u2_zpools[0], + dataset_kind: dataset_kind.clone(), + address: http_address, + gz_address: None, + }); request.services.push(ServiceZoneRequest { id, zone_type: ZoneType::ExternalDns, addresses: vec![internal_ip], - dataset: None, + dataset: Some(crate::storage::dataset::DatasetName::new( + illumos_utils::zpool::ZpoolName::new_external( + u2_zpools[0], + ), + dataset_kind, + )), gz_addresses: vec![], services: vec![ServiceZoneService { id, details: ServiceType::ExternalDns { - http_address: SocketAddrV6::new( - internal_ip, - http_port, - 0, - 0, - ), - dns_address: SocketAddr::new(external_ip, dns_port), + http_address, + dns_address, nic, }, }], - }) + }); } // The first enumerated sleds get assigned the responsibility @@ -388,64 +404,64 @@ impl Plan { // zpools described from the underlying config file. if idx < CRDB_COUNT { let id = Uuid::new_v4(); - let address = addr_alloc.next().expect("Not enough addrs"); + let ip = addr_alloc.next().expect("Not enough addrs"); let port = omicron_common::address::COCKROACH_PORT; - let zone = dns_builder.host_zone(id, address).unwrap(); + let address = SocketAddrV6::new(ip, port, 0, 0); + let zone = dns_builder.host_zone(id, ip).unwrap(); dns_builder .service_backend_zone(ServiceName::Cockroach, &zone, port) .unwrap(); - let address = SocketAddrV6::new(address, port, 0, 0); request.datasets.push(DatasetEnsureBody { id, zpool_id: u2_zpools[0], dataset_kind: crate::params::DatasetKind::CockroachDb, address, + gz_address: None, }); } // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove if idx < CLICKHOUSE_COUNT { let id = Uuid::new_v4(); - let address = addr_alloc.next().expect("Not enough addrs"); + let ip = addr_alloc.next().expect("Not enough addrs"); let port = omicron_common::address::CLICKHOUSE_PORT; - let zone = dns_builder.host_zone(id, address).unwrap(); + let address = SocketAddrV6::new(ip, port, 0, 0); + let zone = dns_builder.host_zone(id, ip).unwrap(); dns_builder .service_backend_zone(ServiceName::Clickhouse, &zone, port) .unwrap(); - let address = SocketAddrV6::new(address, port, 0, 0); request.datasets.push(DatasetEnsureBody { id, zpool_id: u2_zpools[0], dataset_kind: crate::params::DatasetKind::Clickhouse, address, + gz_address: None, }); } // Each zpool gets a crucible zone. // // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove - for zpool_id in u2_zpools { - let address = SocketAddrV6::new( - addr_alloc.next().expect("Not enough addrs"), - omicron_common::address::CRUCIBLE_PORT, - 0, - 0, - ); + for zpool_id in &u2_zpools { + let ip = addr_alloc.next().expect("Not enough addrs"); + let port = omicron_common::address::CRUCIBLE_PORT; + let address = SocketAddrV6::new(ip, port, 0, 0); let id = Uuid::new_v4(); - let zone = dns_builder.host_zone(id, *address.ip()).unwrap(); + let zone = dns_builder.host_zone(id, ip).unwrap(); dns_builder .service_backend_zone( ServiceName::Crucible(id), &zone, - address.port(), + port, ) .unwrap(); request.datasets.push(DatasetEnsureBody { id, - zpool_id, + zpool_id: *zpool_id, dataset_kind: crate::params::DatasetKind::Crucible, address, + gz_address: None, }); } @@ -453,9 +469,12 @@ impl Plan { // responsibility of being internal DNS servers. if idx < dns_subnets.len() { let dns_subnet = &dns_subnets[idx]; - let dns_addr = dns_subnet.dns_address().ip(); + let dns_ip = dns_subnet.dns_address().ip(); + let dns_address = SocketAddrV6::new(dns_ip, DNS_PORT, 0, 0); + let http_address = + SocketAddrV6::new(dns_ip, DNS_HTTP_PORT, 0, 0); let id = Uuid::new_v4(); - let zone = dns_builder.host_zone(id, dns_addr).unwrap(); + let zone = dns_builder.host_zone(id, dns_ip).unwrap(); dns_builder .service_backend_zone( ServiceName::InternalDns, @@ -463,26 +482,15 @@ impl Plan { DNS_HTTP_PORT, ) .unwrap(); - request.services.push(ServiceZoneRequest { + request.datasets.push(DatasetEnsureBody { id, - zone_type: ZoneType::InternalDns, - addresses: vec![dns_addr], - dataset: None, - gz_addresses: vec![dns_subnet.gz_address().ip()], - services: vec![ServiceZoneService { - id, - details: ServiceType::InternalDns { - http_address: SocketAddrV6::new( - dns_addr, - DNS_HTTP_PORT, - 0, - 0, - ), - dns_address: SocketAddrV6::new( - dns_addr, DNS_PORT, 0, 0, - ), - }, - }], + zpool_id: u2_zpools[0], + dataset_kind: crate::params::DatasetKind::InternalDns { + http_address, + dns_address, + }, + address: http_address, + gz_address: Some(dns_subnet.gz_address().ip()), }); } diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 04404e23c81..f304dd087f2 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -62,7 +62,7 @@ use crate::bootstrap::rss_handle::BootstrapAgentHandle; use crate::ledger::{Ledger, Ledgerable}; use crate::nexus::d2n_params; use crate::params::{ - AutonomousServiceOnlyError, DatasetEnsureBody, ServiceType, + AutonomousServiceOnlyError, DatasetEnsureBody, DatasetKind, ServiceType, ServiceZoneRequest, TimeSync, ZoneType, }; use crate::rack_setup::plan::service::{ @@ -381,21 +381,23 @@ impl ServiceInner { // Start up the internal DNS services futures::future::join_all(service_plan.services.iter().map( |(sled_address, services_request)| async move { - let services: Vec<_> = services_request - .services + let datasets: Vec<_> = services_request + .datasets .iter() - .filter_map(|svc| { - if matches!(svc.zone_type, ZoneType::InternalDns) { - Some(svc.clone()) + .filter_map(|dataset| { + if matches!( + dataset.dataset_kind, + DatasetKind::InternalDns { .. } + ) { + Some(dataset.clone()) } else { None } }) .collect(); - if !services.is_empty() { - self.initialize_services(*sled_address, &services).await?; + if !datasets.is_empty() { + self.initialize_datasets(*sled_address, &datasets).await?; } - Ok(()) }, )) @@ -410,39 +412,13 @@ impl ServiceInner { service_plan.services.iter().filter_map( |(_, services_request)| { // iterate services for this sled - let dns_addrs: Vec<_> = services_request - .services + let dns_addrs: Vec = services_request + .datasets .iter() - .filter_map(|svc| { - if !matches!(svc.zone_type, ZoneType::InternalDns) { - // This is not an internal DNS zone. - None - } else { - // This is an internal DNS zone. Find the IP - // and port that have been assigned to it. - // There should be exactly one. - let addrs = svc.services.iter().filter_map(|s| { - if let ServiceType::InternalDns { http_address, .. } = &s.details { - Some(*http_address) - } else { - None - } - }).collect::>(); - - if addrs.len() == 1 { - Some(addrs[0]) - } else { - warn!( - log, - "DNS configuration: expected one \ - InternalDns service for zone with \ - type ZoneType::InternalDns, but \ - found {} (zone {})", - addrs.len(), - svc.id, - ); - None - } + .filter_map(|dataset| { + match dataset.dataset_kind { + DatasetKind::InternalDns { http_address, .. } => Some(http_address), + _ => None, } }) .collect(); @@ -454,7 +430,7 @@ impl ServiceInner { } ) .flatten() - .collect::>(); + .collect::>(); let dns_config = &service_plan.dns_config; for ip_addr in dns_server_ips { diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index e22c6143cff..97cdd930136 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -565,7 +565,11 @@ impl ServiceManager { .map_err(|_| "already set".to_string()) .expect("Sled Agent should only start once"); - self.load_non_storage_services().await?; + self.load_non_storage_services().await.map_err(|e| { + error!(self.inner.log, "failed to launch non-storage services"; "error" => e.to_string()); + e + })?; + // TODO(https://github.com/oxidecomputer/omicron/issues/2973): // These will fail if the disks aren't attached. // Should we have a retry loop here? Kinda like we have with the switch @@ -573,7 +577,10 @@ impl ServiceManager { // // NOTE: We could totally do the same thing with // "load_non_storage_services". - self.load_storage_services().await?; + self.load_storage_services().await.map_err(|e| { + error!(self.inner.log, "failed to launch storage services"; "error" => e.to_string()); + e + })?; Ok(()) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 0e63112966e..f437b9d0b0a 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -554,6 +554,7 @@ impl SledAgent { zpool_id: Uuid, dataset_kind: DatasetKind, address: SocketAddrV6, + gz_address: Option, ) -> Result<(), Error> { // First, ensure the dataset exists let dataset = self @@ -580,7 +581,7 @@ impl SledAgent { zone_type: dataset_kind.zone_type(), addresses: vec![*address.ip()], dataset: Some(dataset), - gz_addresses: vec![], + gz_addresses: gz_address.into_iter().collect(), services, }; self.inner.services.ensure_storage_service(request).await?; diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index c573dbe6a2c..cf2d5a43f6f 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -84,7 +84,7 @@ pub enum Error { }, #[error("Dataset {name:?} exists with a different uuid (has {old}, requested {new})")] - UuidMismatch { name: DatasetName, old: Uuid, new: Uuid }, + UuidMismatch { name: Box, old: Uuid, new: Uuid }, #[error("Error parsing pool {name}'s size: {err}")] BadPoolSize { @@ -327,7 +327,7 @@ impl StorageWorker { if let Ok(id) = id_str.parse::() { if id != dataset_id { return Err(Error::UuidMismatch { - name: dataset_name.clone(), + name: Box::new(dataset_name.clone()), old: id, new: dataset_id, }); diff --git a/smf/external-dns/config.toml b/smf/external-dns/config.toml index 4f572ddbc1b..b90224f48c2 100644 --- a/smf/external-dns/config.toml +++ b/smf/external-dns/config.toml @@ -11,5 +11,5 @@ path = "/dev/stdout" if_exists = "append" [storage] -storage_path = "/var/oxide/dns" +storage_path = "/data" keep_old_generations = 3 diff --git a/smf/internal-dns/config.toml b/smf/internal-dns/config.toml index 4f572ddbc1b..b90224f48c2 100644 --- a/smf/internal-dns/config.toml +++ b/smf/internal-dns/config.toml @@ -11,5 +11,5 @@ path = "/dev/stdout" if_exists = "append" [storage] -storage_path = "/var/oxide/dns" +storage_path = "/data" keep_old_generations = 3