Allow configuring control_plane_storage_buffer

.github/buildomat/jobs/deploy.sh

@@ -429,18 +429,6 @@ do
 	ACTUAL_ZPOOL_COUNT=$(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i | wc -l)
 done
 
-# The bootstrap command creates a disk, so before that: adjust the control plane
-# storage buffer to 0 as the virtual hardware only creates 20G pools
-
-pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list
-
-for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i);
-do
-	pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer "${ZPOOL}" 0
-done
-
-pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list
-
 export RUST_BACKTRACE=1
 export E2E_TLS_CERT IPPOOL_START IPPOOL_END
 eval "$(./target/debug/bootstrap)"

nexus/db-model/src/lib.rs

@@ -95,6 +95,7 @@ mod role_builtin;
 pub mod saga_types;
 mod schema_versions;
 mod service_kind;
+mod setting;
 mod silo;
 mod silo_group;
 mod silo_user;
@@ -207,6 +208,7 @@ pub use saga_types::*;
 pub use schema_versions::*;
 pub use semver_version::*;
 pub use service_kind::*;
+pub use setting::*;
 pub use silo::*;
 pub use silo_group::*;
 pub use silo_user::*;

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(140, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(141, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(141, "nexus-settings"),
         KnownVersion::new(140, "instance-intended-state"),
         KnownVersion::new(139, "webhooks"),
         KnownVersion::new(138, "saga-abandoned-state"),

nexus/db-model/src/setting.rs

@@ -0,0 +1,23 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use super::impl_enum_type;
+use nexus_db_schema::schema::setting;
+
+impl_enum_type!(
+    SettingNameEnum:
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, PartialEq)]
+    pub enum SettingName;
+
+    // Enum values
+    ControlPlaneStorageBuffer => b"control_plane_storage_buffer"
+);
+
+#[derive(Queryable, Insertable, Debug, Selectable, Clone)]
+#[diesel(table_name = setting)]
+pub struct Setting {
+    pub name: SettingName,
+    pub int_value: Option<i64>,
+}

nexus/db-queries/src/db/datastore/mod.rs

@@ -87,6 +87,7 @@ pub mod region_snapshot_replacement;
 mod rendezvous_debug_dataset;
 mod role;
 mod saga;
+mod setting;
 mod silo;
 mod silo_group;
 mod silo_user;

nexus/db-queries/src/db/datastore/rack.rs

@@ -90,6 +90,7 @@ pub struct RackInit {
     pub recovery_user_password_hash: omicron_passwords::PasswordHashString,
     pub dns_update: DnsVersionUpdateBuilder,
     pub allowed_source_ips: AllowedSourceIps,
+    pub control_plane_storage_buffer_gib: u32,
 }
 
 /// Possible errors while trying to initialize rack
@@ -112,6 +113,8 @@ enum RackInitError {
     Database(DieselError),
     // Error adding initial allowed source IP list
     AllowedSourceIpError(Error),
+    // Error changing a Nexus setting
+    ChangeSetting(Error),
 }
 
 // Catch-all for Diesel error conversion into RackInitError, which
@@ -175,6 +178,7 @@ impl From<RackInitError> for Error {
                 err
             )),
             RackInitError::AllowedSourceIpError(err) => err,
+            RackInitError::ChangeSetting(err) => err,
         }
     }
 }
@@ -911,6 +915,17 @@ impl DataStore {
                         DieselError::RollbackTransaction
                     })?;
 
+                    Self::set_control_plane_storage_buffer_gib_impl(
+                        opctx,
+                        &conn,
+                        rack_init.control_plane_storage_buffer_gib,
+                    )
+                    .await
+                    .map_err(|e| {
+                        err.set(RackInitError::ChangeSetting(e)).unwrap();
+                        DieselError::RollbackTransaction
+                    })?;
+
                     let rack = diesel::update(rack_dsl::rack)
                         .filter(rack_dsl::id.eq(rack_id))
                         .set((
@@ -1117,6 +1132,7 @@ mod test {
                     "test suite".to_string(),
                 ),
                 allowed_source_ips: AllowedSourceIps::Any,
+                control_plane_storage_buffer_gib: 0,
             }
         }
     }

nexus/db-queries/src/db/datastore/setting.rs

@@ -0,0 +1,97 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! [`DataStore`] methods on [`Setting`]s.
+
+// Settings are dynamically controlled values for Nexus
+
+use super::DataStore;
+use crate::authz;
+use crate::context::OpContext;
+use crate::db::datastore::ErrorHandler;
+use crate::db::datastore::public_error_from_diesel;
+use async_bb8_diesel::AsyncRunQueryDsl;
+use diesel::prelude::*;
+use nexus_db_lookup::DbConnection;
+use nexus_db_model::Setting;
+use nexus_db_model::SettingName;
+use omicron_common::api::external::ByteCount;
+use omicron_common::api::external::Error;
+
+impl DataStore {
+    pub(crate) async fn set_control_plane_storage_buffer_gib_impl(
+        opctx: &OpContext,
+        conn: &async_bb8_diesel::Connection<DbConnection>,
+        gibibytes: u32,
+    ) -> Result<(), Error> {
+        opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
+
+        use nexus_db_schema::schema::setting::dsl;
+
+        let name = SettingName::ControlPlaneStorageBuffer;
+        let value: i64 = ByteCount::from_gibibytes_u32(gibibytes).into();
+
+        let maybe_existing =
+            Self::get_control_plane_storage_buffer_impl(opctx, conn).await?;
+
+        if maybe_existing.is_some() {
+            diesel::update(dsl::setting)
+                .filter(dsl::name.eq(name))
+                .set(dsl::int_value.eq(value))
+                .execute_async(conn)
+                .await
+                .map(|_| ())
+                .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+        } else {
+            let setting = Setting { name, int_value: Some(value) };
+
+            diesel::insert_into(dsl::setting)
+                .values(setting)
+                .execute_async(conn)
+                .await
+                .map(|_| ())
+                .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+        }
+    }
+
+    pub(crate) async fn get_control_plane_storage_buffer_impl(
+        opctx: &OpContext,
+        conn: &async_bb8_diesel::Connection<DbConnection>,
+    ) -> Result<Option<ByteCount>, Error> {
+        opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
+
+        use nexus_db_schema::schema::setting::dsl;
+
+        let name = SettingName::ControlPlaneStorageBuffer;
+
+        let result = dsl::setting
+            .filter(dsl::name.eq(name))
+            .select(Setting::as_select())
+            .first_async(conn)
+            .await
+            .optional()
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(match result {
+            Some(setting) => {
+                // A row exists with this setting's name
+                match setting.int_value {
+                    Some(value) => Some(ByteCount::try_from(value).unwrap()),
+
+                    None => None,
+                }
+            }
+
+            None => None,
+        })
+    }
+
+    pub async fn get_control_plane_storage_buffer(
+        &self,
+        opctx: &OpContext,
+    ) -> Result<Option<ByteCount>, Error> {
+        let conn = self.pool_connection_authorized(opctx).await?;
+        Self::get_control_plane_storage_buffer_impl(opctx, &conn).await
+    }
+}

nexus/db-schema/src/enums.rs

@@ -41,8 +41,8 @@ define_enums! {
     IdentityProviderTypeEnum => "provider_type",
     IdentityTypeEnum => "identity_type",
     InstanceAutoRestartPolicyEnum => "instance_auto_restart",
-    InstanceStateEnum => "instance_state_v2",
     InstanceIntendedStateEnum => "instance_intended_state",
+    InstanceStateEnum => "instance_state_v2",
     IpAttachStateEnum => "ip_attach_state",
     IpKindEnum => "ip_kind",
     IpPoolResourceTypeEnum => "ip_pool_resource_type",
@@ -64,6 +64,7 @@ define_enums! {
     RouterRouteKindEnum => "router_route_kind",
     SagaStateEnum => "saga_state",
     ServiceKindEnum => "service_kind",
+    SettingNameEnum => "setting_name",
     SledPolicyEnum => "sled_policy",
     SledRoleEnum => "sled_role",
     SledStateEnum => "sled_state",
@@ -85,9 +86,9 @@ define_enums! {
     VpcFirewallRuleProtocolEnum => "vpc_firewall_rule_protocol",
     VpcFirewallRuleStatusEnum => "vpc_firewall_rule_status",
     VpcRouterKindEnum => "vpc_router_kind",
-    WebhookEventClassEnum => "webhook_event_class",
     WebhookDeliveryAttemptResultEnum => "webhook_delivery_attempt_result",
-    WebhookDeliveryTriggerEnum => "webhook_delivery_trigger",
     WebhookDeliveryStateEnum => "webhook_delivery_state",
+    WebhookDeliveryTriggerEnum => "webhook_delivery_trigger",
+    WebhookEventClassEnum => "webhook_event_class",
     ZoneTypeEnum => "zone_type",
 }

nexus/db-schema/src/schema.rs

@@ -2309,3 +2309,10 @@ allow_tables_to_appear_in_same_query!(
     webhook_delivery_attempt
 );
 joinable!(webhook_delivery_attempt -> webhook_delivery (delivery_id));
+
+table! {
+    setting (name) {
+        name -> crate::enums::SettingNameEnum,
+        int_value -> Nullable<Int8>,
+    }
+}

nexus/src/app/background/tasks/physical_disk_adoption.rs

@@ -11,7 +11,6 @@
 //!
 //! In the future, this may become more explicitly operator-controlled.
 
-use crate::app::CONTROL_PLANE_STORAGE_BUFFER;
 use crate::app::background::BackgroundTask;
 use futures::FutureExt;
 use futures::future::BoxFuture;
@@ -20,6 +19,7 @@ use nexus_db_model::Zpool;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::DataStore;
 use nexus_types::identity::Asset;
+use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::DataPageParams;
 use omicron_uuid_kinds::CollectionUuid;
 use omicron_uuid_kinds::GenericUuid;
@@ -62,6 +62,25 @@ impl BackgroundTask for PhysicalDiskAdoption {
                 return json!({ "error": "task disabled" });
             }
 
+            let control_plane_storage_buffer: ByteCount = match self
+                .datastore
+                .get_control_plane_storage_buffer(opctx)
+                .await {
+                    Ok(value) => match value {
+                        Some(value) => value,
+                        None => {
+                            // If no setting is in the database, use 0
+                            ByteCount::from(0)
+                        }
+                    },
+
+                    Err(e) => {
+                        return json!({ "error": format!(
+                            "error getting control plane storage buffer: {e}"
+                        )});
+                    }
+                };
+
             // Only adopt physical disks after rack handoff has completed.
             //
             // This prevents a race condition where the same physical disks
@@ -140,7 +159,7 @@ impl BackgroundTask for PhysicalDiskAdoption {
                     Uuid::new_v4(),
                     inv_disk.sled_id.into_untyped_uuid(),
                     disk.id(),
-                    CONTROL_PLANE_STORAGE_BUFFER.into(),
+                    control_plane_storage_buffer.into(),
                 );
 
                 let result = self.datastore.physical_disk_and_zpool_insert(

nexus/src/app/mod.rs

@@ -31,7 +31,6 @@ use nexus_types::deployment::PendingMgsUpdates;
 use omicron_common::address::DENDRITE_PORT;
 use omicron_common::address::MGD_PORT;
 use omicron_common::address::MGS_PORT;
-use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::Error;
 use omicron_common::api::internal::shared::SwitchLocation;
 use omicron_uuid_kinds::OmicronZoneUuid;
@@ -135,14 +134,6 @@ pub const MAX_DISK_SIZE_BYTES: u64 = 1023 * (1 << 30); // 1023 GiB
 /// This value is aribtrary
 pub const MAX_SSH_KEYS_PER_INSTANCE: u32 = 100;
 
-/// The amount of disk space to reserve for non-Crucible / control plane
-/// storage. This amount represents a buffer that the region allocation query
-/// will not use for each U2.
-///
-/// See oxidecomputer/omicron#7875 for the 250G determination.
-pub const CONTROL_PLANE_STORAGE_BUFFER: ByteCount =
-    ByteCount::from_gibibytes_u32(250);
-
 /// Manages an Oxide fleet -- the heart of the control plane
 pub struct Nexus {
     /// uuid for this nexus instance.

nexus/src/app/rack.rs

@@ -4,7 +4,6 @@
 
 //! Rack management
 
-use crate::app::CONTROL_PLANE_STORAGE_BUFFER;
 use crate::external_api::params;
 use crate::external_api::params::CertificateCreate;
 use crate::external_api::shared::ServiceUsingCertificate;
@@ -53,6 +52,7 @@ use nexus_types::silo::silo_dns_name;
 use omicron_common::address::{Ipv6Subnet, RACK_PREFIX, get_64_subnet};
 use omicron_common::api::external::AddressLotKind;
 use omicron_common::api::external::BgpPeer;
+use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
@@ -137,7 +137,10 @@ impl super::Nexus {
                     pool.id,
                     pool.sled_id,
                     pool.physical_disk_id,
-                    CONTROL_PLANE_STORAGE_BUFFER.into(),
+                    ByteCount::from_gibibytes_u32(
+                        request.control_plane_storage_buffer_gib,
+                    )
+                    .into(),
                 )
             })
             .collect();
@@ -728,6 +731,8 @@ impl super::Nexus {
                         .into(),
                     dns_update,
                     allowed_source_ips: request.allowed_source_ips,
+                    control_plane_storage_buffer_gib: request
+                        .control_plane_storage_buffer_gib,
                 },
             )
             .await?;

nexus/src/lib.rs

@@ -330,6 +330,7 @@ impl nexus_test_interface::NexusServer for Server {
                         bfd: Vec::new(),
                     },
                     allowed_source_ips: AllowedSourceIps::Any,
+                    control_plane_storage_buffer_gib: 0,
                 },
             )
             .await