Wrap zone bundling into a single type

- Adds the `ZoneBundler` for managing the creation and listing of
  zones, and running background cleanup task. This is created pretty
  early, by the `StorageManager`, and cloned to the other objects that
  need access to bundles.
- Move public free functions from the `zone_bundle` crate into methods
  on the `ZoneBundler`, and call them from the instance / service
  managers and sled agent.
- Make the "list-all-zones" endpoint in the sled agent accept a filter
  directly and propagate to the listing routines inside the bundler.
  This removes filtering on the client.
- Additional tests around listing / filtering.

Author: bnaecker

Cargo.lock

@@ -603,20 +603,20 @@
         }
       }
     },
-    "/zones/bundle-cleanup/usage": {
+    "/zones/bundle-cleanup/utilization": {
       "get": {
-        "summary": "Return usage information about all zone bundles.",
-        "operationId": "zone_bundle_usage",
+        "summary": "Return utilization information about all zone bundles.",
+        "operationId": "zone_bundle_utilization",
         "responses": {
           "200": {
             "description": "successful operation",
             "content": {
               "application/json": {
                 "schema": {
-                  "title": "Map_of_BundleUsage",
+                  "title": "Map_of_BundleUtilization",
                   "type": "object",
                   "additionalProperties": {
-                    "$ref": "#/components/schemas/BundleUsage"
+                    "$ref": "#/components/schemas/BundleUtilization"
                   }
                 }
               }
@@ -635,6 +635,17 @@
       "get": {
         "summary": "List all zone bundles that exist, even for now-deleted zones.",
         "operationId": "zone_bundle_list_all",
+        "parameters": [
+          {
+            "in": "query",
+            "name": "filter",
+            "description": "An optional substring used to filter zone bundles.",
+            "schema": {
+              "nullable": true,
+              "type": "string"
+            }
+          }
+        ],
         "responses": {
           "200": {
             "description": "successful operation",
@@ -852,7 +863,7 @@
       }
     },
     "schemas": {
-      "BundleUsage": {
+      "BundleUtilization": {
         "description": "The portion of a debug dataset used for zone bundles.",
         "type": "object",
         "properties": {
@@ -907,13 +918,6 @@
               }
             ]
           },
-          "storage_dirs": {
-            "description": "The directories searched for bundles to prune.",
-            "type": "array",
-            "items": {
-              "type": "string"
-            }
-          },
           "storage_limit": {
             "description": "The limit on the dataset quota available for zone bundles.",
             "allOf": [
@@ -926,7 +930,6 @@
         "required": [
           "period",
           "priority",
-          "storage_dirs",
           "storage_limit"
         ]
       },

openapi/sled-agent.json

@@ -67,7 +67,6 @@ slog-term.workspace = true
 smf.workspace = true
 sp-sim.workspace = true
 tar.workspace = true
-tempfile.workspace = true
 thiserror.workspace = true
 tofino.workspace = true
 tokio = { workspace = true, features = [ "full" ] }

sled-agent/Cargo.toml

@@ -82,7 +82,7 @@ impl ListFields {
 }
 
 #[derive(Clone, Copy, Debug, clap::ValueEnum)]
-enum UsageFields {
+enum UtilizationFields {
     Directory,
     BytesUsed,
     BytesAvailable,
@@ -91,9 +91,9 @@ enum UsageFields {
     PctQuota,
 }
 
-impl std::fmt::Display for UsageFields {
+impl std::fmt::Display for UtilizationFields {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        use UsageFields::*;
+        use UtilizationFields::*;
         match self {
             Directory => write!(f, "directory"),
             BytesUsed => write!(f, "bytes-used"),
@@ -105,9 +105,9 @@ impl std::fmt::Display for UsageFields {
     }
 }
 
-impl UsageFields {
+impl UtilizationFields {
     fn all() -> Vec<Self> {
-        use UsageFields::*;
+        use UtilizationFields::*;
         vec![
             Directory,
             BytesUsed,
@@ -178,14 +178,14 @@ enum Cmd {
     /// Set parameters of the zone bundle cleanup context.
     #[clap(visible_alias = "set-context")]
     SetCleanupContext(SetCleanupContextArgs),
-    /// Return the usage of the datasets allocated for zone bundles.
-    Usage {
+    /// Return the utilization of the datasets allocated for zone bundles.
+    Utilization {
         /// Generate parseable output.
         #[arg(long, short, default_value_t = false)]
         parseable: bool,
         /// Fields to print.
-        #[arg(long, short = 'o', default_values_t = UsageFields::all(), value_delimiter = ',')]
-        fields: Vec<UsageFields>,
+        #[arg(long, short = 'o', default_values_t = UtilizationFields::all(), value_delimiter = ',')]
+        fields: Vec<UtilizationFields>,
     },
     /// Trigger an explicit request to cleanup low-priority zone bundles.
     Cleanup,
@@ -234,19 +234,10 @@ async fn main() -> anyhow::Result<()> {
         }
         Cmd::List { filter, parseable, fields } => {
             let bundles = client
-                .zone_bundle_list_all()
+                .zone_bundle_list_all(filter.as_deref())
                 .await
                 .context("failed to list zone bundles")?
-                .into_inner()
-                .into_iter()
-                .filter(|bundle| {
-                    if let Some(filter) = &filter {
-                        bundle.id.zone_name.contains(filter)
-                    } else {
-                        true
-                    }
-                })
-                .collect::<Vec<_>>();
+                .into_inner();
             if bundles.is_empty() {
                 return Ok(());
             }
@@ -383,7 +374,6 @@ async fn main() -> anyhow::Result<()> {
                 .zone_bundle_cleanup_context()
                 .await
                 .context("failed to fetch cleanup context")?;
-            println!("Storage directories: {:?}", context.storage_dirs);
             println!("Period: {}s", context.period.0.secs);
             println!("Priority: {:?}", context.priority.0);
             println!("Storage limit: {}%", context.storage_limit.0);
@@ -408,42 +398,44 @@ async fn main() -> anyhow::Result<()> {
                 .await
                 .context("failed to update zone bundle cleanup context")?;
         }
-        Cmd::Usage { parseable, fields } => {
-            let usage_by_dir = client
-                .zone_bundle_usage()
+        Cmd::Utilization { parseable, fields } => {
+            let utilization_by_dir = client
+                .zone_bundle_utilization()
                 .await
-                .context("failed to get zone bundle usage")?;
+                .context("failed to get zone bundle utilization")?;
             const BYTES_USED_SIZE: usize = 16;
             const BYTES_AVAIL_SIZE: usize = 16;
             const QUOTA_SIZE: usize = 16;
             const PCT_OF_AVAIL_SIZE: usize = 10;
             const PCT_OF_QUOTA_SIZE: usize = 10;
-            if !usage_by_dir.is_empty() {
-                use UsageFields::*;
+            if !utilization_by_dir.is_empty() {
+                use UtilizationFields::*;
                 if parseable {
-                    for (dir, usage) in usage_by_dir.iter() {
+                    for (dir, utilization) in utilization_by_dir.iter() {
                         for (i, field) in fields.iter().enumerate() {
                             match field {
                                 Directory => print!("{}", dir),
-                                BytesUsed => print!("{}", usage.bytes_used),
+                                BytesUsed => {
+                                    print!("{}", utilization.bytes_used)
+                                }
                                 BytesAvailable => {
-                                    print!("{}", usage.bytes_available)
+                                    print!("{}", utilization.bytes_available)
                                 }
                                 DatasetQuota => {
-                                    print!("{}", usage.dataset_quota)
+                                    print!("{}", utilization.dataset_quota)
                                 }
                                 PctAvailable => print!(
                                     "{}",
                                     as_pct(
-                                        usage.bytes_used,
-                                        usage.bytes_available
+                                        utilization.bytes_used,
+                                        utilization.bytes_available
                                     )
                                 ),
                                 PctQuota => print!(
                                     "{}",
                                     as_pct(
-                                        usage.bytes_used,
-                                        usage.dataset_quota
+                                        utilization.bytes_used,
+                                        utilization.dataset_quota
                                     )
                                 ),
                             }
@@ -454,8 +446,11 @@ async fn main() -> anyhow::Result<()> {
                         println!();
                     }
                 } else {
-                    let dir_col_size =
-                        usage_by_dir.keys().map(|d| d.len()).max().unwrap();
+                    let dir_col_size = utilization_by_dir
+                        .keys()
+                        .map(|d| d.len())
+                        .max()
+                        .unwrap();
                     for field in fields.iter() {
                         match field {
                             Directory => {
@@ -481,34 +476,34 @@ async fn main() -> anyhow::Result<()> {
                         print!(" ");
                     }
                     println!();
-                    for (dir, usage) in usage_by_dir.iter() {
+                    for (dir, utilization) in utilization_by_dir.iter() {
                         for field in fields.iter() {
                             match field {
                                 Directory => print!("{:dir_col_size$}", dir),
                                 BytesUsed => print!(
                                     "{:BYTES_USED_SIZE$}",
-                                    as_human_bytes(usage.bytes_used)
+                                    as_human_bytes(utilization.bytes_used)
                                 ),
                                 BytesAvailable => print!(
                                     "{:BYTES_AVAIL_SIZE$}",
-                                    as_human_bytes(usage.bytes_available)
+                                    as_human_bytes(utilization.bytes_available)
                                 ),
                                 DatasetQuota => print!(
                                     "{:QUOTA_SIZE$}",
-                                    as_human_bytes(usage.dataset_quota)
+                                    as_human_bytes(utilization.dataset_quota)
                                 ),
                                 PctAvailable => print!(
                                     "{:PCT_OF_AVAIL_SIZE$}",
                                     as_pct_str(
-                                        usage.bytes_used,
-                                        usage.bytes_available
+                                        utilization.bytes_used,
+                                        utilization.bytes_available
                                     )
                                 ),
                                 PctQuota => print!(
                                     "{:PCT_OF_QUOTA_SIZE$}",
                                     as_pct_str(
-                                        usage.bytes_used,
-                                        usage.dataset_quota
+                                        utilization.bytes_used,
+                                        utilization.dataset_quota
                                     )
                                 ),
                             }

sled-agent/src/bin/zone-bundle.rs

@@ -220,6 +220,7 @@ impl HardwareMonitor {
             switch_zone_bootstrap_address,
             sled_config.switch_zone_maghemite_links.clone(),
             storage_manager.resources().clone(),
+            storage_manager.zone_bundler().clone(),
         )
         .await?;