wip: give sled-agent the generation of the mark-as-failure

Author: lif

end-to-end-tests/src/instance_launch.rs

@@ -5,9 +5,9 @@ use anyhow::{ensure, Context as _, Result};
 use async_trait::async_trait;
 use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError};
 use oxide_client::types::{
-    ByteCount, DiskCreate, DiskSource, ExternalIpCreate, InstanceCpuCount,
-    InstanceCreate, InstanceDiskAttachment, InstanceNetworkInterfaceAttachment,
-    InstanceState, SshKeyCreate,
+    ByteCount, DiskCreate, DiskSource, ExternalIp, ExternalIpCreate,
+    InstanceCpuCount, InstanceCreate, InstanceDiskAttachment,
+    InstanceNetworkInterfaceAttachment, InstanceState, SshKeyCreate,
 };
 use oxide_client::{ClientDisksExt, ClientInstancesExt, ClientSessionExt};
 use russh::{ChannelMsg, Disconnect};
@@ -223,7 +223,8 @@ async fn instance_launch() -> Result<()> {
                     .most_recent(1024 * 1024)
                     .max_bytes(1024 * 1024)
                     .send()
-                    .await?
+                    .await
+                    .map_err(|e| Error::NotYet)?
                     .data,
             )
             .into_owned();

nexus/src/app/instance.rs

@@ -28,6 +28,7 @@ use nexus_db_queries::db::identity::Resource;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_types::external_api::views;
+use nexus_types::internal_api::views::HandleInstancePutResultResult;
 use omicron_common::address::PROPOLIS_PORT;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::ByteCount;
@@ -1040,7 +1041,7 @@ impl super::Nexus {
         opctx: &OpContext,
         instance_id: &Uuid,
         result: Result<nexus::SledInstanceState, Error>,
-    ) -> Result<views::HandleInstancePutResultResult, Error> {
+    ) -> Result<HandleInstancePutResultResult, Error> {
         let (.., authz_instance) = LookupPath::new(&opctx, &self.db_datastore)
             .instance_id(*instance_id)
             .lookup_for(authz::Action::Modify)
@@ -1053,10 +1054,16 @@ impl super::Nexus {
                     .await?;
                 if !inst_updated && !vmm_updated {
                     // the generation number bumped up by the timeout task.
-                    // TODO check actual state / put it in the TimedOut variant?
-                    Ok(views::HandleInstancePutResultResult::TimedOut)
+                    // TODO check actual state value (need more specificity than these two bools?)
+                    let state = self
+                        .db_datastore
+                        .instance_fetch_with_vmm(opctx, &authz_instance)
+                        .await?;
+                    Ok(HandleInstancePutResultResult::TimedOut {
+                        generation: *state.instance().runtime().gen,
+                    })
                 } else {
-                    Ok(views::HandleInstancePutResultResult::Ok)
+                    Ok(HandleInstancePutResultResult::Ok)
                 }
             }
             Err(error) => {

nexus/types/src/internal_api/views.rs

@@ -6,6 +6,7 @@ use chrono::DateTime;
 use chrono::Utc;
 use futures::future::ready;
 use futures::stream::StreamExt;
+use omicron_common::api::external::Generation;
 use omicron_common::api::external::ObjectStream;
 use schemars::JsonSchema;
 use serde::Serialize;
@@ -304,5 +305,5 @@ pub struct LastResultCompleted {
 #[serde(rename_all = "snake_case", tag = "last_result", content = "details")]
 pub enum HandleInstancePutResultResult {
     Ok,
-    TimedOut,
+    TimedOut { generation: Generation },
 }

openapi/nexus-internal.json

@@ -3826,6 +3826,17 @@
           {
             "type": "object",
             "properties": {
+              "details": {
+                "type": "object",
+                "properties": {
+                  "generation": {
+                    "$ref": "#/components/schemas/Generation"
+                  }
+                },
+                "required": [
+                  "generation"
+                ]
+              },
               "last_result": {
                 "type": "string",
                 "enum": [
@@ -3834,6 +3845,7 @@
               }
             },
             "required": [
+              "details",
               "last_result"
             ]
           }

sled-agent/src/instance_manager.rs

@@ -387,8 +387,12 @@ impl InstanceManager {
                                 .map(nexus_client::ResponseValue::into_inner)
                             {
                                 Ok(HandleInstancePutResultResult::Ok) => {}
-                                Ok(HandleInstancePutResultResult::TimedOut) => {
-                                    error!(log, "Nexus gave up waiting for us to finish creating the propolis zone and abandoned the instance. Rudely terminating it from our side."; "instance_id" => %instance_id);
+                                Ok(
+                                    HandleInstancePutResultResult::TimedOut {
+                                        generation,
+                                    },
+                                ) => {
+                                    error!(log, "Nexus gave up waiting for us to finish creating the propolis zone and abandoned the instance. Rudely terminating it from our side."; "instance_id" => %instance_id, "generation" => %generation);
                                     if let Err(err) = instance.terminate().await
                                     {
                                         error!(log, "Couldn't terminate instance whose creation was timed-out by Nexus"; "instance_id" => %instance_id, "err" => %err);
@@ -413,8 +417,12 @@ impl InstanceManager {
                                 .map(nexus_client::ResponseValue::into_inner)
                             {
                                 Ok(HandleInstancePutResultResult::Ok) => {}
-                                Ok(HandleInstancePutResultResult::TimedOut) => {
-                                    error!(log, "Nexus gave up waiting for us to finish creating the propolis zone and abandoned the instance, but the instance also explicitly failed on our side. Rudely terminating what remains of it."; "instance_id" => %instance_id);
+                                Ok(
+                                    HandleInstancePutResultResult::TimedOut {
+                                        generation,
+                                    },
+                                ) => {
+                                    error!(log, "Nexus gave up waiting for us to finish creating the propolis zone and abandoned the instance, but the instance also explicitly failed on our side. Rudely terminating what remains of it."; "instance_id" => %instance_id, "generation" => %generation);
                                     if let Err(err) = instance.terminate().await
                                     {
                                         error!(log, "Couldn't terminate faulted instance (whose creation was also timed-out by Nexus)"; "instance_id" => %instance_id, "err" => %err);