Skip to content

Commit 83269cf

Browse files
author
lif
committed
wip: further post-rebase fixes
1 parent 3c8a354 commit 83269cf

File tree

1 file changed

+157
-38
lines changed

1 file changed

+157
-38
lines changed

‎sled-agent/src/instance.rs

Lines changed: 157 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ use illumos_utils::link::VnicAllocator;
3030
use illumos_utils::opte::{DhcpCfg, PortManager};
3131
use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory};
3232
use illumos_utils::svc::wait_for_service;
33-
use illumos_utils::zone::Zones;
3433
use illumos_utils::zone::PROPOLIS_ZONE_PREFIX;
3534
use nexus_client::types::HandleInstancePutResultResult;
3635
use omicron_common::address::NEXUS_INTERNAL_PORT;
@@ -53,6 +52,11 @@ use std::sync::Arc;
5352
use tokio::sync::{mpsc, oneshot};
5453
use uuid::Uuid;
5554

55+
#[cfg(test)]
56+
use illumos_utils::zone::MockZones as Zones;
57+
#[cfg(not(test))]
58+
use illumos_utils::zone::Zones;
59+
5660
// The depth of the request queue for the instance.
5761
const QUEUE_SIZE: usize = 32;
5862

@@ -1233,7 +1237,8 @@ impl InstanceRunner {
12331237
state: crate::params::InstanceStateRequested,
12341238
) -> Result<SledInstanceState, Error> {
12351239
use propolis_client::types::InstanceStateRequested as PropolisRequest;
1236-
let (propolis_state, next_published, propolis_ensure_res) = match state {
1240+
let (propolis_state, next_published, propolis_ensure_res) = match state
1241+
{
12371242
InstanceStateRequested::MigrationTarget(migration_params) => {
12381243
let res = self.propolis_ensure(Some(migration_params)).await;
12391244
(None, None, Some(res))
@@ -1278,14 +1283,16 @@ impl InstanceRunner {
12781283
}
12791284
if let Some(res) = propolis_ensure_res {
12801285
match res {
1281-
Ok(()) => match self.nexus_client
1286+
Ok(()) => match self
1287+
.nexus_client
12821288
.client()
12831289
.cpapi_handle_instance_put_success(
12841290
self.id(),
12851291
&self.current_state().await.into(),
12861292
)
12871293
.await?
1288-
.into_inner() {
1294+
.into_inner()
1295+
{
12891296
HandleInstancePutResultResult::Ok => {}
12901297
HandleInstancePutResultResult::TimedOut { generation } => {
12911298
error!(
@@ -1296,15 +1303,17 @@ impl InstanceRunner {
12961303
);
12971304
self.terminate().await;
12981305
}
1299-
}
1300-
Err(instance_put_error) => match self.nexus_client
1306+
},
1307+
Err(instance_put_error) => match self
1308+
.nexus_client
13011309
.client()
13021310
.cpapi_handle_instance_put_failure(
13031311
self.id(),
13041312
&instance_put_error.to_string(),
13051313
)
13061314
.await?
1307-
.into_inner() {
1315+
.into_inner()
1316+
{
13081317
HandleInstancePutResultResult::Ok => {}
13091318
HandleInstancePutResultResult::TimedOut { generation } => {
13101319
error!(
@@ -1315,7 +1324,7 @@ impl InstanceRunner {
13151324
);
13161325
self.terminate().await;
13171326
}
1318-
}
1327+
},
13191328
}
13201329
}
13211330
Ok(self.state.sled_instance_state())
@@ -1642,11 +1651,11 @@ mod tests {
16421651
"couldn't send SledInstanceState to test driver",
16431652
)
16441653
})?;
1645-
self.nexus_disposition
1646-
.borrow()
1647-
.as_ref()
1648-
.cloned()
1649-
.ok_or(omicron_common::api::external::Error::unavail("did not set nexus_disposition in FakeNexusServer"))
1654+
self.nexus_disposition.borrow().as_ref().cloned().ok_or(
1655+
omicron_common::api::external::Error::unavail(
1656+
"did not set nexus_disposition in FakeNexusServer",
1657+
),
1658+
)
16501659
}
16511660

16521661
fn cpapi_handle_instance_put_failure(
@@ -1664,11 +1673,11 @@ mod tests {
16641673
"couldn't send error to test driver",
16651674
)
16661675
})?;
1667-
self.nexus_disposition
1668-
.borrow()
1669-
.as_ref()
1670-
.cloned()
1671-
.ok_or(omicron_common::api::external::Error::unavail("did not set nexus_disposition in FakeNexusServer"))
1676+
self.nexus_disposition.borrow().as_ref().cloned().ok_or(
1677+
omicron_common::api::external::Error::unavail(
1678+
"did not set nexus_disposition in FakeNexusServer",
1679+
),
1680+
)
16721681
}
16731682
}
16741683

@@ -1961,10 +1970,15 @@ mod tests {
19611970
let _mock_vnic_contexts = mock_vnic_contexts();
19621971
let _mock_zone_contexts = mock_zone_contexts();
19631972

1964-
let FakeNexusParts { nexus_client, nexus_server, mut state_rx, disp_tx } =
1965-
FakeNexusParts::new(&logctx);
1973+
let FakeNexusParts {
1974+
nexus_client,
1975+
nexus_server,
1976+
mut state_rx,
1977+
disp_tx,
1978+
} = FakeNexusParts::new(&logctx);
19661979

1967-
disp_tx.send(Some(HandleInstancePutResultResult::Ok))
1980+
disp_tx
1981+
.send(Some(HandleInstancePutResultResult::Ok))
19681982
.expect("couldn't send nexus_disposition");
19691983

19701984
let (_dns_server, resolver, _dns_config_dir) =
@@ -2031,7 +2045,8 @@ mod tests {
20312045
let FakeNexusParts { nexus_client, nexus_server, state_rx, disp_tx } =
20322046
FakeNexusParts::new(&logctx);
20332047

2034-
disp_tx.send(Some(HandleInstancePutResultResult::Ok))
2048+
disp_tx
2049+
.send(Some(HandleInstancePutResultResult::Ok))
20352050
.expect("couldn't send nexus_disposition");
20362051

20372052
let (_dns_server, resolver, _dns_config_dir) =
@@ -2094,18 +2109,15 @@ mod tests {
20942109
Ok(())
20952110
});
20962111
let wait_ctx = illumos_utils::svc::wait_for_service_context();
2097-
wait_ctx.expect().times(..).returning(|_, _, _| Ok(()));
2112+
wait_ctx.expect().times(1..).returning(|_, _, _| Ok(()));
20982113
let zone_id_ctx = MockZones::id_context();
2099-
zone_id_ctx.expect().times(..).returning(|_| Ok(Some(1)));
2100-
let halt_rm_ctx = MockZones::halt_and_remove_logged_context();
2101-
halt_rm_ctx.expect().times(..).returning(|_, _| Ok(()));
2114+
zone_id_ctx.expect().times(1..).returning(|_| Ok(Some(1)));
21022115

21032116
let FakeNexusParts { nexus_client, nexus_server, state_rx, disp_tx } =
21042117
FakeNexusParts::new(&logctx);
21052118

2106-
disp_tx.send(Some(HandleInstancePutResultResult::TimedOut {
2107-
generation: Generation(5)
2108-
}))
2119+
disp_tx
2120+
.send(Some(HandleInstancePutResultResult::Ok))
21092121
.expect("couldn't send nexus_disposition");
21102122

21112123
let (_dns_server, resolver, _dns_config_dir) =
@@ -2152,16 +2164,112 @@ mod tests {
21522164
logctx.cleanup_successful();
21532165
}
21542166

2167+
// tests that we terminate the instance if nexus tells us it timed us out.
21552168
#[tokio::test]
2156-
async fn test_instance_manager_creation_nexus_timeout() {
2169+
async fn test_instance_create_timeout_from_nexus() {
21572170
let logctx = omicron_test_utils::dev::test_setup_log(
2158-
"test_instance_manager_creation_nexus_timeout",
2171+
"test_instance_create_timeout_from_nexus",
21592172
);
21602173

2174+
let (propolis_server, _propolis_client) =
2175+
propolis_mock_server(&logctx.log);
2176+
let propolis_addr = propolis_server.local_addr();
2177+
2178+
// automock'd things used during this test
2179+
let _mock_vnic_contexts = mock_vnic_contexts();
2180+
let _mock_zone_contexts = mock_zone_contexts();
2181+
let halt_rm_ctx = MockZones::halt_and_remove_logged_context();
2182+
halt_rm_ctx.expect().times(1..).returning(|_, _| Ok(()));
2183+
2184+
let FakeNexusParts {
2185+
nexus_client,
2186+
nexus_server,
2187+
mut state_rx,
2188+
disp_tx,
2189+
} = FakeNexusParts::new(&logctx);
2190+
2191+
disp_tx
2192+
.send(Some(HandleInstancePutResultResult::TimedOut {
2193+
generation: Generation::from(5),
2194+
}))
2195+
.expect("couldn't send nexus_disposition");
2196+
2197+
let (_dns_server, resolver, _dns_config_dir) =
2198+
timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server))
2199+
.await
2200+
.expect("timed out making DNS server and Resolver");
2201+
2202+
let nexus_client_with_resolver =
2203+
NexusClientWithResolver::new_with_client(nexus_client, resolver);
2204+
21612205
let storage_handle = fake_storage_manager_with_u2().await;
21622206

2163-
let FakeNexusParts { nexus_client, nexus_server, state_rx, disp_tx } =
2164-
FakeNexusParts::new(&logctx);
2207+
let inst = timeout(
2208+
TIMEOUT_DURATION,
2209+
instance_struct(
2210+
&logctx,
2211+
propolis_addr,
2212+
nexus_client_with_resolver,
2213+
storage_handle,
2214+
),
2215+
)
2216+
.await
2217+
.expect("timed out creating Instance struct");
2218+
2219+
let (put_tx, put_rx) = oneshot::channel();
2220+
2221+
inst.put_state(put_tx, InstanceStateRequested::Running)
2222+
.await
2223+
.expect("failed to send Instance::put_state");
2224+
2225+
timeout(TIMEOUT_DURATION, put_rx)
2226+
.await
2227+
.expect("timed out waiting for Instance::put_state result")
2228+
.expect("failed to receive Instance::put_state result")
2229+
.expect("Instance::put_state failed");
2230+
2231+
timeout(
2232+
TIMEOUT_DURATION,
2233+
state_rx.wait_for(|maybe_state| match maybe_state {
2234+
ReceivedInstanceState::InstancePut(sled_inst_state) => {
2235+
sled_inst_state.vmm_state.state == InstanceState::Destroyed
2236+
}
2237+
_ => false,
2238+
}),
2239+
)
2240+
.await
2241+
.expect("timed out waiting for InstanceState::Destroyed in FakeNexus")
2242+
.expect("failed to receive FakeNexus' InstanceState");
2243+
2244+
logctx.cleanup_successful();
2245+
}
2246+
2247+
#[tokio::test]
2248+
async fn test_instance_manager_creation_timeout_from_nexus() {
2249+
let logctx = omicron_test_utils::dev::test_setup_log(
2250+
"test_instance_manager_creation_timeout_from_nexus",
2251+
);
2252+
2253+
// automock'd things used during this test
2254+
let _mock_vnic_contexts = mock_vnic_contexts();
2255+
let _mock_zone_contexts = mock_zone_contexts();
2256+
let halt_rm_ctx = MockZones::halt_and_remove_logged_context();
2257+
halt_rm_ctx.expect().times(1..).returning(|_, _| Ok(()));
2258+
2259+
let storage_handle = fake_storage_manager_with_u2().await;
2260+
2261+
let FakeNexusParts {
2262+
nexus_client,
2263+
nexus_server,
2264+
mut state_rx,
2265+
disp_tx,
2266+
} = FakeNexusParts::new(&logctx);
2267+
2268+
disp_tx
2269+
.send(Some(HandleInstancePutResultResult::TimedOut {
2270+
generation: Generation::from(5),
2271+
}))
2272+
.expect("couldn't send nexus_disposition");
21652273

21662274
let (_dns_server, resolver, _dns_config_dir) =
21672275
timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server))
@@ -2201,10 +2309,6 @@ mod tests {
22012309
propolis_mock_server(&logctx.log);
22022310
let propolis_addr = propolis_server.local_addr();
22032311

2204-
// automock'd things used during this test
2205-
let _mock_vnic_contexts = mock_vnic_contexts();
2206-
let _mock_zone_contexts = mock_zone_contexts();
2207-
22082312
let instance_id = Uuid::new_v4();
22092313
let propolis_id = Uuid::new_v4();
22102314
let InstanceInitialState {
@@ -2225,7 +2329,22 @@ mod tests {
22252329
.await
22262330
.unwrap();
22272331

2228-
todo!();
2332+
mgr.ensure_state(instance_id, InstanceStateRequested::Running)
2333+
.await
2334+
.unwrap();
2335+
2336+
timeout(
2337+
TIMEOUT_DURATION,
2338+
state_rx.wait_for(|maybe_state| match maybe_state {
2339+
ReceivedInstanceState::InstancePut(sled_inst_state) => {
2340+
sled_inst_state.vmm_state.state == InstanceState::Destroyed
2341+
}
2342+
_ => false,
2343+
}),
2344+
)
2345+
.await
2346+
.expect("timed out waiting for InstanceState::Destroyed in FakeNexus")
2347+
.expect("failed to receive FakeNexus' InstanceState");
22292348

22302349
logctx.cleanup_successful();
22312350
}

0 commit comments

Comments
 (0)