Plot a simulation of the congestion controller (#468)

* Move cubic congestion controller and hybrid slow start to s2n_quic_core

* Add a method to the Congestion Controller trait to get the current size of the congestion window

* Initial simulation plot

* Add snapshot test

* Remove unnecessary lifetime

* Add snapshot file

* add simulations workflow output

* fix realpath error

* Add num_traits, set packet size to MINIMUM_MTU, make threshold pub(super)

* Fix clippy and miri

* Add a test for a network that experiences loss at 5MB

* Have 3MB test use slow start

* Adjust comment

* Add title to tree output and remove index.html

Co-authored-by: Cameron Bytheway <[email protected]>

Author: WesleyRosenblum

.github/config/cargo-deny.toml

@@ -4,9 +4,7 @@ unmaintained = "deny"
 notice = "deny"
 yanked = "deny"
 ignore = [
-    "RUSTSEC-2019-0031", # ring currently uses 'spin' for feature detection
     "RUSTSEC-2020-0016", # mio 0.6 uses 'net2' (removed in 0.7) but tokio is still pinned to 0.6
-    "RUSTSEC-2020-0095", # insta (a dev dependency) uses `difference`
 ]
 
 [bans]
@@ -20,6 +18,7 @@ skip = [
 # compliance is a separate binary target so just ignore it
 skip-tree = [
     { name = "cargo-compliance" },
+    { name = "criterion" }, # criterion is always going to be just a test dependency
 ]
 
 [sources]

.github/workflows/ci.yml

@@ -436,3 +436,48 @@ jobs:
           name: 'perf / report (${{ matrix.script }})'
           status: 'success'
           url: "${{ steps.s3.outputs.URL }}"
+
+  recovery-simulations:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: actions-rs/toolchain@v1
+        id: toolchain
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+      - uses: camshaft/rust-cache@v1
+
+      - name: Run simulations
+        run: |
+          ./scripts/recovery-sim
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: recovery-simulations
+          path: target/recovery-sim
+
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+
+      - name: Upload to S3
+        id: s3
+        run: |
+          TARGET="${{ github.sha }}/recovery-simulations"
+          aws s3 sync target/recovery-sim "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
+          URL="$CDN/$TARGET/index.html"
+          echo "::set-output name=URL::$URL"
+
+      - uses: ouzi-dev/[email protected]
+        with:
+          name: 'recovery-simulations / report'
+          status: 'success'
+          url: "${{ steps.s3.outputs.URL }}"

quic/s2n-quic-core/Cargo.toml

@@ -18,6 +18,7 @@ byteorder = { version = "1.1", default-features = false }
 bytes = { version = "0.6", default-features = false }
 displaydoc = { version = "0.1", default-features = false }
 hex-literal = "0.3"
+num-traits = { version = "0.2", default-features = false, features = ["libm"] }
 s2n-codec = { version = "0.1", path = "../../common/s2n-codec", default-features = false }
 subtle = { version = "2", default-features = false }
 thiserror = { version = "1", optional = true }
@@ -28,6 +29,7 @@ bolero = "0.6"
 compliance = { path = "../../common/compliance/compliance" }
 criterion = "0.3"
 insta = "1.0"
+plotters = { version = "0.3", default-features = false, features = ["svg_backend", "line_series"] }
 s2n-codec = { version = "0.1", path = "../../common/s2n-codec", features = ["testing"] }
 
 [[test]]
@@ -66,3 +68,7 @@ harness = false
 name = "varint"
 path = "tests/varint/fuzz_target.rs"
 harness = false
+
+[[test]]
+name = "recovery-simulation"
+path = "tests/recovery/simulation.rs"

quic/s2n-quic-core/src/recovery/congestion_controller.rs

@@ -26,6 +26,9 @@ impl<'a> PathInfo<'a> {
 }
 
 pub trait CongestionController: 'static + Clone + Send + Debug {
+    /// Returns the size of the current congestion window in bytes
+    fn congestion_window(&self) -> u32;
+
     /// Returns `true` if the congestion window does not have sufficient
     /// space for a packet of `max_datagram_size` considering the current
     /// bytes in flight
@@ -80,6 +83,10 @@ pub mod testing {
     pub struct Unlimited {}
 
     impl CongestionController for Unlimited {
+        fn congestion_window(&self) -> u32 {
+            u32::max_value()
+        }
+
         fn is_congestion_limited(&self) -> bool {
             false
         }
@@ -125,6 +132,10 @@ pub mod testing {
     }
 
     impl CongestionController for MockCongestionController {
+        fn congestion_window(&self) -> u32 {
+            u32::max_value()
+        }
+
         fn is_congestion_limited(&self) -> bool {
             false
         }

quic/s2n-quic-transport/src/recovery/cubic.rs renamed to quic/s2n-quic-core/src/recovery/cubic.rs

@@ -1,13 +1,19 @@
-use crate::recovery::{
-    congestion_controller::CongestionController,
-    cubic::{FastRetransmission::*, State::*},
-    hybrid_slow_start::HybridSlowStart,
+use crate::{
+    counter::Counter,
+    recovery::{
+        congestion_controller::CongestionController,
+        cubic::{FastRetransmission::*, State::*},
+        hybrid_slow_start::HybridSlowStart,
+        RTTEstimator,
+    },
+    time::Timestamp,
 };
 use core::{
     cmp::{max, min},
     time::Duration,
 };
-use s2n_quic_core::{counter::Counter, recovery::RTTEstimator, time::Timestamp};
+#[cfg(not(feature = "std"))]
+use num_traits::Float as _;
 
 //= https://tools.ietf.org/id/draft-ietf-quic-recovery-32.txt#7.3
 //#                 New Path or      +------------+
@@ -81,6 +87,10 @@ pub struct CubicCongestionController {
 type BytesInFlight = Counter<u32>;
 
 impl CongestionController for CubicCongestionController {
+    fn congestion_window(&self) -> u32 {
+        self.congestion_window
+    }
+
     fn is_congestion_limited(&self) -> bool {
         let available_congestion_window =
             self.congestion_window.saturating_sub(*self.bytes_in_flight);
@@ -563,18 +573,12 @@ impl Cubic {
 
 #[cfg(test)]
 mod test {
-    use crate::recovery::{
-        cubic::{
-            BytesInFlight, Cubic, CubicCongestionController,
-            FastRetransmission::{Idle, RequiresTransmission},
-            State::{CongestionAvoidance, Recovery, SlowStart},
-            BETA_CUBIC,
-        },
-        CongestionController,
-    };
-    use s2n_quic_core::{
-        packet::number::PacketNumberSpace, recovery::RTTEstimator, time::Duration,
+    use super::*;
+    use crate::{
+        packet::number::PacketNumberSpace,
+        time::{Clock, NoopClock},
     };
+    use core::time::Duration;
 
     macro_rules! assert_delta {
         ($x:expr, $y:expr, $d:expr) => {
@@ -739,7 +743,7 @@ mod test {
     fn on_packet_sent() {
         let mut cc = CubicCongestionController::new(1000);
         let mut rtt_estimator = RTTEstimator::new(Duration::from_millis(0));
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.congestion_window = 100_000;
 
@@ -793,7 +797,7 @@ mod test {
     #[test]
     fn on_packet_sent_application_limited() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.congestion_window = 100_000;
         cc.bytes_in_flight = BytesInFlight::new(96_500);
@@ -833,7 +837,7 @@ mod test {
     #[test]
     fn on_packet_sent_fast_retransmission() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.congestion_window = 100_000;
         cc.bytes_in_flight = BytesInFlight::new(99900);
@@ -853,7 +857,7 @@ mod test {
     #[test]
     fn congestion_avoidance_after_idle_period() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         let rtt_estimator = &RTTEstimator::new(Duration::from_secs(0));
 
         cc.congestion_window = 3000;
@@ -890,7 +894,7 @@ mod test {
     fn congestion_avoidance_after_fast_convergence() {
         let max_datagram_size = 1200;
         let mut cc = CubicCongestionController::new(max_datagram_size);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         cc.bytes_in_flight = BytesInFlight::new(100);
         cc.congestion_window = 80_000;
         cc.cubic.w_last_max = bytes_to_packets(100_000, max_datagram_size);
@@ -927,7 +931,7 @@ mod test {
     #[test]
     fn on_packet_lost() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         cc.congestion_window = 100_000;
         cc.bytes_in_flight = BytesInFlight::new(100_000);
         cc.state = SlowStart;
@@ -957,7 +961,7 @@ mod test {
     #[test]
     fn on_packet_lost_below_minimum_window() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         cc.congestion_window = cc.minimum_window();
         cc.bytes_in_flight = BytesInFlight::new(cc.minimum_window());
         cc.state = CongestionAvoidance(now);
@@ -970,7 +974,7 @@ mod test {
     #[test]
     fn on_packet_lost_already_in_recovery() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         cc.congestion_window = 10000;
         cc.bytes_in_flight = BytesInFlight::new(1000);
         cc.state = Recovery(now, Idle);
@@ -990,7 +994,7 @@ mod test {
     #[test]
     fn on_packet_lost_persistent_congestion() {
         let mut cc = CubicCongestionController::new(1000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         cc.congestion_window = 10000;
         cc.bytes_in_flight = BytesInFlight::new(1000);
         cc.state = Recovery(now, Idle);
@@ -1059,7 +1063,7 @@ mod test {
     #[test]
     fn on_packet_ack_limited() {
         let mut cc = CubicCongestionController::new(5000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
         cc.congestion_window = 100_000;
         cc.bytes_in_flight = BytesInFlight::new(10000);
         cc.state = SlowStart;
@@ -1079,7 +1083,7 @@ mod test {
     #[compliance::tests("https://tools.ietf.org/id/draft-ietf-quic-recovery-32.txt#7.3.2")]
     fn on_packet_ack_recovery_to_congestion_avoidance() {
         let mut cc = CubicCongestionController::new(5000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.cubic.w_max = bytes_to_packets(25000, 5000);
         cc.state = Recovery(now, Idle);
@@ -1102,7 +1106,7 @@ mod test {
     #[compliance::tests("https://tools.ietf.org/id/draft-ietf-quic-recovery-32.txt#7.3.2")]
     fn on_packet_ack_slow_start_to_congestion_avoidance() {
         let mut cc = CubicCongestionController::new(5000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.state = SlowStart;
         cc.congestion_window = 10000;
@@ -1128,7 +1132,7 @@ mod test {
     #[test]
     fn on_packet_ack_recovery() {
         let mut cc = CubicCongestionController::new(5000);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.state = Recovery(now, Idle);
         cc.congestion_window = 10000;
@@ -1151,7 +1155,7 @@ mod test {
         let max_datagram_size = 5000;
         let mut cc = CubicCongestionController::new(max_datagram_size);
         let mut cc2 = CubicCongestionController::new(max_datagram_size);
-        let now = s2n_quic_platform::time::now();
+        let now = NoopClock.get_time();
 
         cc.state = CongestionAvoidance(now + Duration::from_millis(3300));
         cc.congestion_window = 10000;

quic/s2n-quic-transport/src/recovery/hybrid_slow_start.rs renamed to quic/s2n-quic-core/src/recovery/hybrid_slow_start.rs

@@ -1,5 +1,5 @@
+use crate::time::Timestamp;
 use core::time::Duration;
-use s2n_quic_core::time::Timestamp;
 
 /// An implementation of the Hybrid Slow Start algorithm described in
 /// "Hybrid Slow Start for High-Bandwidth and Long-Distance Networks"
@@ -32,7 +32,7 @@ const THRESHOLD_DIVIDEND: u32 = 8;
 impl HybridSlowStart {
     /// Constructs a new `HybridSlowStart`. `max_datagram_size` is used for determining
     /// the minimum slow start threshold.
-    pub(super) fn new(max_datagram_size: u16) -> Self {
+    pub fn new(max_datagram_size: u16) -> Self {
         Self {
             sample_count: 0,
             last_min_rtt: None,
@@ -51,7 +51,7 @@ impl HybridSlowStart {
     /// a number of samples has increased since the last
     /// round of samples and if so will set the slow start
     /// threshold.
-    pub(super) fn on_rtt_update(
+    pub fn on_rtt_update(
         &mut self,
         congestion_window: u32,
         time_sent: Timestamp,
@@ -107,7 +107,7 @@ impl HybridSlowStart {
     /// slow start threshold to the minimum of the Hybrid Slow Start threshold
     /// and the given congestion window. This will ensure we exit slow start
     /// early enough to avoid further congestion.
-    pub(super) fn on_congestion_event(&mut self, ssthresh: u32) {
+    pub fn on_congestion_event(&mut self, ssthresh: u32) {
         self.threshold = self.threshold.min(ssthresh).max(self.low_ssthresh());
     }
 
@@ -118,7 +118,10 @@ impl HybridSlowStart {
 
 #[cfg(test)]
 mod test {
-    use crate::recovery::hybrid_slow_start::HybridSlowStart;
+    use crate::{
+        recovery::hybrid_slow_start::HybridSlowStart,
+        time::{Clock, NoopClock},
+    };
     use core::time::Duration;
 
     #[test]
@@ -149,7 +152,7 @@ mod test {
     #[test]
     fn on_rtt_update_above_threshold() {
         let mut slow_start = HybridSlowStart::new(10);
-        let time_zero = s2n_quic_platform::time::now();
+        let time_zero = NoopClock.get_time();
         slow_start.threshold = 500;
 
         assert_eq!(slow_start.sample_count, 0);
@@ -165,7 +168,7 @@ mod test {
 
         assert_eq!(slow_start.sample_count, 0);
 
-        let time_zero = s2n_quic_platform::time::now() + Duration::from_secs(10);
+        let time_zero = NoopClock.get_time() + Duration::from_secs(10);
 
         // -- Round 1 --
 

quic/s2n-quic-core/src/recovery/mod.rs

@@ -1,5 +1,8 @@
-pub mod congestion_controller;
-mod rtt_estimator;
-
 pub use congestion_controller::CongestionController;
+pub use cubic::CubicCongestionController;
 pub use rtt_estimator::*;
+
+pub mod congestion_controller;
+pub mod cubic;
+mod hybrid_slow_start;
+mod rtt_estimator;