## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/trueno/src/brick/circuit.rs

    Line
    Count
    Source
    1
    //! Circuit Breaker Pattern
    2
    //!
    3
    //! AWP-02: Protect against cascading failures with three-state circuit breaker.
    4
    5
    use std::time::{Duration, Instant};
    6
    7
    // ----------------------------------------------------------------------------
    8
    // AWP-02: Circuit Breaker
    9
    // ----------------------------------------------------------------------------
    10
    11
    /// Circuit breaker states.
    12
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    13
    pub enum CircuitState {
    14
        /// Circuit is closed (normal operation)
    15
        Closed,
    16
        /// Circuit is open (failing fast)
    17
        Open,
    18
        /// Circuit is half-open (testing recovery)
    19
        HalfOpen,
    20
    }
    21
    22
    /// Circuit breaker for protecting against cascading failures.
    23
    ///
    24
    /// # Example
    25
    /// ```rust
    26
    /// use trueno::brick::CircuitBreaker;
    27
    /// use std::time::Duration;
    28
    ///
    29
    /// let mut breaker = CircuitBreaker::new(3, Duration::from_secs(30));
    30
    ///
    31
    /// // Record failures
    32
    /// breaker.record_failure();
    33
    /// breaker.record_failure();
    34
    /// assert!(breaker.allow_request()); // Still closed
    35
    ///
    36
    /// breaker.record_failure();
    37
    /// assert!(!breaker.allow_request()); // Now open
    38
    /// ```
    39
    pub struct CircuitBreaker {
    40
        /// Current state
    41
        state: CircuitState,
    42
        /// Failure count in current window
    43
        failure_count: usize,
    44
        /// Failure threshold to trip the circuit
    45
        failure_threshold: usize,
    46
        /// Time when circuit opened
    47
        opened_at: Option<Instant>,
    48
        /// Duration to stay open before trying half-open
    49
        open_duration: Duration,
    50
        /// Success count in half-open state
    51
        half_open_successes: usize,
    52
        /// Successes needed to close from half-open
    53
        half_open_threshold: usize,
    54
    }
    55
    56
    impl CircuitBreaker {
    57
        /// Create a new circuit breaker.
    58
    0
        pub fn new(failure_threshold: usize, open_duration: Duration) -> Self {
    59
    0
            Self {
    60
    0
                state: CircuitState::Closed,
    61
    0
                failure_count: 0,
    62
    0
                failure_threshold,
    63
    0
                opened_at: None,
    64
    0
                open_duration,
    65
    0
                half_open_successes: 0,
    66
    0
                half_open_threshold: 1,
    67
    0
            }
    68
    0
        }
    69
    70
        /// Get current state.
    71
        #[must_use]
    72
    0
        pub fn state(&self) -> CircuitState {
    73
    0
            self.state
    74
    0
        }
    75
    76
        /// Check if a request should be allowed.
    77
        #[must_use]
    78
    0
        pub fn allow_request(&mut self) -> bool {
    79
    0
            match self.state {
    80
    0
                CircuitState::Closed => true,
    81
                CircuitState::Open => {
    82
                    // Check if we should transition to half-open
    83
    0
                    if let Some(opened_at) = self.opened_at {
    84
    0
                        if opened_at.elapsed() >= self.open_duration {
    85
    0
                            self.state = CircuitState::HalfOpen;
    86
    0
                            self.half_open_successes = 0;
    87
    0
                            return true; // Allow one request to test
    88
    0
                        }
    89
    0
                    }
    90
    0
                    false
    91
                }
    92
    0
                CircuitState::HalfOpen => true, // Allow requests in half-open
    93
            }
    94
    0
        }
    95
    96
        /// Record a successful operation.
    97
    0
        pub fn record_success(&mut self) {
    98
    0
            match self.state {
    99
    0
                CircuitState::Closed => {
    100
    0
                    // Reset failure count on success
    101
    0
                    self.failure_count = 0;
    102
    0
                }
    103
                CircuitState::HalfOpen => {
    104
    0
                    self.half_open_successes += 1;
    105
    0
                    if self.half_open_successes >= self.half_open_threshold {
    106
    0
                        // Recovered - close the circuit
    107
    0
                        self.state = CircuitState::Closed;
    108
    0
                        self.failure_count = 0;
    109
    0
                        self.opened_at = None;
    110
    0
                    }
    111
                }
    112
    0
                CircuitState::Open => {}
    113
            }
    114
    0
        }
    115
    116
        /// Record a failed operation.
    117
    0
        pub fn record_failure(&mut self) {
    118
    0
            match self.state {
    119
                CircuitState::Closed => {
    120
    0
                    self.failure_count += 1;
    121
    0
                    if self.failure_count >= self.failure_threshold {
    122
    0
                        // Trip the circuit
    123
    0
                        self.state = CircuitState::Open;
    124
    0
                        self.opened_at = Some(Instant::now());
    125
    0
                    }
    126
                }
    127
    0
                CircuitState::HalfOpen => {
    128
    0
                    // Failed during recovery - reopen
    129
    0
                    self.state = CircuitState::Open;
    130
    0
                    self.opened_at = Some(Instant::now());
    131
    0
                }
    132
    0
                CircuitState::Open => {}
    133
            }
    134
    0
        }
    135
    136
        /// Reset the circuit breaker to closed state.
    137
    0
        pub fn reset(&mut self) {
    138
    0
            self.state = CircuitState::Closed;
    139
    0
            self.failure_count = 0;
    140
    0
            self.opened_at = None;
    141
    0
            self.half_open_successes = 0;
    142
    0
        }
    143
    }
    144
    145
    impl Default for CircuitBreaker {
    146
    0
        fn default() -> Self {
    147
    0
            Self::new(5, Duration::from_secs(30))
    148
    0
        }
    149
    }
    150
    151
    #[cfg(test)]
    152
    mod tests {
    153
        use super::*;
    154
    155
        #[test]
    156
        fn test_circuit_state_eq() {
    157
            assert_eq!(CircuitState::Closed, CircuitState::Closed);
    158
            assert_eq!(CircuitState::Open, CircuitState::Open);
    159
            assert_eq!(CircuitState::HalfOpen, CircuitState::HalfOpen);
    160
            assert_ne!(CircuitState::Closed, CircuitState::Open);
    161
        }
    162
    163
        #[test]
    164
        fn test_circuit_breaker_new() {
    165
            let cb = CircuitBreaker::new(5, Duration::from_secs(30));
    166
            assert_eq!(cb.state(), CircuitState::Closed);
    167
            assert_eq!(cb.failure_count, 0);
    168
            assert_eq!(cb.failure_threshold, 5);
    169
        }
    170
    171
        #[test]
    172
        fn test_circuit_breaker_default() {
    173
            let cb = CircuitBreaker::default();
    174
            assert_eq!(cb.state(), CircuitState::Closed);
    175
            assert_eq!(cb.failure_threshold, 5);
    176
            assert_eq!(cb.open_duration, Duration::from_secs(30));
    177
        }
    178
    179
        #[test]
    180
        fn test_circuit_breaker_closed_allows_requests() {
    181
            let mut cb = CircuitBreaker::new(3, Duration::from_secs(30));
    182
            assert!(cb.allow_request());
    183
            assert!(cb.allow_request());
    184
            assert!(cb.allow_request());
    185
        }
    186
    187
        #[test]
    188
        fn test_circuit_breaker_trips_on_failures() {
    189
            let mut cb = CircuitBreaker::new(3, Duration::from_secs(30));
    190
    191
            cb.record_failure();
    192
            assert_eq!(cb.state(), CircuitState::Closed);
    193
    194
            cb.record_failure();
    195
            assert_eq!(cb.state(), CircuitState::Closed);
    196
    197
            cb.record_failure();
    198
            assert_eq!(cb.state(), CircuitState::Open);
    199
        }
    200
    201
        #[test]
    202
        fn test_circuit_breaker_open_blocks_requests() {
    203
            let mut cb = CircuitBreaker::new(2, Duration::from_secs(30));
    204
    205
            cb.record_failure();
    206
            cb.record_failure();
    207
            assert_eq!(cb.state(), CircuitState::Open);
    208
    209
            assert!(!cb.allow_request());
    210
        }
    211
    212
        #[test]
    213
        fn test_circuit_breaker_success_resets_failures() {
    214
            let mut cb = CircuitBreaker::new(3, Duration::from_secs(30));
    215
    216
            cb.record_failure();
    217
            cb.record_failure();
    218
            assert_eq!(cb.failure_count, 2);
    219
    220
            cb.record_success();
    221
            assert_eq!(cb.failure_count, 0);
    222
        }
    223
    224
        #[test]
    225
        fn test_circuit_breaker_half_open_transition() {
    226
            let mut cb = CircuitBreaker::new(2, Duration::from_millis(10));
    227
    228
            // Trip the circuit
    229
            cb.record_failure();
    230
            cb.record_failure();
    231
            assert_eq!(cb.state(), CircuitState::Open);
    232
    233
            // Wait for open duration
    234
            std::thread::sleep(Duration::from_millis(15));
    235
    236
            // Should transition to half-open on allow_request
    237
            assert!(cb.allow_request());
    238
            assert_eq!(cb.state(), CircuitState::HalfOpen);
    239
        }
    240
    241
        #[test]
    242
        fn test_circuit_breaker_half_open_success_closes() {
    243
            let mut cb = CircuitBreaker::new(2, Duration::from_millis(10));
    244
    245
            // Trip and wait for half-open
    246
            cb.record_failure();
    247
            cb.record_failure();
    248
            std::thread::sleep(Duration::from_millis(15));
    249
            let _ = cb.allow_request(); // Transition to half-open
    250
    251
            assert_eq!(cb.state(), CircuitState::HalfOpen);
    252
    253
            // Success should close the circuit
    254
            cb.record_success();
    255
            assert_eq!(cb.state(), CircuitState::Closed);
    256
        }
    257
    258
        #[test]
    259
        fn test_circuit_breaker_half_open_failure_reopens() {
    260
            let mut cb = CircuitBreaker::new(2, Duration::from_millis(10));
    261
    262
            // Trip and wait for half-open
    263
            cb.record_failure();
    264
            cb.record_failure();
    265
            std::thread::sleep(Duration::from_millis(15));
    266
            let _ = cb.allow_request(); // Transition to half-open
    267
    268
            assert_eq!(cb.state(), CircuitState::HalfOpen);
    269
    270
            // Failure should reopen
    271
            cb.record_failure();
    272
            assert_eq!(cb.state(), CircuitState::Open);
    273
        }
    274
    275
        #[test]
    276
        fn test_circuit_breaker_reset() {
    277
            let mut cb = CircuitBreaker::new(2, Duration::from_secs(30));
    278
    279
            cb.record_failure();
    280
            cb.record_failure();
    281
            assert_eq!(cb.state(), CircuitState::Open);
    282
    283
            cb.reset();
    284
            assert_eq!(cb.state(), CircuitState::Closed);
    285
            assert_eq!(cb.failure_count, 0);
    286
        }
    287
    288
        /// FALSIFICATION TEST: Circuit MUST trip at exact threshold
    289
        ///
    290
        /// The circuit breaker must trip at exactly failure_threshold failures,
    291
        /// not before and not after. Off-by-one errors are common.
    292
        #[test]
    293
        fn test_falsify_exact_threshold_trip() {
    294
            for threshold in 1..=10 {
    295
                let mut cb = CircuitBreaker::new(threshold, Duration::from_secs(30));
    296
    297
                // Record threshold-1 failures - should NOT trip
    298
                for i in 0..(threshold - 1) {
    299
                    cb.record_failure();
    300
                    assert_eq!(
    301
                        cb.state(),
    302
                        CircuitState::Closed,
    303
                        "FALSIFICATION FAILED: Circuit tripped after {} failures (threshold={})",
    304
                        i + 1,
    305
                        threshold
    306
                    );
    307
                }
    308
    309
                // The threshold-th failure MUST trip the circuit
    310
                cb.record_failure();
    311
                assert_eq!(
    312
                    cb.state(),
    313
                    CircuitState::Open,
    314
                    "FALSIFICATION FAILED: Circuit did NOT trip at exact threshold={}",
    315
                    threshold
    316
                );
    317
            }
    318
        }
    319
    320
        /// FALSIFICATION TEST: Open circuit MUST block all requests
    321
        ///
    322
        /// While open (and before timeout), the circuit MUST block 100% of requests.
    323
        /// Any leak would cause cascading failures.
    324
        #[test]
    325
        fn test_falsify_open_blocks_all() {
    326
            let mut cb = CircuitBreaker::new(1, Duration::from_secs(30));
    327
    328
            // Trip immediately
    329
            cb.record_failure();
    330
            assert_eq!(cb.state(), CircuitState::Open);
    331
    332
            // Try many requests - ALL must be blocked
    333
            let mut allowed = 0;
    334
            for _ in 0..1000 {
    335
                if cb.allow_request() {
    336
                    allowed += 1;
    337
                }
    338
            }
    339
    340
            assert_eq!(
    341
                allowed, 0,
    342
                "FALSIFICATION FAILED: {} requests leaked through open circuit",
    343
                allowed
    344
            );
    345
        }
    346
    347
        /// FALSIFICATION TEST: Half-open failure MUST immediately reopen
    348
        ///
    349
        /// A single failure in half-open state must immediately reopen the circuit.
    350
        /// If it doesn't, the system could repeatedly hammer a failing service.
    351
        #[test]
    352
        fn test_falsify_half_open_single_failure_reopens() {
    353
            let mut cb = CircuitBreaker::new(1, Duration::from_millis(5));
    354
    355
            // Trip and wait for half-open
    356
            cb.record_failure();
    357
            std::thread::sleep(Duration::from_millis(10));
    358
            let _ = cb.allow_request(); // Transition to half-open
    359
    360
            assert_eq!(cb.state(), CircuitState::HalfOpen);
    361
    362
            // Single failure MUST reopen
    363
            cb.record_failure();
    364
    365
            assert_eq!(
    366
                cb.state(),
    367
                CircuitState::Open,
    368
                "FALSIFICATION FAILED: Half-open did not reopen after failure"
    369
            );
    370
    371
            // Verify opened_at was updated (circuit should wait again)
    372
            assert!(
    373
                cb.opened_at.is_some(),
    374
                "FALSIFICATION FAILED: opened_at not set after reopen"
    375
            );
    376
        }
    377
    378
        /// FALSIFICATION TEST: State machine must be deterministic
    379
        ///
    380
        /// Same sequence of events must always produce same state.
    381
        /// Non-determinism would make the system unpredictable.
    382
        #[test]
    383
        fn test_falsify_state_machine_determinism() {
    384
            // Run the same sequence multiple times
    385
            for _ in 0..10 {
    386
                let mut cb = CircuitBreaker::new(3, Duration::from_millis(5));
    387
    388
                // Sequence: F, S, F, F, F, wait, S
    389
                cb.record_failure();
    390
                assert_eq!(cb.state(), CircuitState::Closed);
    391
    392
                cb.record_success();
    393
                assert_eq!(cb.failure_count, 0); // Reset
    394
    395
                cb.record_failure();
    396
                cb.record_failure();
    397
                cb.record_failure();
    398
                assert_eq!(cb.state(), CircuitState::Open);
    399
    400
                std::thread::sleep(Duration::from_millis(10));
    401
                let _ = cb.allow_request();
    402
                assert_eq!(cb.state(), CircuitState::HalfOpen);
    403
    404
                cb.record_success();
    405
                assert_eq!(cb.state(), CircuitState::Closed);
    406
            }
    407
        }
    408
    409
        /// FALSIFICATION TEST: Failures in Open state are no-ops
    410
        ///
    411
        /// Recording failures while Open should not affect anything.
    412
        /// This prevents "double-counting" failures.
    413
        #[test]
    414
        fn test_falsify_open_ignores_failures() {
    415
            let mut cb = CircuitBreaker::new(2, Duration::from_secs(30));
    416
    417
            // Trip the circuit
    418
            cb.record_failure();
    419
            cb.record_failure();
    420
            let opened_at = cb.opened_at;
    421
    422
            // Record more failures while open
    423
            for _ in 0..100 {
    424
                cb.record_failure();
    425
            }
    426
    427
            // State should still be Open, opened_at unchanged
    428
            assert_eq!(cb.state(), CircuitState::Open);
    429
            assert_eq!(
    430
                cb.opened_at, opened_at,
    431
                "FALSIFICATION FAILED: opened_at changed while in Open state"
    432
            );
    433
        }
    434
    }