Coverage Report

Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/trueno/src/brick/circuit.rs
Line
Count
Source
1
//! Circuit Breaker Pattern
2
//!
3
//! AWP-02: Protect against cascading failures with three-state circuit breaker.
4
5
use std::time::{Duration, Instant};
6
7
// ----------------------------------------------------------------------------
8
// AWP-02: Circuit Breaker
9
// ----------------------------------------------------------------------------
10
11
/// Circuit breaker states.
12
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13
pub enum CircuitState {
14
    /// Circuit is closed (normal operation)
15
    Closed,
16
    /// Circuit is open (failing fast)
17
    Open,
18
    /// Circuit is half-open (testing recovery)
19
    HalfOpen,
20
}
21
22
/// Circuit breaker for protecting against cascading failures.
23
///
24
/// # Example
25
/// ```rust
26
/// use trueno::brick::CircuitBreaker;
27
/// use std::time::Duration;
28
///
29
/// let mut breaker = CircuitBreaker::new(3, Duration::from_secs(30));
30
///
31
/// // Record failures
32
/// breaker.record_failure();
33
/// breaker.record_failure();
34
/// assert!(breaker.allow_request()); // Still closed
35
///
36
/// breaker.record_failure();
37
/// assert!(!breaker.allow_request()); // Now open
38
/// ```
39
pub struct CircuitBreaker {
40
    /// Current state
41
    state: CircuitState,
42
    /// Failure count in current window
43
    failure_count: usize,
44
    /// Failure threshold to trip the circuit
45
    failure_threshold: usize,
46
    /// Time when circuit opened
47
    opened_at: Option<Instant>,
48
    /// Duration to stay open before trying half-open
49
    open_duration: Duration,
50
    /// Success count in half-open state
51
    half_open_successes: usize,
52
    /// Successes needed to close from half-open
53
    half_open_threshold: usize,
54
}
55
56
impl CircuitBreaker {
57
    /// Create a new circuit breaker.
58
0
    pub fn new(failure_threshold: usize, open_duration: Duration) -> Self {
59
0
        Self {
60
0
            state: CircuitState::Closed,
61
0
            failure_count: 0,
62
0
            failure_threshold,
63
0
            opened_at: None,
64
0
            open_duration,
65
0
            half_open_successes: 0,
66
0
            half_open_threshold: 1,
67
0
        }
68
0
    }
69
70
    /// Get current state.
71
    #[must_use]
72
0
    pub fn state(&self) -> CircuitState {
73
0
        self.state
74
0
    }
75
76
    /// Check if a request should be allowed.
77
    #[must_use]
78
0
    pub fn allow_request(&mut self) -> bool {
79
0
        match self.state {
80
0
            CircuitState::Closed => true,
81
            CircuitState::Open => {
82
                // Check if we should transition to half-open
83
0
                if let Some(opened_at) = self.opened_at {
84
0
                    if opened_at.elapsed() >= self.open_duration {
85
0
                        self.state = CircuitState::HalfOpen;
86
0
                        self.half_open_successes = 0;
87
0
                        return true; // Allow one request to test
88
0
                    }
89
0
                }
90
0
                false
91
            }
92
0
            CircuitState::HalfOpen => true, // Allow requests in half-open
93
        }
94
0
    }
95
96
    /// Record a successful operation.
97
0
    pub fn record_success(&mut self) {
98
0
        match self.state {
99
0
            CircuitState::Closed => {
100
0
                // Reset failure count on success
101
0
                self.failure_count = 0;
102
0
            }
103
            CircuitState::HalfOpen => {
104
0
                self.half_open_successes += 1;
105
0
                if self.half_open_successes >= self.half_open_threshold {
106
0
                    // Recovered - close the circuit
107
0
                    self.state = CircuitState::Closed;
108
0
                    self.failure_count = 0;
109
0
                    self.opened_at = None;
110
0
                }
111
            }
112
0
            CircuitState::Open => {}
113
        }
114
0
    }
115
116
    /// Record a failed operation.
117
0
    pub fn record_failure(&mut self) {
118
0
        match self.state {
119
            CircuitState::Closed => {
120
0
                self.failure_count += 1;
121
0
                if self.failure_count >= self.failure_threshold {
122
0
                    // Trip the circuit
123
0
                    self.state = CircuitState::Open;
124
0
                    self.opened_at = Some(Instant::now());
125
0
                }
126
            }
127
0
            CircuitState::HalfOpen => {
128
0
                // Failed during recovery - reopen
129
0
                self.state = CircuitState::Open;
130
0
                self.opened_at = Some(Instant::now());
131
0
            }
132
0
            CircuitState::Open => {}
133
        }
134
0
    }
135
136
    /// Reset the circuit breaker to closed state.
137
0
    pub fn reset(&mut self) {
138
0
        self.state = CircuitState::Closed;
139
0
        self.failure_count = 0;
140
0
        self.opened_at = None;
141
0
        self.half_open_successes = 0;
142
0
    }
143
}
144
145
impl Default for CircuitBreaker {
146
0
    fn default() -> Self {
147
0
        Self::new(5, Duration::from_secs(30))
148
0
    }
149
}
150
151
#[cfg(test)]
152
mod tests {
153
    use super::*;
154
155
    #[test]
156
    fn test_circuit_state_eq() {
157
        assert_eq!(CircuitState::Closed, CircuitState::Closed);
158
        assert_eq!(CircuitState::Open, CircuitState::Open);
159
        assert_eq!(CircuitState::HalfOpen, CircuitState::HalfOpen);
160
        assert_ne!(CircuitState::Closed, CircuitState::Open);
161
    }
162
163
    #[test]
164
    fn test_circuit_breaker_new() {
165
        let cb = CircuitBreaker::new(5, Duration::from_secs(30));
166
        assert_eq!(cb.state(), CircuitState::Closed);
167
        assert_eq!(cb.failure_count, 0);
168
        assert_eq!(cb.failure_threshold, 5);
169
    }
170
171
    #[test]
172
    fn test_circuit_breaker_default() {
173
        let cb = CircuitBreaker::default();
174
        assert_eq!(cb.state(), CircuitState::Closed);
175
        assert_eq!(cb.failure_threshold, 5);
176
        assert_eq!(cb.open_duration, Duration::from_secs(30));
177
    }
178
179
    #[test]
180
    fn test_circuit_breaker_closed_allows_requests() {
181
        let mut cb = CircuitBreaker::new(3, Duration::from_secs(30));
182
        assert!(cb.allow_request());
183
        assert!(cb.allow_request());
184
        assert!(cb.allow_request());
185
    }
186
187
    #[test]
188
    fn test_circuit_breaker_trips_on_failures() {
189
        let mut cb = CircuitBreaker::new(3, Duration::from_secs(30));
190
191
        cb.record_failure();
192
        assert_eq!(cb.state(), CircuitState::Closed);
193
194
        cb.record_failure();
195
        assert_eq!(cb.state(), CircuitState::Closed);
196
197
        cb.record_failure();
198
        assert_eq!(cb.state(), CircuitState::Open);
199
    }
200
201
    #[test]
202
    fn test_circuit_breaker_open_blocks_requests() {
203
        let mut cb = CircuitBreaker::new(2, Duration::from_secs(30));
204
205
        cb.record_failure();
206
        cb.record_failure();
207
        assert_eq!(cb.state(), CircuitState::Open);
208
209
        assert!(!cb.allow_request());
210
    }
211
212
    #[test]
213
    fn test_circuit_breaker_success_resets_failures() {
214
        let mut cb = CircuitBreaker::new(3, Duration::from_secs(30));
215
216
        cb.record_failure();
217
        cb.record_failure();
218
        assert_eq!(cb.failure_count, 2);
219
220
        cb.record_success();
221
        assert_eq!(cb.failure_count, 0);
222
    }
223
224
    #[test]
225
    fn test_circuit_breaker_half_open_transition() {
226
        let mut cb = CircuitBreaker::new(2, Duration::from_millis(10));
227
228
        // Trip the circuit
229
        cb.record_failure();
230
        cb.record_failure();
231
        assert_eq!(cb.state(), CircuitState::Open);
232
233
        // Wait for open duration
234
        std::thread::sleep(Duration::from_millis(15));
235
236
        // Should transition to half-open on allow_request
237
        assert!(cb.allow_request());
238
        assert_eq!(cb.state(), CircuitState::HalfOpen);
239
    }
240
241
    #[test]
242
    fn test_circuit_breaker_half_open_success_closes() {
243
        let mut cb = CircuitBreaker::new(2, Duration::from_millis(10));
244
245
        // Trip and wait for half-open
246
        cb.record_failure();
247
        cb.record_failure();
248
        std::thread::sleep(Duration::from_millis(15));
249
        let _ = cb.allow_request(); // Transition to half-open
250
251
        assert_eq!(cb.state(), CircuitState::HalfOpen);
252
253
        // Success should close the circuit
254
        cb.record_success();
255
        assert_eq!(cb.state(), CircuitState::Closed);
256
    }
257
258
    #[test]
259
    fn test_circuit_breaker_half_open_failure_reopens() {
260
        let mut cb = CircuitBreaker::new(2, Duration::from_millis(10));
261
262
        // Trip and wait for half-open
263
        cb.record_failure();
264
        cb.record_failure();
265
        std::thread::sleep(Duration::from_millis(15));
266
        let _ = cb.allow_request(); // Transition to half-open
267
268
        assert_eq!(cb.state(), CircuitState::HalfOpen);
269
270
        // Failure should reopen
271
        cb.record_failure();
272
        assert_eq!(cb.state(), CircuitState::Open);
273
    }
274
275
    #[test]
276
    fn test_circuit_breaker_reset() {
277
        let mut cb = CircuitBreaker::new(2, Duration::from_secs(30));
278
279
        cb.record_failure();
280
        cb.record_failure();
281
        assert_eq!(cb.state(), CircuitState::Open);
282
283
        cb.reset();
284
        assert_eq!(cb.state(), CircuitState::Closed);
285
        assert_eq!(cb.failure_count, 0);
286
    }
287
288
    /// FALSIFICATION TEST: Circuit MUST trip at exact threshold
289
    ///
290
    /// The circuit breaker must trip at exactly failure_threshold failures,
291
    /// not before and not after. Off-by-one errors are common.
292
    #[test]
293
    fn test_falsify_exact_threshold_trip() {
294
        for threshold in 1..=10 {
295
            let mut cb = CircuitBreaker::new(threshold, Duration::from_secs(30));
296
297
            // Record threshold-1 failures - should NOT trip
298
            for i in 0..(threshold - 1) {
299
                cb.record_failure();
300
                assert_eq!(
301
                    cb.state(),
302
                    CircuitState::Closed,
303
                    "FALSIFICATION FAILED: Circuit tripped after {} failures (threshold={})",
304
                    i + 1,
305
                    threshold
306
                );
307
            }
308
309
            // The threshold-th failure MUST trip the circuit
310
            cb.record_failure();
311
            assert_eq!(
312
                cb.state(),
313
                CircuitState::Open,
314
                "FALSIFICATION FAILED: Circuit did NOT trip at exact threshold={}",
315
                threshold
316
            );
317
        }
318
    }
319
320
    /// FALSIFICATION TEST: Open circuit MUST block all requests
321
    ///
322
    /// While open (and before timeout), the circuit MUST block 100% of requests.
323
    /// Any leak would cause cascading failures.
324
    #[test]
325
    fn test_falsify_open_blocks_all() {
326
        let mut cb = CircuitBreaker::new(1, Duration::from_secs(30));
327
328
        // Trip immediately
329
        cb.record_failure();
330
        assert_eq!(cb.state(), CircuitState::Open);
331
332
        // Try many requests - ALL must be blocked
333
        let mut allowed = 0;
334
        for _ in 0..1000 {
335
            if cb.allow_request() {
336
                allowed += 1;
337
            }
338
        }
339
340
        assert_eq!(
341
            allowed, 0,
342
            "FALSIFICATION FAILED: {} requests leaked through open circuit",
343
            allowed
344
        );
345
    }
346
347
    /// FALSIFICATION TEST: Half-open failure MUST immediately reopen
348
    ///
349
    /// A single failure in half-open state must immediately reopen the circuit.
350
    /// If it doesn't, the system could repeatedly hammer a failing service.
351
    #[test]
352
    fn test_falsify_half_open_single_failure_reopens() {
353
        let mut cb = CircuitBreaker::new(1, Duration::from_millis(5));
354
355
        // Trip and wait for half-open
356
        cb.record_failure();
357
        std::thread::sleep(Duration::from_millis(10));
358
        let _ = cb.allow_request(); // Transition to half-open
359
360
        assert_eq!(cb.state(), CircuitState::HalfOpen);
361
362
        // Single failure MUST reopen
363
        cb.record_failure();
364
365
        assert_eq!(
366
            cb.state(),
367
            CircuitState::Open,
368
            "FALSIFICATION FAILED: Half-open did not reopen after failure"
369
        );
370
371
        // Verify opened_at was updated (circuit should wait again)
372
        assert!(
373
            cb.opened_at.is_some(),
374
            "FALSIFICATION FAILED: opened_at not set after reopen"
375
        );
376
    }
377
378
    /// FALSIFICATION TEST: State machine must be deterministic
379
    ///
380
    /// Same sequence of events must always produce same state.
381
    /// Non-determinism would make the system unpredictable.
382
    #[test]
383
    fn test_falsify_state_machine_determinism() {
384
        // Run the same sequence multiple times
385
        for _ in 0..10 {
386
            let mut cb = CircuitBreaker::new(3, Duration::from_millis(5));
387
388
            // Sequence: F, S, F, F, F, wait, S
389
            cb.record_failure();
390
            assert_eq!(cb.state(), CircuitState::Closed);
391
392
            cb.record_success();
393
            assert_eq!(cb.failure_count, 0); // Reset
394
395
            cb.record_failure();
396
            cb.record_failure();
397
            cb.record_failure();
398
            assert_eq!(cb.state(), CircuitState::Open);
399
400
            std::thread::sleep(Duration::from_millis(10));
401
            let _ = cb.allow_request();
402
            assert_eq!(cb.state(), CircuitState::HalfOpen);
403
404
            cb.record_success();
405
            assert_eq!(cb.state(), CircuitState::Closed);
406
        }
407
    }
408
409
    /// FALSIFICATION TEST: Failures in Open state are no-ops
410
    ///
411
    /// Recording failures while Open should not affect anything.
412
    /// This prevents "double-counting" failures.
413
    #[test]
414
    fn test_falsify_open_ignores_failures() {
415
        let mut cb = CircuitBreaker::new(2, Duration::from_secs(30));
416
417
        // Trip the circuit
418
        cb.record_failure();
419
        cb.record_failure();
420
        let opened_at = cb.opened_at;
421
422
        // Record more failures while open
423
        for _ in 0..100 {
424
            cb.record_failure();
425
        }
426
427
        // State should still be Open, opened_at unchanged
428
        assert_eq!(cb.state(), CircuitState::Open);
429
        assert_eq!(
430
            cb.opened_at, opened_at,
431
            "FALSIFICATION FAILED: opened_at changed while in Open state"
432
        );
433
    }
434
}