/home/noah/src/trueno/src/brick/circuit.rs
Line | Count | Source |
1 | | //! Circuit Breaker Pattern |
2 | | //! |
3 | | //! AWP-02: Protect against cascading failures with three-state circuit breaker. |
4 | | |
5 | | use std::time::{Duration, Instant}; |
6 | | |
7 | | // ---------------------------------------------------------------------------- |
8 | | // AWP-02: Circuit Breaker |
9 | | // ---------------------------------------------------------------------------- |
10 | | |
11 | | /// Circuit breaker states. |
12 | | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
13 | | pub enum CircuitState { |
14 | | /// Circuit is closed (normal operation) |
15 | | Closed, |
16 | | /// Circuit is open (failing fast) |
17 | | Open, |
18 | | /// Circuit is half-open (testing recovery) |
19 | | HalfOpen, |
20 | | } |
21 | | |
22 | | /// Circuit breaker for protecting against cascading failures. |
23 | | /// |
24 | | /// # Example |
25 | | /// ```rust |
26 | | /// use trueno::brick::CircuitBreaker; |
27 | | /// use std::time::Duration; |
28 | | /// |
29 | | /// let mut breaker = CircuitBreaker::new(3, Duration::from_secs(30)); |
30 | | /// |
31 | | /// // Record failures |
32 | | /// breaker.record_failure(); |
33 | | /// breaker.record_failure(); |
34 | | /// assert!(breaker.allow_request()); // Still closed |
35 | | /// |
36 | | /// breaker.record_failure(); |
37 | | /// assert!(!breaker.allow_request()); // Now open |
38 | | /// ``` |
39 | | pub struct CircuitBreaker { |
40 | | /// Current state |
41 | | state: CircuitState, |
42 | | /// Failure count in current window |
43 | | failure_count: usize, |
44 | | /// Failure threshold to trip the circuit |
45 | | failure_threshold: usize, |
46 | | /// Time when circuit opened |
47 | | opened_at: Option<Instant>, |
48 | | /// Duration to stay open before trying half-open |
49 | | open_duration: Duration, |
50 | | /// Success count in half-open state |
51 | | half_open_successes: usize, |
52 | | /// Successes needed to close from half-open |
53 | | half_open_threshold: usize, |
54 | | } |
55 | | |
56 | | impl CircuitBreaker { |
57 | | /// Create a new circuit breaker. |
58 | 0 | pub fn new(failure_threshold: usize, open_duration: Duration) -> Self { |
59 | 0 | Self { |
60 | 0 | state: CircuitState::Closed, |
61 | 0 | failure_count: 0, |
62 | 0 | failure_threshold, |
63 | 0 | opened_at: None, |
64 | 0 | open_duration, |
65 | 0 | half_open_successes: 0, |
66 | 0 | half_open_threshold: 1, |
67 | 0 | } |
68 | 0 | } |
69 | | |
70 | | /// Get current state. |
71 | | #[must_use] |
72 | 0 | pub fn state(&self) -> CircuitState { |
73 | 0 | self.state |
74 | 0 | } |
75 | | |
76 | | /// Check if a request should be allowed. |
77 | | #[must_use] |
78 | 0 | pub fn allow_request(&mut self) -> bool { |
79 | 0 | match self.state { |
80 | 0 | CircuitState::Closed => true, |
81 | | CircuitState::Open => { |
82 | | // Check if we should transition to half-open |
83 | 0 | if let Some(opened_at) = self.opened_at { |
84 | 0 | if opened_at.elapsed() >= self.open_duration { |
85 | 0 | self.state = CircuitState::HalfOpen; |
86 | 0 | self.half_open_successes = 0; |
87 | 0 | return true; // Allow one request to test |
88 | 0 | } |
89 | 0 | } |
90 | 0 | false |
91 | | } |
92 | 0 | CircuitState::HalfOpen => true, // Allow requests in half-open |
93 | | } |
94 | 0 | } |
95 | | |
96 | | /// Record a successful operation. |
97 | 0 | pub fn record_success(&mut self) { |
98 | 0 | match self.state { |
99 | 0 | CircuitState::Closed => { |
100 | 0 | // Reset failure count on success |
101 | 0 | self.failure_count = 0; |
102 | 0 | } |
103 | | CircuitState::HalfOpen => { |
104 | 0 | self.half_open_successes += 1; |
105 | 0 | if self.half_open_successes >= self.half_open_threshold { |
106 | 0 | // Recovered - close the circuit |
107 | 0 | self.state = CircuitState::Closed; |
108 | 0 | self.failure_count = 0; |
109 | 0 | self.opened_at = None; |
110 | 0 | } |
111 | | } |
112 | 0 | CircuitState::Open => {} |
113 | | } |
114 | 0 | } |
115 | | |
116 | | /// Record a failed operation. |
117 | 0 | pub fn record_failure(&mut self) { |
118 | 0 | match self.state { |
119 | | CircuitState::Closed => { |
120 | 0 | self.failure_count += 1; |
121 | 0 | if self.failure_count >= self.failure_threshold { |
122 | 0 | // Trip the circuit |
123 | 0 | self.state = CircuitState::Open; |
124 | 0 | self.opened_at = Some(Instant::now()); |
125 | 0 | } |
126 | | } |
127 | 0 | CircuitState::HalfOpen => { |
128 | 0 | // Failed during recovery - reopen |
129 | 0 | self.state = CircuitState::Open; |
130 | 0 | self.opened_at = Some(Instant::now()); |
131 | 0 | } |
132 | 0 | CircuitState::Open => {} |
133 | | } |
134 | 0 | } |
135 | | |
136 | | /// Reset the circuit breaker to closed state. |
137 | 0 | pub fn reset(&mut self) { |
138 | 0 | self.state = CircuitState::Closed; |
139 | 0 | self.failure_count = 0; |
140 | 0 | self.opened_at = None; |
141 | 0 | self.half_open_successes = 0; |
142 | 0 | } |
143 | | } |
144 | | |
145 | | impl Default for CircuitBreaker { |
146 | 0 | fn default() -> Self { |
147 | 0 | Self::new(5, Duration::from_secs(30)) |
148 | 0 | } |
149 | | } |
150 | | |
151 | | #[cfg(test)] |
152 | | mod tests { |
153 | | use super::*; |
154 | | |
155 | | #[test] |
156 | | fn test_circuit_state_eq() { |
157 | | assert_eq!(CircuitState::Closed, CircuitState::Closed); |
158 | | assert_eq!(CircuitState::Open, CircuitState::Open); |
159 | | assert_eq!(CircuitState::HalfOpen, CircuitState::HalfOpen); |
160 | | assert_ne!(CircuitState::Closed, CircuitState::Open); |
161 | | } |
162 | | |
163 | | #[test] |
164 | | fn test_circuit_breaker_new() { |
165 | | let cb = CircuitBreaker::new(5, Duration::from_secs(30)); |
166 | | assert_eq!(cb.state(), CircuitState::Closed); |
167 | | assert_eq!(cb.failure_count, 0); |
168 | | assert_eq!(cb.failure_threshold, 5); |
169 | | } |
170 | | |
171 | | #[test] |
172 | | fn test_circuit_breaker_default() { |
173 | | let cb = CircuitBreaker::default(); |
174 | | assert_eq!(cb.state(), CircuitState::Closed); |
175 | | assert_eq!(cb.failure_threshold, 5); |
176 | | assert_eq!(cb.open_duration, Duration::from_secs(30)); |
177 | | } |
178 | | |
179 | | #[test] |
180 | | fn test_circuit_breaker_closed_allows_requests() { |
181 | | let mut cb = CircuitBreaker::new(3, Duration::from_secs(30)); |
182 | | assert!(cb.allow_request()); |
183 | | assert!(cb.allow_request()); |
184 | | assert!(cb.allow_request()); |
185 | | } |
186 | | |
187 | | #[test] |
188 | | fn test_circuit_breaker_trips_on_failures() { |
189 | | let mut cb = CircuitBreaker::new(3, Duration::from_secs(30)); |
190 | | |
191 | | cb.record_failure(); |
192 | | assert_eq!(cb.state(), CircuitState::Closed); |
193 | | |
194 | | cb.record_failure(); |
195 | | assert_eq!(cb.state(), CircuitState::Closed); |
196 | | |
197 | | cb.record_failure(); |
198 | | assert_eq!(cb.state(), CircuitState::Open); |
199 | | } |
200 | | |
201 | | #[test] |
202 | | fn test_circuit_breaker_open_blocks_requests() { |
203 | | let mut cb = CircuitBreaker::new(2, Duration::from_secs(30)); |
204 | | |
205 | | cb.record_failure(); |
206 | | cb.record_failure(); |
207 | | assert_eq!(cb.state(), CircuitState::Open); |
208 | | |
209 | | assert!(!cb.allow_request()); |
210 | | } |
211 | | |
212 | | #[test] |
213 | | fn test_circuit_breaker_success_resets_failures() { |
214 | | let mut cb = CircuitBreaker::new(3, Duration::from_secs(30)); |
215 | | |
216 | | cb.record_failure(); |
217 | | cb.record_failure(); |
218 | | assert_eq!(cb.failure_count, 2); |
219 | | |
220 | | cb.record_success(); |
221 | | assert_eq!(cb.failure_count, 0); |
222 | | } |
223 | | |
224 | | #[test] |
225 | | fn test_circuit_breaker_half_open_transition() { |
226 | | let mut cb = CircuitBreaker::new(2, Duration::from_millis(10)); |
227 | | |
228 | | // Trip the circuit |
229 | | cb.record_failure(); |
230 | | cb.record_failure(); |
231 | | assert_eq!(cb.state(), CircuitState::Open); |
232 | | |
233 | | // Wait for open duration |
234 | | std::thread::sleep(Duration::from_millis(15)); |
235 | | |
236 | | // Should transition to half-open on allow_request |
237 | | assert!(cb.allow_request()); |
238 | | assert_eq!(cb.state(), CircuitState::HalfOpen); |
239 | | } |
240 | | |
241 | | #[test] |
242 | | fn test_circuit_breaker_half_open_success_closes() { |
243 | | let mut cb = CircuitBreaker::new(2, Duration::from_millis(10)); |
244 | | |
245 | | // Trip and wait for half-open |
246 | | cb.record_failure(); |
247 | | cb.record_failure(); |
248 | | std::thread::sleep(Duration::from_millis(15)); |
249 | | let _ = cb.allow_request(); // Transition to half-open |
250 | | |
251 | | assert_eq!(cb.state(), CircuitState::HalfOpen); |
252 | | |
253 | | // Success should close the circuit |
254 | | cb.record_success(); |
255 | | assert_eq!(cb.state(), CircuitState::Closed); |
256 | | } |
257 | | |
258 | | #[test] |
259 | | fn test_circuit_breaker_half_open_failure_reopens() { |
260 | | let mut cb = CircuitBreaker::new(2, Duration::from_millis(10)); |
261 | | |
262 | | // Trip and wait for half-open |
263 | | cb.record_failure(); |
264 | | cb.record_failure(); |
265 | | std::thread::sleep(Duration::from_millis(15)); |
266 | | let _ = cb.allow_request(); // Transition to half-open |
267 | | |
268 | | assert_eq!(cb.state(), CircuitState::HalfOpen); |
269 | | |
270 | | // Failure should reopen |
271 | | cb.record_failure(); |
272 | | assert_eq!(cb.state(), CircuitState::Open); |
273 | | } |
274 | | |
275 | | #[test] |
276 | | fn test_circuit_breaker_reset() { |
277 | | let mut cb = CircuitBreaker::new(2, Duration::from_secs(30)); |
278 | | |
279 | | cb.record_failure(); |
280 | | cb.record_failure(); |
281 | | assert_eq!(cb.state(), CircuitState::Open); |
282 | | |
283 | | cb.reset(); |
284 | | assert_eq!(cb.state(), CircuitState::Closed); |
285 | | assert_eq!(cb.failure_count, 0); |
286 | | } |
287 | | |
288 | | /// FALSIFICATION TEST: Circuit MUST trip at exact threshold |
289 | | /// |
290 | | /// The circuit breaker must trip at exactly failure_threshold failures, |
291 | | /// not before and not after. Off-by-one errors are common. |
292 | | #[test] |
293 | | fn test_falsify_exact_threshold_trip() { |
294 | | for threshold in 1..=10 { |
295 | | let mut cb = CircuitBreaker::new(threshold, Duration::from_secs(30)); |
296 | | |
297 | | // Record threshold-1 failures - should NOT trip |
298 | | for i in 0..(threshold - 1) { |
299 | | cb.record_failure(); |
300 | | assert_eq!( |
301 | | cb.state(), |
302 | | CircuitState::Closed, |
303 | | "FALSIFICATION FAILED: Circuit tripped after {} failures (threshold={})", |
304 | | i + 1, |
305 | | threshold |
306 | | ); |
307 | | } |
308 | | |
309 | | // The threshold-th failure MUST trip the circuit |
310 | | cb.record_failure(); |
311 | | assert_eq!( |
312 | | cb.state(), |
313 | | CircuitState::Open, |
314 | | "FALSIFICATION FAILED: Circuit did NOT trip at exact threshold={}", |
315 | | threshold |
316 | | ); |
317 | | } |
318 | | } |
319 | | |
320 | | /// FALSIFICATION TEST: Open circuit MUST block all requests |
321 | | /// |
322 | | /// While open (and before timeout), the circuit MUST block 100% of requests. |
323 | | /// Any leak would cause cascading failures. |
324 | | #[test] |
325 | | fn test_falsify_open_blocks_all() { |
326 | | let mut cb = CircuitBreaker::new(1, Duration::from_secs(30)); |
327 | | |
328 | | // Trip immediately |
329 | | cb.record_failure(); |
330 | | assert_eq!(cb.state(), CircuitState::Open); |
331 | | |
332 | | // Try many requests - ALL must be blocked |
333 | | let mut allowed = 0; |
334 | | for _ in 0..1000 { |
335 | | if cb.allow_request() { |
336 | | allowed += 1; |
337 | | } |
338 | | } |
339 | | |
340 | | assert_eq!( |
341 | | allowed, 0, |
342 | | "FALSIFICATION FAILED: {} requests leaked through open circuit", |
343 | | allowed |
344 | | ); |
345 | | } |
346 | | |
347 | | /// FALSIFICATION TEST: Half-open failure MUST immediately reopen |
348 | | /// |
349 | | /// A single failure in half-open state must immediately reopen the circuit. |
350 | | /// If it doesn't, the system could repeatedly hammer a failing service. |
351 | | #[test] |
352 | | fn test_falsify_half_open_single_failure_reopens() { |
353 | | let mut cb = CircuitBreaker::new(1, Duration::from_millis(5)); |
354 | | |
355 | | // Trip and wait for half-open |
356 | | cb.record_failure(); |
357 | | std::thread::sleep(Duration::from_millis(10)); |
358 | | let _ = cb.allow_request(); // Transition to half-open |
359 | | |
360 | | assert_eq!(cb.state(), CircuitState::HalfOpen); |
361 | | |
362 | | // Single failure MUST reopen |
363 | | cb.record_failure(); |
364 | | |
365 | | assert_eq!( |
366 | | cb.state(), |
367 | | CircuitState::Open, |
368 | | "FALSIFICATION FAILED: Half-open did not reopen after failure" |
369 | | ); |
370 | | |
371 | | // Verify opened_at was updated (circuit should wait again) |
372 | | assert!( |
373 | | cb.opened_at.is_some(), |
374 | | "FALSIFICATION FAILED: opened_at not set after reopen" |
375 | | ); |
376 | | } |
377 | | |
378 | | /// FALSIFICATION TEST: State machine must be deterministic |
379 | | /// |
380 | | /// Same sequence of events must always produce same state. |
381 | | /// Non-determinism would make the system unpredictable. |
382 | | #[test] |
383 | | fn test_falsify_state_machine_determinism() { |
384 | | // Run the same sequence multiple times |
385 | | for _ in 0..10 { |
386 | | let mut cb = CircuitBreaker::new(3, Duration::from_millis(5)); |
387 | | |
388 | | // Sequence: F, S, F, F, F, wait, S |
389 | | cb.record_failure(); |
390 | | assert_eq!(cb.state(), CircuitState::Closed); |
391 | | |
392 | | cb.record_success(); |
393 | | assert_eq!(cb.failure_count, 0); // Reset |
394 | | |
395 | | cb.record_failure(); |
396 | | cb.record_failure(); |
397 | | cb.record_failure(); |
398 | | assert_eq!(cb.state(), CircuitState::Open); |
399 | | |
400 | | std::thread::sleep(Duration::from_millis(10)); |
401 | | let _ = cb.allow_request(); |
402 | | assert_eq!(cb.state(), CircuitState::HalfOpen); |
403 | | |
404 | | cb.record_success(); |
405 | | assert_eq!(cb.state(), CircuitState::Closed); |
406 | | } |
407 | | } |
408 | | |
409 | | /// FALSIFICATION TEST: Failures in Open state are no-ops |
410 | | /// |
411 | | /// Recording failures while Open should not affect anything. |
412 | | /// This prevents "double-counting" failures. |
413 | | #[test] |
414 | | fn test_falsify_open_ignores_failures() { |
415 | | let mut cb = CircuitBreaker::new(2, Duration::from_secs(30)); |
416 | | |
417 | | // Trip the circuit |
418 | | cb.record_failure(); |
419 | | cb.record_failure(); |
420 | | let opened_at = cb.opened_at; |
421 | | |
422 | | // Record more failures while open |
423 | | for _ in 0..100 { |
424 | | cb.record_failure(); |
425 | | } |
426 | | |
427 | | // State should still be Open, opened_at unchanged |
428 | | assert_eq!(cb.state(), CircuitState::Open); |
429 | | assert_eq!( |
430 | | cb.opened_at, opened_at, |
431 | | "FALSIFICATION FAILED: opened_at changed while in Open state" |
432 | | ); |
433 | | } |
434 | | } |