/home/noah/src/trueno/src/brick/shutdown.rs
Line | Count | Source |
1 | | //! Graceful Shutdown Coordination |
2 | | //! |
3 | | //! AWP-07: Two-phase shutdown with active operation tracking. |
4 | | |
5 | | use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; |
6 | | use std::time::{Duration, Instant}; |
7 | | |
8 | | // ---------------------------------------------------------------------------- |
9 | | // AWP-07: Graceful Shutdown |
10 | | // ---------------------------------------------------------------------------- |
11 | | |
12 | | /// Result of a graceful shutdown operation. |
13 | | #[derive(Debug, Clone, PartialEq, Eq)] |
14 | | pub enum ShutdownResult { |
15 | | /// All operations completed cleanly. |
16 | | Clean, |
17 | | /// Timeout reached with operations still active. |
18 | | Timeout { |
19 | | /// Number of operations still active. |
20 | | remaining: usize, |
21 | | }, |
22 | | } |
23 | | |
24 | | /// Graceful shutdown coordinator. |
25 | | /// |
26 | | /// # Example |
27 | | /// ```rust |
28 | | /// use trueno::brick::GracefulShutdown; |
29 | | /// use std::time::Duration; |
30 | | /// |
31 | | /// let shutdown = GracefulShutdown::new(Duration::from_secs(5)); |
32 | | /// |
33 | | /// // Register an operation |
34 | | /// let guard = shutdown.register().ok_or("shutdown in progress")?; |
35 | | /// |
36 | | /// // ... do work ... |
37 | | /// |
38 | | /// drop(guard); // Operation complete |
39 | | /// |
40 | | /// // Initiate shutdown |
41 | | /// let result = shutdown.shutdown(); |
42 | | /// assert_eq!(result, trueno::brick::ShutdownResult::Clean); |
43 | | /// # Ok::<(), &'static str>(()) |
44 | | /// ``` |
45 | | pub struct GracefulShutdown { |
46 | | /// Flag indicating shutdown has been requested. |
47 | | shutdown_requested: AtomicBool, |
48 | | /// Number of active operations. |
49 | | active_count: AtomicUsize, |
50 | | /// Shutdown timeout. |
51 | | timeout: Duration, |
52 | | } |
53 | | |
54 | | impl GracefulShutdown { |
55 | | /// Create a new shutdown coordinator. |
56 | 0 | pub fn new(timeout: Duration) -> Self { |
57 | 0 | Self { |
58 | 0 | shutdown_requested: AtomicBool::new(false), |
59 | 0 | active_count: AtomicUsize::new(0), |
60 | 0 | timeout, |
61 | 0 | } |
62 | 0 | } |
63 | | |
64 | | /// Check if shutdown has been requested. |
65 | | #[must_use] |
66 | 0 | pub fn is_shutdown_requested(&self) -> bool { |
67 | 0 | self.shutdown_requested.load(Ordering::Acquire) |
68 | 0 | } |
69 | | |
70 | | /// Get the current active operation count. |
71 | | #[must_use] |
72 | 0 | pub fn active_count(&self) -> usize { |
73 | 0 | self.active_count.load(Ordering::Acquire) |
74 | 0 | } |
75 | | |
76 | | /// Register an active operation. |
77 | | /// |
78 | | /// Returns `None` if shutdown has already been requested. |
79 | 0 | pub fn register(&self) -> Option<ShutdownGuard<'_>> { |
80 | 0 | if self.is_shutdown_requested() { |
81 | 0 | return None; // Reject new operations during shutdown |
82 | 0 | } |
83 | 0 | self.active_count.fetch_add(1, Ordering::AcqRel); |
84 | 0 | Some(ShutdownGuard { shutdown: self }) |
85 | 0 | } |
86 | | |
87 | | /// Initiate graceful shutdown. |
88 | | /// |
89 | | /// This will: |
90 | | /// 1. Stop accepting new operations |
91 | | /// 2. Wait for in-flight operations to complete (up to timeout) |
92 | | /// 3. Return the result |
93 | 0 | pub fn shutdown(&self) -> ShutdownResult { |
94 | | // Phase 1: Stop accepting new operations |
95 | 0 | self.shutdown_requested.store(true, Ordering::Release); |
96 | | |
97 | | // Phase 2: Wait for in-flight operations |
98 | 0 | let deadline = Instant::now() + self.timeout; |
99 | | |
100 | | loop { |
101 | 0 | let active = self.active_count.load(Ordering::Acquire); |
102 | 0 | if active == 0 { |
103 | 0 | return ShutdownResult::Clean; |
104 | 0 | } |
105 | 0 | if Instant::now() >= deadline { |
106 | 0 | return ShutdownResult::Timeout { remaining: active }; |
107 | 0 | } |
108 | 0 | std::thread::sleep(Duration::from_millis(10)); |
109 | | } |
110 | 0 | } |
111 | | |
112 | | /// Reset the shutdown coordinator for reuse. |
113 | 0 | pub fn reset(&self) { |
114 | 0 | self.shutdown_requested.store(false, Ordering::Release); |
115 | | // Note: active_count should already be 0 if shutdown completed cleanly |
116 | 0 | } |
117 | | } |
118 | | |
119 | | impl Default for GracefulShutdown { |
120 | 0 | fn default() -> Self { |
121 | 0 | Self::new(Duration::from_secs(30)) |
122 | 0 | } |
123 | | } |
124 | | |
125 | | /// Guard that decrements active count on drop. |
126 | | pub struct ShutdownGuard<'a> { |
127 | | shutdown: &'a GracefulShutdown, |
128 | | } |
129 | | |
130 | | impl Drop for ShutdownGuard<'_> { |
131 | 0 | fn drop(&mut self) { |
132 | 0 | self.shutdown.active_count.fetch_sub(1, Ordering::AcqRel); |
133 | 0 | } |
134 | | } |
135 | | |
136 | | #[cfg(test)] |
137 | | mod tests { |
138 | | use super::*; |
139 | | |
140 | | #[test] |
141 | | fn test_shutdown_result_eq() { |
142 | | assert_eq!(ShutdownResult::Clean, ShutdownResult::Clean); |
143 | | assert_eq!( |
144 | | ShutdownResult::Timeout { remaining: 5 }, |
145 | | ShutdownResult::Timeout { remaining: 5 } |
146 | | ); |
147 | | assert_ne!(ShutdownResult::Clean, ShutdownResult::Timeout { remaining: 0 }); |
148 | | } |
149 | | |
150 | | #[test] |
151 | | fn test_graceful_shutdown_new() { |
152 | | let shutdown = GracefulShutdown::new(Duration::from_secs(5)); |
153 | | assert!(!shutdown.is_shutdown_requested()); |
154 | | assert_eq!(shutdown.active_count(), 0); |
155 | | } |
156 | | |
157 | | #[test] |
158 | | fn test_graceful_shutdown_default() { |
159 | | let shutdown = GracefulShutdown::default(); |
160 | | assert!(!shutdown.is_shutdown_requested()); |
161 | | assert_eq!(shutdown.timeout, Duration::from_secs(30)); |
162 | | } |
163 | | |
164 | | #[test] |
165 | | fn test_graceful_shutdown_register() { |
166 | | let shutdown = GracefulShutdown::new(Duration::from_secs(5)); |
167 | | |
168 | | let guard1 = shutdown.register(); |
169 | | assert!(guard1.is_some()); |
170 | | assert_eq!(shutdown.active_count(), 1); |
171 | | |
172 | | let guard2 = shutdown.register(); |
173 | | assert!(guard2.is_some()); |
174 | | assert_eq!(shutdown.active_count(), 2); |
175 | | |
176 | | drop(guard1); |
177 | | assert_eq!(shutdown.active_count(), 1); |
178 | | |
179 | | drop(guard2); |
180 | | assert_eq!(shutdown.active_count(), 0); |
181 | | } |
182 | | |
183 | | #[test] |
184 | | fn test_graceful_shutdown_clean() { |
185 | | let shutdown = GracefulShutdown::new(Duration::from_millis(100)); |
186 | | |
187 | | // Register and immediately drop |
188 | | let guard = shutdown.register(); |
189 | | drop(guard); |
190 | | |
191 | | let result = shutdown.shutdown(); |
192 | | assert_eq!(result, ShutdownResult::Clean); |
193 | | } |
194 | | |
195 | | #[test] |
196 | | fn test_graceful_shutdown_rejects_after_shutdown() { |
197 | | let shutdown = GracefulShutdown::new(Duration::from_millis(10)); |
198 | | |
199 | | // Initiate shutdown |
200 | | let result = shutdown.shutdown(); |
201 | | assert_eq!(result, ShutdownResult::Clean); |
202 | | |
203 | | // Try to register after shutdown |
204 | | let guard = shutdown.register(); |
205 | | assert!(guard.is_none(), "Should reject new operations after shutdown"); |
206 | | } |
207 | | |
208 | | #[test] |
209 | | fn test_graceful_shutdown_reset() { |
210 | | let shutdown = GracefulShutdown::new(Duration::from_millis(10)); |
211 | | |
212 | | // Shutdown |
213 | | shutdown.shutdown(); |
214 | | assert!(shutdown.is_shutdown_requested()); |
215 | | |
216 | | // Reset |
217 | | shutdown.reset(); |
218 | | assert!(!shutdown.is_shutdown_requested()); |
219 | | |
220 | | // Should accept new operations again |
221 | | let guard = shutdown.register(); |
222 | | assert!(guard.is_some()); |
223 | | } |
224 | | |
225 | | /// FALSIFICATION TEST: Verify timeout behavior |
226 | | /// |
227 | | /// The shutdown coordinator MUST return Timeout when operations |
228 | | /// don't complete within the specified duration. |
229 | | #[test] |
230 | | fn test_falsify_shutdown_timeout() { |
231 | | let shutdown = GracefulShutdown::new(Duration::from_millis(50)); |
232 | | |
233 | | // Register an operation that we WON'T drop |
234 | | let _guard = shutdown.register(); |
235 | | assert_eq!(shutdown.active_count(), 1); |
236 | | |
237 | | // Initiate shutdown - should timeout since operation is still active |
238 | | let start = Instant::now(); |
239 | | let result = shutdown.shutdown(); |
240 | | let elapsed = start.elapsed(); |
241 | | |
242 | | // CRITICAL ASSERTIONS |
243 | | match result { |
244 | | ShutdownResult::Timeout { remaining } => { |
245 | | assert_eq!(remaining, 1, "Should report exactly 1 remaining operation"); |
246 | | } |
247 | | ShutdownResult::Clean => { |
248 | | panic!("FALSIFICATION FAILED: Shutdown reported Clean with active operation"); |
249 | | } |
250 | | } |
251 | | |
252 | | // Verify we actually waited (within tolerance) |
253 | | assert!( |
254 | | elapsed >= Duration::from_millis(40), |
255 | | "FALSIFICATION FAILED: Shutdown returned too early ({:?} < 40ms)", |
256 | | elapsed |
257 | | ); |
258 | | } |
259 | | |
260 | | /// FALSIFICATION TEST: Verify guard drop decrements count |
261 | | /// |
262 | | /// If the guard drop doesn't work correctly, the system will |
263 | | /// either hang forever or report wrong active counts. |
264 | | #[test] |
265 | | fn test_falsify_guard_drop_semantics() { |
266 | | let shutdown = GracefulShutdown::new(Duration::from_secs(1)); |
267 | | |
268 | | // Create many guards |
269 | | let mut guards: Vec<_> = (0..100).filter_map(|_| shutdown.register()).collect(); |
270 | | assert_eq!( |
271 | | shutdown.active_count(), |
272 | | 100, |
273 | | "FALSIFICATION FAILED: Not all guards registered" |
274 | | ); |
275 | | |
276 | | // Drop half by truncating the vector (keeps first 50, drops last 50) |
277 | | guards.truncate(50); |
278 | | |
279 | | // CRITICAL: Active count must be exactly 50 |
280 | | // If fetch_sub has a bug (e.g., fetch_add by mistake), this catches it |
281 | | assert_eq!( |
282 | | shutdown.active_count(), |
283 | | 50, |
284 | | "FALSIFICATION FAILED: Guard drop did not correctly decrement count" |
285 | | ); |
286 | | |
287 | | // Drop remaining 50 |
288 | | drop(guards); |
289 | | assert_eq!( |
290 | | shutdown.active_count(), |
291 | | 0, |
292 | | "FALSIFICATION FAILED: Final drop did not clear all guards" |
293 | | ); |
294 | | } |
295 | | |
296 | | /// FALSIFICATION TEST: Verify concurrent registration rejection |
297 | | /// |
298 | | /// Once shutdown starts, NO new operations should be accepted, |
299 | | /// even if they race with the shutdown call. |
300 | | #[test] |
301 | | fn test_falsify_rejection_after_shutdown_flag() { |
302 | | let shutdown = GracefulShutdown::new(Duration::from_secs(5)); |
303 | | |
304 | | // Manually set shutdown flag (simulating shutdown start) |
305 | | shutdown.shutdown_requested.store(true, Ordering::Release); |
306 | | |
307 | | // Try to register many operations |
308 | | let mut accepted = 0; |
309 | | for _ in 0..100 { |
310 | | if shutdown.register().is_some() { |
311 | | accepted += 1; |
312 | | } |
313 | | } |
314 | | |
315 | | // CRITICAL: NO operations should have been accepted |
316 | | assert_eq!( |
317 | | accepted, 0, |
318 | | "FALSIFICATION FAILED: {} operations accepted after shutdown flag set", |
319 | | accepted |
320 | | ); |
321 | | } |
322 | | } |