Coverage Report

Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/trueno/src/brick/shutdown.rs
Line
Count
Source
1
//! Graceful Shutdown Coordination
2
//!
3
//! AWP-07: Two-phase shutdown with active operation tracking.
4
5
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
6
use std::time::{Duration, Instant};
7
8
// ----------------------------------------------------------------------------
9
// AWP-07: Graceful Shutdown
10
// ----------------------------------------------------------------------------
11
12
/// Result of a graceful shutdown operation.
13
#[derive(Debug, Clone, PartialEq, Eq)]
14
pub enum ShutdownResult {
15
    /// All operations completed cleanly.
16
    Clean,
17
    /// Timeout reached with operations still active.
18
    Timeout {
19
        /// Number of operations still active.
20
        remaining: usize,
21
    },
22
}
23
24
/// Graceful shutdown coordinator.
25
///
26
/// # Example
27
/// ```rust
28
/// use trueno::brick::GracefulShutdown;
29
/// use std::time::Duration;
30
///
31
/// let shutdown = GracefulShutdown::new(Duration::from_secs(5));
32
///
33
/// // Register an operation
34
/// let guard = shutdown.register().ok_or("shutdown in progress")?;
35
///
36
/// // ... do work ...
37
///
38
/// drop(guard);  // Operation complete
39
///
40
/// // Initiate shutdown
41
/// let result = shutdown.shutdown();
42
/// assert_eq!(result, trueno::brick::ShutdownResult::Clean);
43
/// # Ok::<(), &'static str>(())
44
/// ```
45
pub struct GracefulShutdown {
46
    /// Flag indicating shutdown has been requested.
47
    shutdown_requested: AtomicBool,
48
    /// Number of active operations.
49
    active_count: AtomicUsize,
50
    /// Shutdown timeout.
51
    timeout: Duration,
52
}
53
54
impl GracefulShutdown {
55
    /// Create a new shutdown coordinator.
56
0
    pub fn new(timeout: Duration) -> Self {
57
0
        Self {
58
0
            shutdown_requested: AtomicBool::new(false),
59
0
            active_count: AtomicUsize::new(0),
60
0
            timeout,
61
0
        }
62
0
    }
63
64
    /// Check if shutdown has been requested.
65
    #[must_use]
66
0
    pub fn is_shutdown_requested(&self) -> bool {
67
0
        self.shutdown_requested.load(Ordering::Acquire)
68
0
    }
69
70
    /// Get the current active operation count.
71
    #[must_use]
72
0
    pub fn active_count(&self) -> usize {
73
0
        self.active_count.load(Ordering::Acquire)
74
0
    }
75
76
    /// Register an active operation.
77
    ///
78
    /// Returns `None` if shutdown has already been requested.
79
0
    pub fn register(&self) -> Option<ShutdownGuard<'_>> {
80
0
        if self.is_shutdown_requested() {
81
0
            return None; // Reject new operations during shutdown
82
0
        }
83
0
        self.active_count.fetch_add(1, Ordering::AcqRel);
84
0
        Some(ShutdownGuard { shutdown: self })
85
0
    }
86
87
    /// Initiate graceful shutdown.
88
    ///
89
    /// This will:
90
    /// 1. Stop accepting new operations
91
    /// 2. Wait for in-flight operations to complete (up to timeout)
92
    /// 3. Return the result
93
0
    pub fn shutdown(&self) -> ShutdownResult {
94
        // Phase 1: Stop accepting new operations
95
0
        self.shutdown_requested.store(true, Ordering::Release);
96
97
        // Phase 2: Wait for in-flight operations
98
0
        let deadline = Instant::now() + self.timeout;
99
100
        loop {
101
0
            let active = self.active_count.load(Ordering::Acquire);
102
0
            if active == 0 {
103
0
                return ShutdownResult::Clean;
104
0
            }
105
0
            if Instant::now() >= deadline {
106
0
                return ShutdownResult::Timeout { remaining: active };
107
0
            }
108
0
            std::thread::sleep(Duration::from_millis(10));
109
        }
110
0
    }
111
112
    /// Reset the shutdown coordinator for reuse.
113
0
    pub fn reset(&self) {
114
0
        self.shutdown_requested.store(false, Ordering::Release);
115
        // Note: active_count should already be 0 if shutdown completed cleanly
116
0
    }
117
}
118
119
impl Default for GracefulShutdown {
120
0
    fn default() -> Self {
121
0
        Self::new(Duration::from_secs(30))
122
0
    }
123
}
124
125
/// Guard that decrements active count on drop.
126
pub struct ShutdownGuard<'a> {
127
    shutdown: &'a GracefulShutdown,
128
}
129
130
impl Drop for ShutdownGuard<'_> {
131
0
    fn drop(&mut self) {
132
0
        self.shutdown.active_count.fetch_sub(1, Ordering::AcqRel);
133
0
    }
134
}
135
136
#[cfg(test)]
137
mod tests {
138
    use super::*;
139
140
    #[test]
141
    fn test_shutdown_result_eq() {
142
        assert_eq!(ShutdownResult::Clean, ShutdownResult::Clean);
143
        assert_eq!(
144
            ShutdownResult::Timeout { remaining: 5 },
145
            ShutdownResult::Timeout { remaining: 5 }
146
        );
147
        assert_ne!(ShutdownResult::Clean, ShutdownResult::Timeout { remaining: 0 });
148
    }
149
150
    #[test]
151
    fn test_graceful_shutdown_new() {
152
        let shutdown = GracefulShutdown::new(Duration::from_secs(5));
153
        assert!(!shutdown.is_shutdown_requested());
154
        assert_eq!(shutdown.active_count(), 0);
155
    }
156
157
    #[test]
158
    fn test_graceful_shutdown_default() {
159
        let shutdown = GracefulShutdown::default();
160
        assert!(!shutdown.is_shutdown_requested());
161
        assert_eq!(shutdown.timeout, Duration::from_secs(30));
162
    }
163
164
    #[test]
165
    fn test_graceful_shutdown_register() {
166
        let shutdown = GracefulShutdown::new(Duration::from_secs(5));
167
168
        let guard1 = shutdown.register();
169
        assert!(guard1.is_some());
170
        assert_eq!(shutdown.active_count(), 1);
171
172
        let guard2 = shutdown.register();
173
        assert!(guard2.is_some());
174
        assert_eq!(shutdown.active_count(), 2);
175
176
        drop(guard1);
177
        assert_eq!(shutdown.active_count(), 1);
178
179
        drop(guard2);
180
        assert_eq!(shutdown.active_count(), 0);
181
    }
182
183
    #[test]
184
    fn test_graceful_shutdown_clean() {
185
        let shutdown = GracefulShutdown::new(Duration::from_millis(100));
186
187
        // Register and immediately drop
188
        let guard = shutdown.register();
189
        drop(guard);
190
191
        let result = shutdown.shutdown();
192
        assert_eq!(result, ShutdownResult::Clean);
193
    }
194
195
    #[test]
196
    fn test_graceful_shutdown_rejects_after_shutdown() {
197
        let shutdown = GracefulShutdown::new(Duration::from_millis(10));
198
199
        // Initiate shutdown
200
        let result = shutdown.shutdown();
201
        assert_eq!(result, ShutdownResult::Clean);
202
203
        // Try to register after shutdown
204
        let guard = shutdown.register();
205
        assert!(guard.is_none(), "Should reject new operations after shutdown");
206
    }
207
208
    #[test]
209
    fn test_graceful_shutdown_reset() {
210
        let shutdown = GracefulShutdown::new(Duration::from_millis(10));
211
212
        // Shutdown
213
        shutdown.shutdown();
214
        assert!(shutdown.is_shutdown_requested());
215
216
        // Reset
217
        shutdown.reset();
218
        assert!(!shutdown.is_shutdown_requested());
219
220
        // Should accept new operations again
221
        let guard = shutdown.register();
222
        assert!(guard.is_some());
223
    }
224
225
    /// FALSIFICATION TEST: Verify timeout behavior
226
    ///
227
    /// The shutdown coordinator MUST return Timeout when operations
228
    /// don't complete within the specified duration.
229
    #[test]
230
    fn test_falsify_shutdown_timeout() {
231
        let shutdown = GracefulShutdown::new(Duration::from_millis(50));
232
233
        // Register an operation that we WON'T drop
234
        let _guard = shutdown.register();
235
        assert_eq!(shutdown.active_count(), 1);
236
237
        // Initiate shutdown - should timeout since operation is still active
238
        let start = Instant::now();
239
        let result = shutdown.shutdown();
240
        let elapsed = start.elapsed();
241
242
        // CRITICAL ASSERTIONS
243
        match result {
244
            ShutdownResult::Timeout { remaining } => {
245
                assert_eq!(remaining, 1, "Should report exactly 1 remaining operation");
246
            }
247
            ShutdownResult::Clean => {
248
                panic!("FALSIFICATION FAILED: Shutdown reported Clean with active operation");
249
            }
250
        }
251
252
        // Verify we actually waited (within tolerance)
253
        assert!(
254
            elapsed >= Duration::from_millis(40),
255
            "FALSIFICATION FAILED: Shutdown returned too early ({:?} < 40ms)",
256
            elapsed
257
        );
258
    }
259
260
    /// FALSIFICATION TEST: Verify guard drop decrements count
261
    ///
262
    /// If the guard drop doesn't work correctly, the system will
263
    /// either hang forever or report wrong active counts.
264
    #[test]
265
    fn test_falsify_guard_drop_semantics() {
266
        let shutdown = GracefulShutdown::new(Duration::from_secs(1));
267
268
        // Create many guards
269
        let mut guards: Vec<_> = (0..100).filter_map(|_| shutdown.register()).collect();
270
        assert_eq!(
271
            shutdown.active_count(),
272
            100,
273
            "FALSIFICATION FAILED: Not all guards registered"
274
        );
275
276
        // Drop half by truncating the vector (keeps first 50, drops last 50)
277
        guards.truncate(50);
278
279
        // CRITICAL: Active count must be exactly 50
280
        // If fetch_sub has a bug (e.g., fetch_add by mistake), this catches it
281
        assert_eq!(
282
            shutdown.active_count(),
283
            50,
284
            "FALSIFICATION FAILED: Guard drop did not correctly decrement count"
285
        );
286
287
        // Drop remaining 50
288
        drop(guards);
289
        assert_eq!(
290
            shutdown.active_count(),
291
            0,
292
            "FALSIFICATION FAILED: Final drop did not clear all guards"
293
        );
294
    }
295
296
    /// FALSIFICATION TEST: Verify concurrent registration rejection
297
    ///
298
    /// Once shutdown starts, NO new operations should be accepted,
299
    /// even if they race with the shutdown call.
300
    #[test]
301
    fn test_falsify_rejection_after_shutdown_flag() {
302
        let shutdown = GracefulShutdown::new(Duration::from_secs(5));
303
304
        // Manually set shutdown flag (simulating shutdown start)
305
        shutdown.shutdown_requested.store(true, Ordering::Release);
306
307
        // Try to register many operations
308
        let mut accepted = 0;
309
        for _ in 0..100 {
310
            if shutdown.register().is_some() {
311
                accepted += 1;
312
            }
313
        }
314
315
        // CRITICAL: NO operations should have been accepted
316
        assert_eq!(
317
            accepted, 0,
318
            "FALSIFICATION FAILED: {} operations accepted after shutdown flag set",
319
            accepted
320
        );
321
    }
322
}