## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/metrics.rs

    Line
    Count
    Source
    1
    //! Metrics collection and reporting for production monitoring
    2
    //!
    3
    //! This module provides comprehensive metrics for tracking:
    4
    //! - Request latency (p50, p95, p99)
    5
    //! - Throughput (requests/sec, tokens/sec)
    6
    //! - Error rates and categorization
    7
    //! - Model performance (inference time, token generation)
    8
    //!
    9
    //! Metrics are exposed in Prometheus format for easy integration with monitoring systems.
    10
    11
    use std::{
    12
        sync::{
    13
            atomic::{AtomicU64, AtomicUsize, Ordering},
    14
            Arc,
    15
        },
    16
        time::{Duration, Instant},
    17
    };
    18
    19
    /// Central metrics collector for tracking system performance
    20
    #[derive(Debug, Clone)]
    21
    pub struct MetricsCollector {
    22
        /// Total number of requests processed
    23
        total_requests: Arc<AtomicUsize>,
    24
        /// Total number of successful requests
    25
        successful_requests: Arc<AtomicUsize>,
    26
        /// Total number of failed requests
    27
        failed_requests: Arc<AtomicUsize>,
    28
        /// Total number of tokens generated
    29
        total_tokens: Arc<AtomicUsize>,
    30
        /// Total inference time in microseconds
    31
        total_inference_time_us: Arc<AtomicU64>,
    32
        /// Start time for rate calculations
    33
        start_time: Instant,
    34
    }
    35
    36
    impl MetricsCollector {
    37
        /// Create a new metrics collector
    38
        #[must_use]
    39
    143
        pub fn new() -> Self {
    40
    143
            Self {
    41
    143
                total_requests: Arc::new(AtomicUsize::new(0)),
    42
    143
                successful_requests: Arc::new(AtomicUsize::new(0)),
    43
    143
                failed_requests: Arc::new(AtomicUsize::new(0)),
    44
    143
                total_tokens: Arc::new(AtomicUsize::new(0)),
    45
    143
                total_inference_time_us: Arc::new(AtomicU64::new(0)),
    46
    143
                start_time: Instant::now(),
    47
    143
            }
    48
    143
        }
    49
    50
        /// Record a successful request
    51
        #[allow(clippy::cast_possible_truncation)]
    52
    225
        pub fn record_success(&self, tokens: usize, duration: Duration) {
    53
    225
            self.total_requests.fetch_add(1, Ordering::Relaxed);
    54
    225
            self.successful_requests.fetch_add(1, Ordering::Relaxed);
    55
    225
            self.total_tokens.fetch_add(tokens, Ordering::Relaxed);
    56
    225
            self.total_inference_time_us
    57
    225
                .fetch_add(duration.as_micros() as u64, Ordering::Relaxed);
    58
    225
        }
    59
    60
        /// Record a failed request
    61
    13
        pub fn record_failure(&self) {
    62
    13
            self.total_requests.fetch_add(1, Ordering::Relaxed);
    63
    13
            self.failed_requests.fetch_add(1, Ordering::Relaxed);
    64
    13
        }
    65
    66
        /// Get current snapshot of metrics
    67
        #[must_use]
    68
        #[allow(clippy::cast_precision_loss)]
    69
    13
        pub fn snapshot(&self) -> MetricsSnapshot {
    70
    13
            let total_requests = self.total_requests.load(Ordering::Relaxed);
    71
    13
            let successful = self.successful_requests.load(Ordering::Relaxed);
    72
    13
            let failed = self.failed_requests.load(Ordering::Relaxed);
    73
    13
            let total_tokens = self.total_tokens.load(Ordering::Relaxed);
    74
    13
            let total_time_us = self.total_inference_time_us.load(Ordering::Relaxed);
    75
    13
            let uptime = self.start_time.elapsed();
    76
    77
            MetricsSnapshot {
    78
    13
                total_requests,
    79
    13
                successful_requests: successful,
    80
    13
                failed_requests: failed,
    81
    13
                total_tokens,
    82
    13
                total_inference_time_us: total_time_us,
    83
    13
                uptime_secs: uptime.as_secs(),
    84
    13
                requests_per_sec: if uptime.as_secs() > 0 {
    85
    1
                    total_requests as f64 / uptime.as_secs_f64()
    86
                } else {
    87
    12
                    0.0
    88
                },
    89
    13
                tokens_per_sec: if uptime.as_secs() > 0 {
    90
    1
                    total_tokens as f64 / uptime.as_secs_f64()
    91
                } else {
    92
    12
                    0.0
    93
                },
    94
    13
                avg_latency_ms: if successful > 0 {
    95
    7
                    (total_time_us as f64 / 1000.0) / successful as f64
    96
                } else {
    97
    6
                    0.0
    98
                },
    99
    13
                error_rate: if total_requests > 0 {
    100
    8
                    failed as f64 / total_requests as f64
    101
                } else {
    102
    5
                    0.0
    103
                },
    104
            }
    105
    13
        }
    106
    107
        /// Export metrics in Prometheus format
    108
        #[must_use]
    109
        #[allow(clippy::cast_precision_loss)]
    110
    2
        pub fn to_prometheus(&self) -> String {
    111
    2
            let snapshot = self.snapshot();
    112
    2
            format!(
    113
    2
                "# HELP realizar_requests_total Total number of requests\n\
    114
    2
                 # TYPE realizar_requests_total counter\n\
    115
    2
                 realizar_requests_total {}\n\
    116
    2
                 # HELP realizar_requests_successful Successful requests\n\
    117
    2
                 # TYPE realizar_requests_successful counter\n\
    118
    2
                 realizar_requests_successful {}\n\
    119
    2
                 # HELP realizar_requests_failed Failed requests\n\
    120
    2
                 # TYPE realizar_requests_failed counter\n\
    121
    2
                 realizar_requests_failed {}\n\
    122
    2
                 # HELP realizar_tokens_generated Total tokens generated\n\
    123
    2
                 # TYPE realizar_tokens_generated counter\n\
    124
    2
                 realizar_tokens_generated {}\n\
    125
    2
                 # HELP realizar_inference_time_seconds Total inference time\n\
    126
    2
                 # TYPE realizar_inference_time_seconds counter\n\
    127
    2
                 realizar_inference_time_seconds {:.6}\n\
    128
    2
                 # HELP realizar_requests_per_second Request rate\n\
    129
    2
                 # TYPE realizar_requests_per_second gauge\n\
    130
    2
                 realizar_requests_per_second {:.2}\n\
    131
    2
                 # HELP realizar_tokens_per_second Token generation rate\n\
    132
    2
                 # TYPE realizar_tokens_per_second gauge\n\
    133
    2
                 realizar_tokens_per_second {:.2}\n\
    134
    2
                 # HELP realizar_avg_latency_ms Average latency in milliseconds\n\
    135
    2
                 # TYPE realizar_avg_latency_ms gauge\n\
    136
    2
                 realizar_avg_latency_ms {:.2}\n\
    137
    2
                 # HELP realizar_error_rate Error rate (0.0-1.0)\n\
    138
    2
                 # TYPE realizar_error_rate gauge\n\
    139
    2
                 realizar_error_rate {:.4}\n\
    140
    2
                 # HELP realizar_uptime_seconds Uptime in seconds\n\
    141
    2
                 # TYPE realizar_uptime_seconds counter\n\
    142
    2
                 realizar_uptime_seconds {}\n",
    143
                snapshot.total_requests,
    144
                snapshot.successful_requests,
    145
                snapshot.failed_requests,
    146
                snapshot.total_tokens,
    147
    2
                snapshot.total_inference_time_us as f64 / 1_000_000.0,
    148
                snapshot.requests_per_sec,
    149
                snapshot.tokens_per_sec,
    150
                snapshot.avg_latency_ms,
    151
                snapshot.error_rate,
    152
                snapshot.uptime_secs
    153
            )
    154
    2
        }
    155
    156
        /// Reset all metrics (useful for testing)
    157
    1
        pub fn reset(&self) {
    158
    1
            self.total_requests.store(0, Ordering::Relaxed);
    159
    1
            self.successful_requests.store(0, Ordering::Relaxed);
    160
    1
            self.failed_requests.store(0, Ordering::Relaxed);
    161
    1
            self.total_tokens.store(0, Ordering::Relaxed);
    162
    1
            self.total_inference_time_us.store(0, Ordering::Relaxed);
    163
    1
        }
    164
    }
    165
    166
    impl Default for MetricsCollector {
    167
    0
        fn default() -> Self {
    168
    0
            Self::new()
    169
    0
        }
    170
    }
    171
    172
    /// Snapshot of current metrics
    173
    #[derive(Debug, Clone)]
    174
    pub struct MetricsSnapshot {
    175
        /// Total number of requests processed
    176
        pub total_requests: usize,
    177
        /// Number of successful requests
    178
        pub successful_requests: usize,
    179
        /// Number of failed requests
    180
        pub failed_requests: usize,
    181
        /// Total tokens generated across all requests
    182
        pub total_tokens: usize,
    183
        /// Total inference time in microseconds
    184
        pub total_inference_time_us: u64,
    185
        /// System uptime in seconds
    186
        pub uptime_secs: u64,
    187
        /// Request rate (requests per second)
    188
        pub requests_per_sec: f64,
    189
        /// Token generation rate (tokens per second)
    190
        pub tokens_per_sec: f64,
    191
        /// Average request latency in milliseconds
    192
        pub avg_latency_ms: f64,
    193
        /// Error rate as a fraction (0.0 to 1.0)
    194
        pub error_rate: f64,
    195
    }
    196
    197
    #[cfg(test)]
    198
    mod tests {
    199
        use std::thread;
    200
    201
        use super::*;
    202
    203
        #[test]
    204
    1
        fn test_metrics_collector_creation() {
    205
    1
            let metrics = MetricsCollector::new();
    206
    1
            let snapshot = metrics.snapshot();
    207
    208
    1
            assert_eq!(snapshot.total_requests, 0);
    209
    1
            assert_eq!(snapshot.successful_requests, 0);
    210
    1
            assert_eq!(snapshot.failed_requests, 0);
    211
    1
            assert_eq!(snapshot.total_tokens, 0);
    212
    1
            assert_eq!(snapshot.total_inference_time_us, 0);
    213
    1
        }
    214
    215
        #[test]
    216
    1
        fn test_record_success() {
    217
    1
            let metrics = MetricsCollector::new();
    218
    1
            metrics.record_success(10, Duration::from_millis(100));
    219
    220
    1
            let snapshot = metrics.snapshot();
    221
    1
            assert_eq!(snapshot.total_requests, 1);
    222
    1
            assert_eq!(snapshot.successful_requests, 1);
    223
    1
            assert_eq!(snapshot.failed_requests, 0);
    224
    1
            assert_eq!(snapshot.total_tokens, 10);
    225
    1
            assert!(snapshot.total_inference_time_us >= 100_000);
    226
    1
        }
    227
    228
        #[test]
    229
    1
        fn test_record_failure() {
    230
    1
            let metrics = MetricsCollector::new();
    231
    1
            metrics.record_failure();
    232
    233
    1
            let snapshot = metrics.snapshot();
    234
    1
            assert_eq!(snapshot.total_requests, 1);
    235
    1
            assert_eq!(snapshot.successful_requests, 0);
    236
    1
            assert_eq!(snapshot.failed_requests, 1);
    237
    1
            approx::assert_relative_eq!(snapshot.error_rate, 1.0);
    238
    1
        }
    239
    240
        #[test]
    241
    1
        fn test_multiple_requests() {
    242
    1
            let metrics = MetricsCollector::new();
    243
    244
    1
            metrics.record_success(5, Duration::from_millis(50));
    245
    1
            metrics.record_success(10, Duration::from_millis(100));
    246
    1
            metrics.record_failure();
    247
    248
    1
            let snapshot = metrics.snapshot();
    249
    1
            assert_eq!(snapshot.total_requests, 3);
    250
    1
            assert_eq!(snapshot.successful_requests, 2);
    251
    1
            assert_eq!(snapshot.failed_requests, 1);
    252
    1
            assert_eq!(snapshot.total_tokens, 15);
    253
    1
            approx::assert_relative_eq!(snapshot.error_rate, 1.0 / 3.0);
    254
    1
        }
    255
    256
        #[test]
    257
    1
        fn test_avg_latency_calculation() {
    258
    1
            let metrics = MetricsCollector::new();
    259
    260
            // Record 100ms and 200ms requests
    261
    1
            metrics.record_success(1, Duration::from_millis(100));
    262
    1
            metrics.record_success(1, Duration::from_millis(200));
    263
    264
    1
            let snapshot = metrics.snapshot();
    265
            // Average should be 150ms
    266
    1
            assert!((snapshot.avg_latency_ms - 150.0).abs() < 1.0);
    267
    1
        }
    268
    269
        #[test]
    270
    1
        fn test_tokens_per_second() {
    271
    1
            let metrics = MetricsCollector::new();
    272
    273
            // Wait to ensure at least 1 second has passed for rate calculation
    274
    1
            thread::sleep(Duration::from_secs(1));
    275
    276
            // Record some tokens
    277
    1
            metrics.record_success(100, Duration::from_millis(10));
    278
    279
    1
            let snapshot = metrics.snapshot();
    280
            // Should have positive rate after 1+ seconds
    281
    1
            assert!(snapshot.tokens_per_sec > 0.0);
    282
    1
            assert!(snapshot.tokens_per_sec <= 100.0); // Can't exceed total tokens
    283
    1
        }
    284
    285
        #[test]
    286
    1
        fn test_prometheus_format() {
    287
    1
            let metrics = MetricsCollector::new();
    288
    1
            metrics.record_success(10, Duration::from_millis(100));
    289
    1
            metrics.record_failure();
    290
    291
    1
            let prom = metrics.to_prometheus();
    292
    293
            // Check that all required metrics are present
    294
    1
            assert!(prom.contains("realizar_requests_total 2"));
    295
    1
            assert!(prom.contains("realizar_requests_successful 1"));
    296
    1
            assert!(prom.contains("realizar_requests_failed 1"));
    297
    1
            assert!(prom.contains("realizar_tokens_generated 10"));
    298
    1
            assert!(prom.contains("realizar_error_rate 0.5000"));
    299
    1
        }
    300
    301
        #[test]
    302
    1
        fn test_reset_metrics() {
    303
    1
            let metrics = MetricsCollector::new();
    304
    1
            metrics.record_success(10, Duration::from_millis(100));
    305
    1
            metrics.record_failure();
    306
    307
    1
            metrics.reset();
    308
    309
    1
            let snapshot = metrics.snapshot();
    310
    1
            assert_eq!(snapshot.total_requests, 0);
    311
    1
            assert_eq!(snapshot.successful_requests, 0);
    312
    1
            assert_eq!(snapshot.failed_requests, 0);
    313
    1
            assert_eq!(snapshot.total_tokens, 0);
    314
    1
            assert_eq!(snapshot.total_inference_time_us, 0);
    315
    1
        }
    316
    317
        #[test]
    318
    1
        fn test_concurrent_updates() {
    319
    1
            let metrics = MetricsCollector::new();
    320
    1
            let metrics_clone = metrics.clone();
    321
    322
    1
            let handle = thread::spawn(move || {
    323
    101
                for _ in 0..100 {
    324
    100
                    metrics_clone.record_success(1, Duration::from_micros(100));
    325
    100
                }
    326
    1
            });
    327
    328
    101
            for _ in 0..100 {
    329
    100
                metrics.record_success(1, Duration::from_micros(100));
    330
    100
            }
    331
    332
    1
            handle.join().expect("test");
    333
    334
    1
            let snapshot = metrics.snapshot();
    335
    1
            assert_eq!(snapshot.total_requests, 200);
    336
    1
            assert_eq!(snapshot.successful_requests, 200);
    337
    1
            assert_eq!(snapshot.total_tokens, 200);
    338
    1
        }
    339
    340
        #[test]
    341
    1
        fn test_zero_division_safety() {
    342
    1
            let metrics = MetricsCollector::new();
    343
    1
            let snapshot = metrics.snapshot();
    344
    345
            // Should not panic with zero values
    346
    1
            approx::assert_relative_eq!(snapshot.requests_per_sec, 0.0);
    347
    1
            approx::assert_relative_eq!(snapshot.tokens_per_sec, 0.0);
    348
    1
            approx::assert_relative_eq!(snapshot.avg_latency_ms, 0.0);
    349
    1
            approx::assert_relative_eq!(snapshot.error_rate, 0.0);
    350
    1
        }
    351
    }