## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/cache.rs

    Line
    Count
    Source
    1
    //! Model caching and warming for reduced latency
    2
    //!
    3
    //! Provides LRU cache for model instances to reduce cold start latency and
    4
    //! improve throughput for repeated model usage.
    5
    //!
    6
    //! ## Features
    7
    //!
    8
    //! - **LRU Eviction**: Least Recently Used models are evicted when cache is full
    9
    //! - **Cache Warming**: Pre-load models on startup for zero cold starts
    10
    //! - **Metrics**: Track cache hits, misses, and evictions
    11
    //! - **Thread-Safe**: Concurrent access via Arc<RwLock>
    12
    //!
    13
    //! ## Example
    14
    //!
    15
    //! ```rust,ignore
    16
    //! use realizar::cache::ModelCache;
    17
    //!
    18
    //! let cache = ModelCache::new(10); // capacity: 10 models
    19
    //! cache.warm(&["model1", "model2"])?;
    20
    //!
    21
    //! let model = cache.get_or_load("model1", || load_model("model1"))?;
    22
    //! ```
    23
    24
    use std::{
    25
        collections::HashMap,
    26
        sync::{Arc, RwLock},
    27
    };
    28
    29
    use crate::{
    30
        error::RealizarError,
    31
        layers::{Model, ModelConfig},
    32
        tokenizer::BPETokenizer,
    33
    };
    34
    35
    /// Type alias for model and tokenizer pair
    36
    pub type ModelPair = (Arc<Model>, Arc<BPETokenizer>);
    37
    38
    /// Type alias for the internal cache storage
    39
    type CacheStorage = Arc<RwLock<HashMap<CacheKey, CacheEntry>>>;
    40
    41
    /// Cache key for identifying cached models
    42
    #[derive(Debug, Clone, PartialEq, Eq, Hash)]
    43
    pub struct CacheKey {
    44
        /// Model identifier (name, path, or config hash)
    45
        pub id: String,
    46
    }
    47
    48
    impl CacheKey {
    49
        /// Create a new cache key
    50
        #[must_use]
    51
    17
        pub fn new(id: String) -> Self {
    52
    17
            Self { id }
    53
    17
        }
    54
    55
        /// Create cache key from model config
    56
        #[must_use]
    57
    1
        pub fn from_config(config: &ModelConfig) -> Self {
    58
            // Simple hash based on config parameters
    59
    1
            let id = format!(
    60
    1
                "v{}_h{}_n{}_l{}_i{}",
    61
                config.vocab_size,
    62
                config.hidden_dim,
    63
                config.num_heads,
    64
                config.num_layers,
    65
                config.intermediate_dim
    66
            );
    67
    1
            Self::new(id)
    68
    1
        }
    69
    }
    70
    71
    /// Cached model entry with metadata
    72
    #[derive(Clone)]
    73
    pub struct CacheEntry {
    74
        /// The cached model
    75
        pub model: Arc<Model>,
    76
        /// The tokenizer for this model
    77
        pub tokenizer: Arc<BPETokenizer>,
    78
        /// Access count for LRU tracking
    79
        access_count: u64,
    80
        /// Last access timestamp
    81
        last_access: std::time::Instant,
    82
    }
    83
    84
    impl CacheEntry {
    85
        /// Create a new cache entry
    86
        #[must_use]
    87
    10
        pub fn new(model: Model, tokenizer: BPETokenizer) -> Self {
    88
    10
            Self {
    89
    10
                model: Arc::new(model),
    90
    10
                tokenizer: Arc::new(tokenizer),
    91
    10
                access_count: 0,
    92
    10
                last_access: std::time::Instant::now(),
    93
    10
            }
    94
    10
        }
    95
    96
        /// Record an access to this entry
    97
    8
        fn record_access(&mut self) {
    98
    8
            self.access_count += 1;
    99
    8
            self.last_access = std::time::Instant::now();
    100
    8
        }
    101
    }
    102
    103
    /// Cache metrics for monitoring
    104
    #[derive(Debug, Default, Clone)]
    105
    pub struct CacheMetrics {
    106
        /// Total cache hits
    107
        pub hits: u64,
    108
        /// Total cache misses
    109
        pub misses: u64,
    110
        /// Total evictions
    111
        pub evictions: u64,
    112
        /// Current cache size
    113
        pub size: usize,
    114
    }
    115
    116
    impl CacheMetrics {
    117
        /// Calculate hit rate as percentage
    118
        #[must_use]
    119
        #[allow(clippy::cast_precision_loss)]
    120
    3
        pub fn hit_rate(&self) -> f64 {
    121
    3
            let total = self.hits + self.misses;
    122
    3
            if total == 0 {
    123
    1
                0.0
    124
            } else {
    125
    2
                (self.hits as f64 / total as f64) * 100.0
    126
            }
    127
    3
        }
    128
    }
    129
    130
    /// LRU model cache
    131
    pub struct ModelCache {
    132
        /// Cached entries
    133
        cache: CacheStorage,
    134
        /// Maximum cache capacity
    135
        capacity: usize,
    136
        /// Cache metrics
    137
        metrics: Arc<RwLock<CacheMetrics>>,
    138
    }
    139
    140
    impl ModelCache {
    141
        /// Create a new model cache with the specified capacity
    142
        ///
    143
        /// # Arguments
    144
        ///
    145
        /// * `capacity` - Maximum number of models to cache
    146
        #[must_use]
    147
    30
        pub fn new(capacity: usize) -> Self {
    148
    30
            Self {
    149
    30
                cache: Arc::new(RwLock::new(HashMap::new())),
    150
    30
                capacity,
    151
    30
                metrics: Arc::new(RwLock::new(CacheMetrics::default())),
    152
    30
            }
    153
    30
        }
    154
    155
        /// Get a model from cache or load it using the provided function
    156
        ///
    157
        /// # Arguments
    158
        ///
    159
        /// * `key` - Cache key for the model
    160
        /// * `loader` - Function to load the model if not cached
    161
        ///
    162
        /// # Errors
    163
        ///
    164
        /// Returns error if model loading fails
    165
        ///
    166
        /// # Panics
    167
        ///
    168
        /// Panics if the internal `RwLock` is poisoned (extremely rare, indicates thread panic while holding lock)
    169
    15
        pub fn get_or_load<F>(&self, key: &CacheKey, loader: F) -> Result<ModelPair, RealizarError>
    170
    15
        where
    171
    15
            F: FnOnce() -> Result<(Model, BPETokenizer), RealizarError>,
    172
        {
    173
            // Try to get from cache first (read lock)
    174
            {
    175
    15
                let mut cache = self
    176
    15
                    .cache
    177
    15
                    .write()
    178
    15
                    .expect("RwLock poisoned: thread panicked while holding cache write lock");
    179
    15
                if let Some(entry6) = cache.get_mut(key) {
    180
    6
                    entry.record_access();
    181
    6
                    let mut metrics = self
    182
    6
                        .metrics
    183
    6
                        .write()
    184
    6
                        .expect("RwLock poisoned: thread panicked while holding metrics write lock");
    185
    6
                    metrics.hits += 1;
    186
    6
                    return Ok((entry.model.clone(), entry.tokenizer.clone()));
    187
    9
                }
    188
            }
    189
    190
            // Cache miss - load the model
    191
    9
            let (model, tokenizer) = loader()?0;
    192
    9
            let entry = CacheEntry::new(model, tokenizer);
    193
    194
            // Insert into cache (write lock)
    195
            {
    196
    9
                let mut cache = self
    197
    9
                    .cache
    198
    9
                    .write()
    199
    9
                    .expect("RwLock poisoned: thread panicked while holding cache write lock");
    200
    9
                let mut metrics = self
    201
    9
                    .metrics
    202
    9
                    .write()
    203
    9
                    .expect("RwLock poisoned: thread panicked while holding metrics write lock");
    204
    205
    9
                metrics.misses += 1;
    206
    207
                // Check if we need to evict
    208
    9
                if cache.len() >= self.capacity && !1cache1.contains_key(key) {
    209
    1
                    Self::evict_lru(&mut cache, &mut metrics);
    210
    8
                }
    211
    212
    9
                cache.insert(key.clone(), entry.clone());
    213
    9
                metrics.size = cache.len();
    214
            }
    215
    216
    9
            Ok((entry.model, entry.tokenizer))
    217
    15
        }
    218
    219
        /// Evict the least recently used entry from the cache
    220
    1
        fn evict_lru(cache: &mut HashMap<CacheKey, CacheEntry>, metrics: &mut CacheMetrics) {
    221
    1
            if let Some((lru_key, _)) = cache
    222
    1
                .iter()
    223
    1
                .min_by_key(|(_, entry)| entry.last_access)
    224
    1
                .map(|(k, e)| (k.clone(), e.clone()))
    225
    1
            {
    226
    1
                cache.remove(&lru_key);
    227
    1
                metrics.evictions += 1;
    228
    1
            }0
    229
    1
        }
    230
    231
        /// Get current cache metrics
    232
        ///
    233
        /// # Panics
    234
        ///
    235
        /// Panics if the internal `RwLock` is poisoned
    236
        #[must_use]
    237
    6
        pub fn metrics(&self) -> CacheMetrics {
    238
    6
            self.metrics
    239
    6
                .read()
    240
    6
                .expect("RwLock poisoned: thread panicked while holding metrics read lock")
    241
    6
                .clone()
    242
    6
        }
    243
    244
        /// Clear all cached models
    245
        ///
    246
        /// # Panics
    247
        ///
    248
        /// Panics if the internal `RwLock` is poisoned
    249
    1
        pub fn clear(&self) {
    250
    1
            let mut cache = self
    251
    1
                .cache
    252
    1
                .write()
    253
    1
                .expect("RwLock poisoned: thread panicked while holding cache write lock");
    254
    1
            cache.clear();
    255
    1
            let mut metrics = self
    256
    1
                .metrics
    257
    1
                .write()
    258
    1
                .expect("RwLock poisoned: thread panicked while holding metrics write lock");
    259
    1
            metrics.size = 0;
    260
    1
        }
    261
    262
        /// Get the number of cached models
    263
        ///
    264
        /// # Panics
    265
        ///
    266
        /// Panics if the internal `RwLock` is poisoned
    267
        #[must_use]
    268
    7
        pub fn len(&self) -> usize {
    269
    7
            self.cache
    270
    7
                .read()
    271
    7
                .expect("RwLock poisoned: thread panicked while holding cache read lock")
    272
    7
                .len()
    273
    7
        }
    274
    275
        /// Check if the cache is empty
    276
        ///
    277
        /// # Panics
    278
        ///
    279
        /// Panics if the internal `RwLock` is poisoned
    280
        #[must_use]
    281
    2
        pub fn is_empty(&self) -> bool {
    282
    2
            self.len() == 0
    283
    2
        }
    284
    }
    285
    286
    #[cfg(test)]
    287
    mod tests {
    288
        use super::*;
    289
    290
        type TestModelResult = Result<(Model, BPETokenizer), RealizarError>;
    291
    292
    10
        fn create_test_model(vocab_size: usize) -> TestModelResult {
    293
    10
            let config = ModelConfig {
    294
    10
                vocab_size,
    295
    10
                hidden_dim: 16,
    296
    10
                num_heads: 1,
    297
    10
                num_layers: 1,
    298
    10
                intermediate_dim: 32,
    299
    10
                eps: 1e-5,
    300
    10
            };
    301
    10
            let model = Model::new(config)?0;
    302
    303
    10
            let vocab: Vec<String> = (0..vocab_size)
    304
    550
                .map10(|i| {
    305
    550
                    if i == 0 {
    306
    10
                        "<unk>".to_string()
    307
                    } else {
    308
    540
                        format!("token{i}")
    309
                    }
    310
    550
                })
    311
    10
                .collect();
    312
    10
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?0;
    313
    314
    10
            Ok((model, tokenizer))
    315
    10
        }
    316
    317
        #[test]
    318
    1
        fn test_cache_key_creation() {
    319
    1
            let key = CacheKey::new("model1".to_string());
    320
    1
            assert_eq!(key.id, "model1");
    321
    1
        }
    322
    323
        #[test]
    324
    1
        fn test_cache_key_from_config() {
    325
    1
            let config = ModelConfig {
    326
    1
                vocab_size: 100,
    327
    1
                hidden_dim: 32,
    328
    1
                num_heads: 2,
    329
    1
                num_layers: 4,
    330
    1
                intermediate_dim: 64,
    331
    1
                eps: 1e-5,
    332
    1
            };
    333
    1
            let key = CacheKey::from_config(&config);
    334
    1
            assert_eq!(key.id, "v100_h32_n2_l4_i64");
    335
    1
        }
    336
    337
        #[test]
    338
    1
        fn test_cache_creation() {
    339
    1
            let cache = ModelCache::new(10);
    340
    1
            assert_eq!(cache.capacity, 10);
    341
    1
            assert_eq!(cache.len(), 0);
    342
    1
            assert!(cache.is_empty());
    343
    1
        }
    344
    345
        #[test]
    346
    1
        fn test_cache_hit() {
    347
    1
            let cache = ModelCache::new(10);
    348
    1
            let key = CacheKey::new("test".to_string());
    349
    350
            // First access - cache miss
    351
    1
            let result1 = cache.get_or_load(&key, || create_test_model(50));
    352
    1
            assert!(result1.is_ok());
    353
    354
    1
            let metrics1 = cache.metrics();
    355
    1
            assert_eq!(metrics1.hits, 0);
    356
    1
            assert_eq!(metrics1.misses, 1);
    357
    1
            assert_eq!(metrics1.size, 1);
    358
    359
            // Second access - cache hit
    360
    1
            let result2 = cache.get_or_load(&key, || create_test_model0(50));
    361
    1
            assert!(result2.is_ok());
    362
    363
    1
            let metrics2 = cache.metrics();
    364
    1
            assert_eq!(metrics2.hits, 1);
    365
    1
            assert_eq!(metrics2.misses, 1);
    366
    1
            assert_eq!(metrics2.size, 1);
    367
    1
        }
    368
    369
        #[test]
    370
    1
        fn test_cache_miss() {
    371
    1
            let cache = ModelCache::new(10);
    372
    1
            let key1 = CacheKey::new("model1".to_string());
    373
    1
            let key2 = CacheKey::new("model2".to_string());
    374
    375
    1
            cache
    376
    1
                .get_or_load(&key1, || create_test_model(50))
    377
    1
                .expect("test");
    378
    1
            cache
    379
    1
                .get_or_load(&key2, || create_test_model(60))
    380
    1
                .expect("test");
    381
    382
    1
            let metrics = cache.metrics();
    383
    1
            assert_eq!(metrics.hits, 0);
    384
    1
            assert_eq!(metrics.misses, 2);
    385
    1
            assert_eq!(metrics.size, 2);
    386
    1
        }
    387
    388
        #[test]
    389
    1
        fn test_lru_eviction() {
    390
    1
            let cache = ModelCache::new(2); // Small capacity
    391
    392
    1
            let key1 = CacheKey::new("model1".to_string());
    393
    1
            let key2 = CacheKey::new("model2".to_string());
    394
    1
            let key3 = CacheKey::new("model3".to_string());
    395
    396
            // Fill cache to capacity
    397
    1
            cache
    398
    1
                .get_or_load(&key1, || create_test_model(50))
    399
    1
                .expect("test");
    400
    1
            cache
    401
    1
                .get_or_load(&key2, || create_test_model(60))
    402
    1
                .expect("test");
    403
    404
    1
            assert_eq!(cache.len(), 2);
    405
    406
            // Add third model - should evict LRU (model1)
    407
    1
            cache
    408
    1
                .get_or_load(&key3, || create_test_model(70))
    409
    1
                .expect("test");
    410
    411
    1
            assert_eq!(cache.len(), 2);
    412
    1
            let metrics = cache.metrics();
    413
    1
            assert_eq!(metrics.evictions, 1);
    414
    1
        }
    415
    416
        #[test]
    417
    1
        fn test_cache_clear() {
    418
    1
            let cache = ModelCache::new(10);
    419
    420
    1
            let key1 = CacheKey::new("model1".to_string());
    421
    1
            let key2 = CacheKey::new("model2".to_string());
    422
    423
    1
            cache
    424
    1
                .get_or_load(&key1, || create_test_model(50))
    425
    1
                .expect("test");
    426
    1
            cache
    427
    1
                .get_or_load(&key2, || create_test_model(60))
    428
    1
                .expect("test");
    429
    430
    1
            assert_eq!(cache.len(), 2);
    431
    432
    1
            cache.clear();
    433
    434
    1
            assert_eq!(cache.len(), 0);
    435
    1
            assert!(cache.is_empty());
    436
    1
        }
    437
    438
        #[test]
    439
    1
        fn test_hit_rate_calculation() {
    440
    1
            let mut metrics = CacheMetrics::default();
    441
    1
            assert!(metrics.hit_rate().abs() < 0.1);
    442
    443
    1
            metrics.hits = 7;
    444
    1
            metrics.misses = 3;
    445
    1
            assert!((metrics.hit_rate() - 70.0).abs() < 0.1);
    446
    447
    1
            metrics.hits = 100;
    448
    1
            metrics.misses = 0;
    449
    1
            assert!((metrics.hit_rate() - 100.0).abs() < 0.1);
    450
    1
        }
    451
    452
        #[test]
    453
    1
        fn test_cache_entry_access_tracking() {
    454
    1
            let (model, tokenizer) = create_test_model(50).expect("test");
    455
    1
            let mut entry = CacheEntry::new(model, tokenizer);
    456
    457
    1
            assert_eq!(entry.access_count, 0);
    458
    459
    1
            entry.record_access();
    460
    1
            assert_eq!(entry.access_count, 1);
    461
    462
    1
            entry.record_access();
    463
    1
            assert_eq!(entry.access_count, 2);
    464
    1
        }
    465
    466
        #[test]
    467
    1
        fn test_concurrent_access() {
    468
            use std::thread;
    469
    470
    1
            let cache = Arc::new(ModelCache::new(10));
    471
    1
            let key = CacheKey::new("concurrent".to_string());
    472
    473
            // Pre-populate cache to avoid race condition on first load
    474
    1
            cache
    475
    1
                .get_or_load(&key, || create_test_model(50))
    476
    1
                .expect("test");
    477
    478
            // Reset metrics to get clean counts
    479
    1
            let initial_metrics = cache.metrics();
    480
    1
            let initial_hits = initial_metrics.hits;
    481
    1
            let initial_misses = initial_metrics.misses;
    482
    483
            // Spawn multiple threads accessing the same key (all should hit)
    484
    1
            let handles: Vec<_> = (0..5)
    485
    5
                .map1(|_| {
    486
    5
                    let cache = cache.clone();
    487
    5
                    let key = key.clone();
    488
    5
                    thread::spawn(move || {
    489
    5
                        cache
    490
    5
                            .get_or_load(&key, || create_test_model0(50))
    491
    5
                            .expect("test");
    492
    5
                    })
    493
    5
                })
    494
    1
                .collect();
    495
    496
    6
            for handle5 in handles {
    497
    5
                handle.join().expect("test");
    498
    5
            }
    499
    500
            // All threads should have hit the cache
    501
    1
            let metrics = cache.metrics();
    502
    1
            assert_eq!(metrics.size, 1);
    503
    1
            assert_eq!(metrics.hits, initial_hits + 5); // Exactly 5 hits
    504
    1
            assert_eq!(metrics.misses, initial_misses); // No new misses
    505
    1
        }
    506
    }