## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/inference/norm.rs

    Line
    Count
    Source
    1
    //! Normalization and position encoding operations
    2
    //!
    3
    //! Provides layer normalization, RMS normalization, and rotary position embeddings
    4
    //! used in transformer inference.
    5
    //!
    6
    //! ## Normalization Functions
    7
    //!
    8
    //! - [`simd_layer_norm`] - Standard layer normalization with mean and variance
    9
    //! - [`simd_rms_norm`] - RMS normalization (faster, used in LLaMA/Mistral)
    10
    //!
    11
    //! ## Position Encoding
    12
    //!
    13
    //! - [`apply_rope`] - Rotary Position Embeddings (RoPE)
    14
    15
    /// SIMD-accelerated layer normalization
    16
    ///
    17
    /// LayerNorm(x) = (x - mean) / sqrt(var + eps) * weight + bias
    18
    ///
    19
    /// # Arguments
    20
    ///
    21
    /// * `input` - Input vector to normalize
    22
    /// * `weight` - Scale parameters (gamma)
    23
    /// * `bias` - Optional shift parameters (beta)
    24
    /// * `eps` - Small constant for numerical stability (typically 1e-5)
    25
    ///
    26
    /// # Example
    27
    ///
    28
    /// ```
    29
    /// use realizar::inference::simd_layer_norm;
    30
    ///
    31
    /// let input = vec![1.0, 2.0, 3.0, 4.0];
    32
    /// let weight = vec![1.0, 1.0, 1.0, 1.0];
    33
    /// let output = simd_layer_norm(&input, &weight, None, 1e-5);
    34
    ///
    35
    /// // Output should have mean ≈ 0 and std ≈ 1
    36
    /// let mean: f32 = output.iter().sum::<f32>() / output.len() as f32;
    37
    /// assert!(mean.abs() < 1e-5);
    38
    /// ```
    39
    #[must_use]
    40
    11
    pub fn simd_layer_norm(input: &[f32], weight: &[f32], bias: Option<&[f32]>, eps: f32) -> Vec<f32> {
    41
    11
        let n = input.len();
    42
    11
        if n == 0 {
    43
    1
            return Vec::new();
    44
    10
        }
    45
    46
        // Compute mean
    47
    10
        let mean: f32 = input.iter().sum::<f32>() / n as f32;
    48
    49
        // Compute variance
    50
    37
        let var10: f3210 = input10.iter10().map10(|x| (x - mean).powi(2)).sum10::<f32>() / n as f3210;
    51
    52
        // Normalize
    53
    10
        let inv_std = 1.0 / (var + eps).sqrt();
    54
    37
        let mut output10: Vec<f32>10 = input10.iter10().map10(|x| (x - mean) * inv_std).collect10();
    55
    56
        // Apply affine transformation
    57
    37
        for (i, out) in output.iter_mut()10.enumerate10() {
    58
    37
            *out *= weight[i];
    59
    37
            if let Some(b4) = bias {
    60
    4
                *out += b[i];
    61
    33
            }
    62
        }
    63
    64
    10
        output
    65
    11
    }
    66
    67
    /// SIMD-accelerated RMS normalization
    68
    ///
    69
    /// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
    70
    ///
    71
    /// RMS normalization is faster than LayerNorm as it doesn't require
    72
    /// computing the mean. Used in LLaMA, Mistral, and other modern LLMs.
    73
    ///
    74
    /// # Arguments
    75
    ///
    76
    /// * `input` - Input vector to normalize
    77
    /// * `weight` - Scale parameters
    78
    /// * `eps` - Small constant for numerical stability (typically 1e-5)
    79
    ///
    80
    /// # Example
    81
    ///
    82
    /// ```
    83
    /// use realizar::inference::simd_rms_norm;
    84
    ///
    85
    /// let input = vec![1.0, 2.0, 3.0];
    86
    /// let weight = vec![1.0, 1.0, 1.0];
    87
    /// let output = simd_rms_norm(&input, &weight, 1e-5);
    88
    ///
    89
    /// // RMS of [1,2,3] ≈ 2.16, so normalized ≈ [0.46, 0.93, 1.39]
    90
    /// assert!((output[0] - 0.4629).abs() < 0.01);
    91
    /// ```
    92
    #[must_use]
    93
    12
    pub fn simd_rms_norm(input: &[f32], weight: &[f32], eps: f32) -> Vec<f32> {
    94
    12
        let n = input.len();
    95
    12
        if n == 0 {
    96
    1
            return Vec::new();
    97
    11
        }
    98
    99
        // Compute RMS
    100
    33
        let sum_sq11: f3211 = input11.iter11().map11(|x| x * x).sum11();
    101
    11
        let rms = (sum_sq / n as f32 + eps).sqrt();
    102
    11
        let inv_rms = 1.0 / rms;
    103
    104
        // Normalize and scale
    105
    11
        input
    106
    11
            .iter()
    107
    11
            .zip(weight.iter())
    108
    33
            .map11(|(x, w)| x * inv_rms * w)
    109
    11
            .collect()
    110
    12
    }
    111
    112
    /// Apply rotary position embeddings (RoPE)
    113
    ///
    114
    /// RoPE encodes position information by rotating pairs of dimensions.
    115
    /// This enables relative position encoding that generalizes to longer sequences.
    116
    ///
    117
    /// # Arguments
    118
    ///
    119
    /// * `x` - Mutable slice to apply RoPE to [hidden_dim]
    120
    /// * `hidden_dim` - Total hidden dimension (must equal x.len())
    121
    /// * `num_heads` - Number of attention heads
    122
    /// * `position` - Token position in sequence (0-indexed)
    123
    /// * `theta` - Base frequency (typically 10000.0)
    124
    ///
    125
    /// # Algorithm
    126
    ///
    127
    /// For each head and each pair of dimensions (i, i + d/2):
    128
    /// ```text
    129
    /// freq = 1 / theta^(2i/d)
    130
    /// angle = position * freq
    131
    /// x[i]     = x[i] * cos(angle) - x[i+d/2] * sin(angle)
    132
    /// x[i+d/2] = x[i] * sin(angle) + x[i+d/2] * cos(angle)
    133
    /// ```
    134
    ///
    135
    /// # Example
    136
    ///
    137
    /// ```
    138
    /// use realizar::inference::apply_rope;
    139
    ///
    140
    /// let mut x = vec![1.0; 64];  // 64 hidden dim
    141
    /// apply_rope(&mut x, 64, 4, 0, 10000.0);  // Position 0
    142
    ///
    143
    /// // At position 0, rotations are identity (angle = 0)
    144
    /// assert!((x[0] - 1.0).abs() < 1e-5);
    145
    /// ```
    146
    17
    pub fn apply_rope(x: &mut [f32], hidden_dim: usize, num_heads: usize, position: usize, theta: f32) {
    147
    17
        let head_dim = hidden_dim / num_heads;
    148
    17
        let half_dim = head_dim / 2;
    149
    150
    52
        for h in 0..num_heads17 {
    151
    52
            let head_offset = h * head_dim;
    152
    153
    120
            for i in 0..half_dim52 {
    154
    120
                let freq = 1.0 / theta.powf(2.0 * i as f32 / head_dim as f32);
    155
    120
                let angle = position as f32 * freq;
    156
    120
                let cos_val = angle.cos();
    157
    120
                let sin_val = angle.sin();
    158
    120
    159
    120
                let idx0 = head_offset + i;
    160
    120
                let idx1 = head_offset + i + half_dim;
    161
    120
    162
    120
                let x0 = x[idx0];
    163
    120
                let x1 = x[idx1];
    164
    120
    165
    120
                x[idx0] = x0 * cos_val - x1 * sin_val;
    166
    120
                x[idx1] = x0 * sin_val + x1 * cos_val;
    167
    120
            }
    168
        }
    169
    17
    }
    170
    171
    // ============================================================================
    172
    // EXTREME TDD: Comprehensive Tests
    173
    // ============================================================================
    174
    175
    #[cfg(test)]
    176
    mod tests {
    177
        use super::*;
    178
    179
        // ------------------------------------------------------------------------
    180
        // simd_layer_norm Tests
    181
        // ------------------------------------------------------------------------
    182
    183
        #[test]
    184
    1
        fn test_layer_norm_basic() {
    185
    1
            let input = vec![1.0, 2.0, 3.0, 4.0];
    186
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    187
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    188
    189
            // Output should have mean ≈ 0
    190
    1
            let mean: f32 = output.iter().sum::<f32>() / output.len() as f32;
    191
    1
            assert!(mean.abs() < 1e-5, "Mean should be ~0, got {}"0, mean);
    192
    193
            // Output should have std ≈ 1
    194
    4
            let var1: f321 = output.iter()1.map1(|x| (x - mean).powi(2)).sum1::<f32>() / output.len() as f321;
    195
    1
            let std = var.sqrt();
    196
    1
            assert!((std - 1.0).abs() < 0.01, "Std should be ~1, got {}"0, std);
    197
    1
        }
    198
    199
        #[test]
    200
    1
        fn test_layer_norm_with_scale() {
    201
    1
            let input = vec![1.0, 2.0, 3.0, 4.0];
    202
    1
            let weight = vec![2.0, 2.0, 2.0, 2.0];
    203
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    204
    205
            // With scale=2, std should be ~2
    206
    1
            let mean: f32 = output.iter().sum::<f32>() / output.len() as f32;
    207
    4
            let var1: f321 = output.iter()1.map1(|x| (x - mean).powi(2)).sum1::<f32>() / output.len() as f321;
    208
    1
            let std = var.sqrt();
    209
    1
            assert!((std - 2.0).abs() < 0.01, "Std should be ~2, got {}"0, std);
    210
    1
        }
    211
    212
        #[test]
    213
    1
        fn test_layer_norm_with_bias() {
    214
    1
            let input = vec![1.0, 2.0, 3.0, 4.0];
    215
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    216
    1
            let bias = vec![5.0, 5.0, 5.0, 5.0];
    217
    1
            let output = simd_layer_norm(&input, &weight, Some(&bias), 1e-5);
    218
    219
            // With bias=5, mean should be ~5
    220
    1
            let mean: f32 = output.iter().sum::<f32>() / output.len() as f32;
    221
    1
            assert!((mean - 5.0).abs() < 0.01, "Mean should be ~5, got {}"0, mean);
    222
    1
        }
    223
    224
        #[test]
    225
    1
        fn test_layer_norm_empty() {
    226
    1
            let input: Vec<f32> = vec![];
    227
    1
            let weight: Vec<f32> = vec![];
    228
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    229
    1
            assert!(output.is_empty());
    230
    1
        }
    231
    232
        #[test]
    233
    1
        fn test_layer_norm_single_element() {
    234
    1
            let input = vec![5.0];
    235
    1
            let weight = vec![1.0];
    236
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    237
            // Single element: mean=5, var=0, so normalized = 0
    238
    1
            assert!((output[0]).abs() < 1e-3);
    239
    1
        }
    240
    241
        #[test]
    242
    1
        fn test_layer_norm_uniform_input() {
    243
    1
            let input = vec![3.0, 3.0, 3.0, 3.0];
    244
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    245
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    246
            // Uniform input: mean=3, var=0+eps, normalized ≈ 0
    247
    5
            for &x4 in &output {
    248
    4
                assert!(x.abs() < 0.1);
    249
            }
    250
    1
        }
    251
    252
        #[test]
    253
    1
        fn test_layer_norm_negative_values() {
    254
    1
            let input = vec![-2.0, -1.0, 1.0, 2.0];
    255
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    256
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    257
    258
            // Mean should be 0, values should preserve sign relationship
    259
    1
            assert!(output[0] < output[1]);
    260
    1
            assert!(output[1] < output[2]);
    261
    1
            assert!(output[2] < output[3]);
    262
    1
        }
    263
    264
        #[test]
    265
    1
        fn test_layer_norm_large_values() {
    266
    1
            let input = vec![1000.0, 2000.0, 3000.0, 4000.0];
    267
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    268
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    269
    270
            // Should still have mean ≈ 0 and std ≈ 1
    271
    1
            let mean: f32 = output.iter().sum::<f32>() / output.len() as f32;
    272
    1
            assert!(mean.abs() < 1e-4);
    273
    1
        }
    274
    275
        // ------------------------------------------------------------------------
    276
        // simd_rms_norm Tests
    277
        // ------------------------------------------------------------------------
    278
    279
        #[test]
    280
    1
        fn test_rms_norm_basic() {
    281
    1
            let input = vec![1.0, 2.0, 3.0];
    282
    1
            let weight = vec![1.0, 1.0, 1.0];
    283
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    284
    285
            // RMS = sqrt((1 + 4 + 9) / 3) = sqrt(14/3) ≈ 2.16
    286
    1
            let rms = (14.0_f32 / 3.0).sqrt();
    287
    3
            let expected1: Vec<f32>1 = input.iter()1.map1(|x| x / rms).collect1();
    288
    289
    3
            for (out, exp) in output.iter()1.zip1(expected.iter()1) {
    290
    3
                assert!((out - exp).abs() < 1e-5);
    291
            }
    292
    1
        }
    293
    294
        #[test]
    295
    1
        fn test_rms_norm_with_scale() {
    296
    1
            let input = vec![1.0, 2.0, 3.0];
    297
    1
            let weight = vec![2.0, 2.0, 2.0];
    298
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    299
    300
    1
            let rms = (14.0_f32 / 3.0).sqrt();
    301
    3
            let expected1: Vec<f32>1 = input.iter()1.map1(|x| x / rms * 2.0).collect1();
    302
    303
    3
            for (out, exp) in output.iter()1.zip1(expected.iter()1) {
    304
    3
                assert!((out - exp).abs() < 1e-5);
    305
            }
    306
    1
        }
    307
    308
        #[test]
    309
    1
        fn test_rms_norm_empty() {
    310
    1
            let input: Vec<f32> = vec![];
    311
    1
            let weight: Vec<f32> = vec![];
    312
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    313
    1
            assert!(output.is_empty());
    314
    1
        }
    315
    316
        #[test]
    317
    1
        fn test_rms_norm_single_element() {
    318
    1
            let input = vec![5.0];
    319
    1
            let weight = vec![1.0];
    320
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    321
            // RMS of [5] = 5, so output = 5/5 = 1
    322
    1
            assert!((output[0] - 1.0).abs() < 1e-5);
    323
    1
        }
    324
    325
        #[test]
    326
    1
        fn test_rms_norm_unit_vector() {
    327
            // For input [1, 0, 0] with weight [1, 1, 1]
    328
            // RMS = sqrt(mean(x^2)) = sqrt(1/3)
    329
            // output = input / RMS * weight = [sqrt(3), 0, 0]
    330
    1
            let input = vec![1.0, 0.0, 0.0];
    331
    1
            let weight = vec![1.0, 1.0, 1.0];
    332
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    333
    334
    1
            let expected = 3.0_f32.sqrt(); // sqrt(3) ≈ 1.732
    335
    1
            assert!(
    336
    1
                (output[0] - expected).abs() < 1e-4,
    337
    0
                "Expected {}, got {}",
    338
                expected,
    339
    0
                output[0]
    340
            );
    341
    1
            assert!(output[1].abs() < 1e-5);
    342
    1
            assert!(output[2].abs() < 1e-5);
    343
    1
        }
    344
    345
        #[test]
    346
    1
        fn test_rms_norm_zeros() {
    347
    1
            let input = vec![0.0, 0.0, 0.0];
    348
    1
            let weight = vec![1.0, 1.0, 1.0];
    349
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    350
    351
            // RMS = sqrt(eps), output = 0 / sqrt(eps) = 0
    352
    4
            for &x3 in &output {
    353
    3
                assert!(x.abs() < 1e-2);
    354
            }
    355
    1
        }
    356
    357
        #[test]
    358
    1
        fn test_rms_norm_negative_values() {
    359
    1
            let input = vec![-3.0, 4.0];
    360
    1
            let weight = vec![1.0, 1.0];
    361
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    362
    363
            // RMS = sqrt((9 + 16) / 2) = sqrt(12.5) ≈ 3.54
    364
    1
            let rms = (12.5_f32).sqrt();
    365
    1
            assert!((output[0] - (-3.0 / rms)).abs() < 1e-5);
    366
    1
            assert!((output[1] - (4.0 / rms)).abs() < 1e-5);
    367
    1
        }
    368
    369
        #[test]
    370
    1
        fn test_rms_norm_preserves_direction() {
    371
    1
            let input = vec![3.0, 4.0]; // 3-4-5 right triangle
    372
    1
            let weight = vec![1.0, 1.0];
    373
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    374
    375
            // Direction should be preserved: output[1]/output[0] = 4/3
    376
    1
            let ratio = output[1] / output[0];
    377
    1
            assert!((ratio - 4.0 / 3.0).abs() < 1e-5);
    378
    1
        }
    379
    380
        // ------------------------------------------------------------------------
    381
        // apply_rope Tests
    382
        // ------------------------------------------------------------------------
    383
    384
        #[test]
    385
    1
        fn test_rope_position_zero() {
    386
    1
            let mut x = vec![1.0, 2.0, 3.0, 4.0]; // 4 hidden, 1 head, head_dim=4
    387
    1
            let original = x.clone();
    388
    1
            apply_rope(&mut x, 4, 1, 0, 10000.0);
    389
    390
            // At position 0, angle = 0, cos(0) = 1, sin(0) = 0
    391
            // So output should equal input
    392
    4
            for (out, orig) in x.iter()1.zip1(original.iter()1) {
    393
    4
                assert!((out - orig).abs() < 1e-5);
    394
            }
    395
    1
        }
    396
    397
        #[test]
    398
    1
        fn test_rope_rotation_property() {
    399
    1
            let mut x = vec![1.0, 0.0, 0.0, 1.0]; // 4 hidden, 1 head
    400
    1
            apply_rope(&mut x, 4, 1, 1, 10000.0);
    401
    402
            // After rotation, magnitude should be preserved for each pair
    403
    1
            let mag0 = (x[0] * x[0] + x[2] * x[2]).sqrt();
    404
    1
            let mag1 = (x[1] * x[1] + x[3] * x[3]).sqrt();
    405
    406
    1
            assert!((mag0 - 1.0).abs() < 1e-5, "Magnitude of pair 0 should be 1"0);
    407
    1
            assert!((mag1 - 1.0).abs() < 1e-5, "Magnitude of pair 1 should be 1"0);
    408
    1
        }
    409
    410
        #[test]
    411
    1
        fn test_rope_multiple_heads() {
    412
    1
            let mut x = vec![1.0; 8]; // 8 hidden, 2 heads, head_dim = 4
    413
    1
            let original = x.clone();
    414
    1
            apply_rope(&mut x, 8, 2, 0, 10000.0);
    415
    416
            // At position 0, should be unchanged
    417
    8
            for (out, orig) in x.iter()1.zip1(original.iter()1) {
    418
    8
                assert!((out - orig).abs() < 1e-5);
    419
            }
    420
    1
        }
    421
    422
        #[test]
    423
    1
        fn test_rope_different_positions() {
    424
    1
            let mut x1 = vec![1.0; 4];
    425
    1
            let mut x2 = vec![1.0; 4];
    426
    427
    1
            apply_rope(&mut x1, 4, 1, 0, 10000.0);
    428
    1
            apply_rope(&mut x2, 4, 1, 1, 10000.0);
    429
    430
            // Different positions should give different results
    431
    1
            assert!((x1[0] - x2[0]).abs() > 1e-6 || (0x1[1]0 - x2[1]).abs() > 1e-6);
    432
    1
        }
    433
    434
        #[test]
    435
    1
        fn test_rope_theta_scaling() {
    436
    1
            let mut x1 = vec![1.0; 4];
    437
    1
            let mut x2 = vec![1.0; 4];
    438
    439
    1
            apply_rope(&mut x1, 4, 1, 10, 10000.0);
    440
    1
            apply_rope(&mut x2, 4, 1, 10, 1000.0);
    441
    442
            // Different theta affects higher frequency dimensions (i > 0)
    443
            // For i=0, freq = 1/theta^0 = 1 (same regardless of theta)
    444
            // For i=1, freq = 1/theta^(2/head_dim) which differs by theta
    445
            // So check x[1] or x[3] (the second pair uses i=1)
    446
    1
            assert!(
    447
    1
                (x1[1] - x2[1]).abs() > 1e-5 || (0x1[3]0 - x2[3]).abs() > 1e-5,
    448
    0
                "Different theta should give different results for non-zero frequency indices"
    449
            );
    450
    1
        }
    451
    452
        #[test]
    453
    1
        fn test_rope_large_position() {
    454
    1
            let mut x = vec![1.0, 2.0, 3.0, 4.0];
    455
    1
            apply_rope(&mut x, 4, 1, 1000, 10000.0);
    456
    457
            // Results should be finite
    458
    5
            for &val4 in &x {
    459
    4
                assert!(val.is_finite());
    460
            }
    461
    1
        }
    462
    463
        #[test]
    464
    1
        fn test_rope_eight_heads() {
    465
    1
            let hidden_dim = 64;
    466
    1
            let num_heads = 8;
    467
    1
            let mut x = vec![0.5; hidden_dim];
    468
    469
    1
            apply_rope(&mut x, hidden_dim, num_heads, 5, 10000.0);
    470
    471
            // All values should be finite
    472
    65
            for &val64 in &x {
    473
    64
                assert!(val.is_finite());
    474
            }
    475
    1
        }
    476
    477
        #[test]
    478
    1
        fn test_rope_preserves_length() {
    479
    1
            let mut x = vec![3.0, 4.0, 0.0, 0.0]; // pairs: (3,0), (4,0)
    480
    1
            apply_rope(&mut x, 4, 1, 1, 10000.0);
    481
    482
    1
            assert_eq!(x.len(), 4);
    483
    1
        }
    484
    485
        // ------------------------------------------------------------------------
    486
        // Integration Tests
    487
        // ------------------------------------------------------------------------
    488
    489
        #[test]
    490
    1
        fn test_norm_then_rope() {
    491
    1
            let input = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
    492
    1
            let weight = vec![1.0; 8];
    493
    494
            // First normalize
    495
    1
            let normalized = simd_rms_norm(&input, &weight, 1e-5);
    496
    497
            // Then apply RoPE
    498
    1
            let mut output = normalized;
    499
    1
            apply_rope(&mut output, 8, 2, 5, 10000.0);
    500
    501
            // Results should be finite
    502
    9
            for &val8 in &output {
    503
    8
                assert!(val.is_finite());
    504
            }
    505
    1
        }
    506
    507
        #[test]
    508
    1
        fn test_layer_norm_vs_rms_norm() {
    509
    1
            let input = vec![1.0, 2.0, 3.0, 4.0];
    510
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    511
    512
    1
            let ln_output = simd_layer_norm(&input, &weight, None, 1e-5);
    513
    1
            let rms_output = simd_rms_norm(&input, &weight, 1e-5);
    514
    515
            // LayerNorm centers (mean=0), RMSNorm doesn't
    516
    1
            let ln_mean: f32 = ln_output.iter().sum::<f32>() / 4.0;
    517
    1
            let rms_mean: f32 = rms_output.iter().sum::<f32>() / 4.0;
    518
    519
    1
            assert!(ln_mean.abs() < 1e-5, "LayerNorm should have mean ~0"0);
    520
    1
            assert!(rms_mean.abs() > 0.1, "RMSNorm should not center"0);
    521
    1
        }
    522
    523
        // ------------------------------------------------------------------------
    524
        // Edge Cases
    525
        // ------------------------------------------------------------------------
    526
    527
        #[test]
    528
    1
        fn test_layer_norm_eps_impact() {
    529
    1
            let input = vec![0.0, 0.0, 0.0, 0.0];
    530
    1
            let weight = vec![1.0, 1.0, 1.0, 1.0];
    531
    532
            // With var=0, eps prevents division by zero
    533
    1
            let output = simd_layer_norm(&input, &weight, None, 1e-5);
    534
    5
            for &val4 in &output {
    535
    4
                assert!(val.is_finite());
    536
            }
    537
    1
        }
    538
    539
        #[test]
    540
    1
        fn test_rms_norm_eps_impact() {
    541
    1
            let input = vec![0.0, 0.0];
    542
    1
            let weight = vec![1.0, 1.0];
    543
    544
            // With sum_sq=0, eps prevents division by zero
    545
    1
            let output = simd_rms_norm(&input, &weight, 1e-5);
    546
    3
            for &val2 in &output {
    547
    2
                assert!(val.is_finite());
    548
            }
    549
    1
        }
    550
    551
        #[test]
    552
    1
        fn test_rope_half_dim_calculation() {
    553
            // Test with various head dimensions
    554
    4
            for (hidden_dim, num_heads) in [(8, 2)1, (16, 4)1, (32, 8)1, (64, 16)1] {
    555
    4
                let mut x = vec![1.0; hidden_dim];
    556
    4
                apply_rope(&mut x, hidden_dim, num_heads, 1, 10000.0);
    557
    558
                // Should not panic and should produce finite values
    559
    124
                for &val120 in &x {
    560
    120
                    assert!(val.is_finite());
    561
                }
    562
            }
    563
    1
        }
    564
    }