## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/trueno/src/vector/ops/activations.rs

    Line
    Count
    Source
    1
    //! Activation functions for Vector<f32>
    2
    //!
    3
    //! This module provides neural network activation functions optimized with
    4
    //! multi-backend SIMD support (Scalar, SSE2, AVX2, AVX-512, NEON, WASM SIMD).
    5
    //!
    6
    //! ## Activation Functions
    7
    //!
    8
    //! - [`softmax`](crate::Vector::softmax): Softmax normalization for classification
    9
    //! - [`log_softmax`](crate::Vector::log_softmax): Numerically stable log-softmax
    10
    //! - [`relu`](crate::Vector::relu): Rectified Linear Unit
    11
    //! - [`sigmoid`](crate::Vector::sigmoid): Logistic sigmoid
    12
    //! - [`leaky_relu`](crate::Vector::leaky_relu): Leaky ReLU with configurable slope
    13
    //! - [`elu`](crate::Vector::elu): Exponential Linear Unit
    14
    //! - [`gelu`](crate::Vector::gelu): Gaussian Error Linear Unit
    15
    //! - [`swish`](crate::Vector::swish): Self-gated activation (SiLU)
    16
    //! - [`hardswish`](crate::Vector::hardswish): Efficient hardware-friendly swish
    17
    //! - [`mish`](crate::Vector::mish): Self-regularizing activation
    18
    //! - [`selu`](crate::Vector::selu): Scaled Exponential Linear Unit
    19
    20
    use crate::backends::scalar::ScalarBackend;
    21
    use crate::backends::VectorBackend;
    22
    use crate::vector::Vector;
    23
    use crate::{Backend, Result, TruenoError};
    24
    25
    /// Backend dispatch macro for unary operations - centralizes platform-specific SIMD dispatch
    26
    /// to eliminate code duplication across activation functions.
    27
    ///
    28
    /// # Safety
    29
    /// The macro wraps unsafe backend calls internally, so callers don't need unsafe blocks.
    30
    macro_rules! dispatch_unary_op {
    31
        ($backend:expr, $op:ident, $input:expr, $output:expr) => {{
    32
            #[cfg(target_arch = "x86_64")]
    33
            use crate::backends::{avx2::Avx2Backend, sse2::Sse2Backend};
    34
            // SAFETY: CPU features verified at runtime before backend selection
    35
            unsafe {
    36
                match $backend {
    37
                    Backend::Scalar => ScalarBackend::$op($input, $output),
    38
                    #[cfg(target_arch = "x86_64")]
    39
                    Backend::SSE2 | Backend::AVX => Sse2Backend::$op($input, $output),
    40
                    #[cfg(target_arch = "x86_64")]
    41
                    Backend::AVX2 | Backend::AVX512 => Avx2Backend::$op($input, $output),
    42
                    #[cfg(not(target_arch = "x86_64"))]
    43
                    Backend::SSE2 | Backend::AVX | Backend::AVX2 | Backend::AVX512 => {
    44
                        ScalarBackend::$op($input, $output)
    45
                    }
    46
                    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
    47
                    Backend::NEON => {
    48
                        use crate::backends::neon::NeonBackend;
    49
                        NeonBackend::$op($input, $output)
    50
                    }
    51
                    #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))]
    52
                    Backend::NEON => ScalarBackend::$op($input, $output),
    53
                    #[cfg(target_arch = "wasm32")]
    54
                    Backend::WasmSIMD => {
    55
                        use crate::backends::wasm::WasmBackend;
    56
                        WasmBackend::$op($input, $output)
    57
                    }
    58
                    #[cfg(not(target_arch = "wasm32"))]
    59
                    Backend::WasmSIMD => ScalarBackend::$op($input, $output),
    60
                    Backend::GPU | Backend::Auto => ScalarBackend::$op($input, $output),
    61
                }
    62
            }
    63
        }};
    64
    }
    65
    66
    impl Vector<f32> {
    67
        /// Softmax activation function
    68
        ///
    69
        /// Converts a vector of real values into a probability distribution.
    70
        /// Formula: softmax(x)\[i\] = exp(x\[i\] - max(x)) / sum(exp(x\[j\] - max(x)))
    71
        ///
    72
        /// Uses the numerically stable version with max subtraction to prevent overflow.
    73
        /// The output is a probability distribution: all values in [0, 1] and sum to 1.
    74
        ///
    75
        /// This is the standard activation function for multi-class classification in neural networks.
    76
        ///
    77
        /// # Examples
    78
        ///
    79
        /// ```
    80
        /// use trueno::Vector;
    81
        ///
    82
        /// let logits = Vector::from_slice(&[1.0, 2.0, 3.0]);
    83
        /// let probs = logits.softmax()?;
    84
        ///
    85
        /// // Verify sum ≈ 1
    86
        /// let sum: f32 = probs.as_slice().iter().sum();
    87
        /// assert!((sum - 1.0).abs() < 1e-5);
    88
        ///
    89
        /// // Verify all values in [0, 1]
    90
        /// for &p in probs.as_slice() {
    91
        ///     assert!(p >= 0.0 && p <= 1.0);
    92
        /// }
    93
        /// # Ok::<(), trueno::TruenoError>(())
    94
        /// ```
    95
        ///
    96
        /// # Empty vectors
    97
        ///
    98
        /// Returns EmptyVector error for empty vectors (cannot compute softmax).
    99
    10.4k
        pub fn softmax(&self) -> Result<Self> {
    100
    10.4k
            if self.data.is_empty() {
    101
    0
                return Err(TruenoError::EmptyVector);
    102
    10.4k
            }
    103
    104
            // OpComplexity::Medium - GPU threshold: >10K elements (multi-pass overhead)
    105
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    106
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 4-368x slower, see docs/performance-analysis.md
    107
    108
            // Try GPU first for large vectors
    109
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    110
            {
    111
    10.4k
                if self.data.len() >= GPU_THRESHOLD {
    112
                    use crate::backends::gpu::GpuDevice;
    113
    0
                    if GpuDevice::is_available() {
    114
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    115
    0
                        let mut result = vec![0.0; self.data.len()];
    116
    0
                        if gpu.softmax(&self.data, &mut result).is_ok() {
    117
    0
                            return Ok(Vector::from_vec(result));
    118
    0
                        }
    119
    0
                    }
    120
    10.4k
                }
    121
            }
    122
    123
            // Scalar fallback: Multi-pass softmax for numerical stability
    124
            // Find max for numerical stability (prevents overflow in exp)
    125
    10.4k
            let max_val = self.max()?0;
    126
    127
            // Compute exp(x - max) for each element
    128
    227k
            let exp_vals10.4k: Vec<f32>10.4k = self.data.iter()10.4k.map10.4k(|&x| (x - max_val).exp()).collect10.4k();
    129
    130
            // Compute sum of exponentials
    131
    10.4k
            let sum_exp: f32 = exp_vals.iter().sum();
    132
    133
            // Normalize by sum
    134
    227k
            let data10.4k: Vec<f32>10.4k = exp_vals.iter()10.4k.map10.4k(|&e| e / sum_exp).collect10.4k();
    135
    136
    10.4k
            Ok(Vector::from_vec(data))
    137
    10.4k
        }
    138
    139
        /// Log-softmax activation function
    140
        ///
    141
        /// Computes the logarithm of the softmax function in a numerically stable way.
    142
        /// Formula: log_softmax(x)\[i\] = x\[i\] - max(x) - log(sum(exp(x\[j\] - max(x))))
    143
        ///
    144
        /// This is more numerically stable than computing log(softmax(x)) and is commonly
    145
        /// used in neural networks for computing cross-entropy loss.
    146
        ///
    147
        /// # Examples
    148
        ///
    149
        /// ```
    150
        /// use trueno::Vector;
    151
        ///
    152
        /// let logits = Vector::from_slice(&[1.0, 2.0, 3.0]);
    153
        /// let log_probs = logits.log_softmax()?;
    154
        ///
    155
        /// // Verify exp(log_softmax) = softmax
    156
        /// let probs_from_log: Vec<f32> = log_probs.as_slice().iter().map(|&x| x.exp()).collect();
    157
        /// let sum: f32 = probs_from_log.iter().sum();
    158
        /// assert!((sum - 1.0).abs() < 1e-5);
    159
        /// # Ok::<(), trueno::TruenoError>(())
    160
        /// ```
    161
        ///
    162
        /// # Empty vectors
    163
        ///
    164
        /// Returns EmptyVector error for empty vectors.
    165
    0
        pub fn log_softmax(&self) -> Result<Self> {
    166
    0
            if self.data.is_empty() {
    167
    0
                return Err(TruenoError::EmptyVector);
    168
    0
            }
    169
    170
            // OpComplexity::Medium - GPU threshold: >10K elements (multi-pass overhead)
    171
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    172
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 4-368x slower, see docs/performance-analysis.md
    173
    174
            // Try GPU first for large vectors
    175
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    176
            {
    177
    0
                if self.data.len() >= GPU_THRESHOLD {
    178
                    use crate::backends::gpu::GpuDevice;
    179
    0
                    if GpuDevice::is_available() {
    180
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    181
    0
                        let mut result = vec![0.0; self.data.len()];
    182
    0
                        if gpu.log_softmax(&self.data, &mut result).is_ok() {
    183
    0
                            return Ok(Vector::from_vec(result));
    184
    0
                        }
    185
    0
                    }
    186
    0
                }
    187
            }
    188
    189
            // Scalar fallback: Multi-pass log_softmax for numerical stability
    190
            // Find max for numerical stability
    191
    0
            let max_val = self.max()?;
    192
    193
            // Compute exp(x - max) for each element
    194
    0
            let exp_vals: Vec<f32> = self.data.iter().map(|&x| (x - max_val).exp()).collect();
    195
    196
            // Compute log of sum of exponentials
    197
    0
            let sum_exp: f32 = exp_vals.iter().sum();
    198
    0
            let log_sum_exp = sum_exp.ln();
    199
    200
            // log_softmax(x)[i] = x[i] - max - log_sum_exp
    201
    0
            let data: Vec<f32> = self
    202
    0
                .data
    203
    0
                .iter()
    204
    0
                .map(|&x| x - max_val - log_sum_exp)
    205
    0
                .collect();
    206
    207
    0
            Ok(Vector::from_vec(data))
    208
    0
        }
    209
    210
        /// ReLU (Rectified Linear Unit) activation function
    211
        ///
    212
        /// Computes the element-wise ReLU: max(0, x).
    213
        /// ReLU is one of the most widely used activation functions in neural networks.
    214
        ///
    215
        /// # Formula
    216
        ///
    217
        /// ```text
    218
        /// relu(x)[i] = max(0, x\[i\])
    219
        ///            = x\[i\]  if x\[i\] > 0
    220
        ///            = 0     otherwise
    221
        /// ```
    222
        ///
    223
        /// # Properties
    224
        ///
    225
        /// - **Non-linearity**: Introduces non-linearity while preserving linearity for positive values
    226
        /// - **Sparsity**: Produces exactly zero for negative inputs (sparse activations)
    227
        /// - **Gradient**: Derivative is 1 for positive inputs, 0 for negative (solves vanishing gradient)
    228
        /// - **Computational efficiency**: Simple max operation, no exponentials
    229
        ///
    230
        /// # Applications
    231
        ///
    232
        /// - **Deep neural networks**: Default activation for hidden layers
    233
        /// - **Convolutional networks**: Standard activation in CNNs
    234
        /// - **Feature learning**: Encourages sparse representations
    235
        ///
    236
        /// # Performance
    237
        ///
    238
        /// This operation is memory-bound. SIMD provides modest speedups since
    239
        /// the computation (comparison and selection) is simpler than memory access.
    240
        ///
    241
        /// # Errors
    242
        ///
    243
        /// Returns `EmptyVector` if the input vector is empty.
    244
        ///
    245
        /// # Examples
    246
        ///
    247
        /// ```
    248
        /// use trueno::Vector;
    249
        ///
    250
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    251
        /// let result = v.relu()?;
    252
        /// assert_eq!(result.as_slice(), &[0.0, 0.0, 0.0, 1.0, 2.0]);
    253
        /// # Ok::<(), trueno::TruenoError>(())
    254
        /// ```
    255
    0
        pub fn relu(&self) -> Result<Self> {
    256
    0
            if self.data.is_empty() {
    257
    0
                return Err(TruenoError::EmptyVector);
    258
    0
            }
    259
    260
            // OpComplexity::Low - GPU threshold: >100K elements
    261
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    262
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 2-800x slower, see docs/performance-analysis.md
    263
    264
            // Try GPU first for large vectors
    265
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    266
            {
    267
    0
                if self.data.len() >= GPU_THRESHOLD {
    268
                    use crate::backends::gpu::GpuDevice;
    269
    0
                    if GpuDevice::is_available() {
    270
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    271
    0
                        let mut result = vec![0.0; self.data.len()];
    272
    0
                        if gpu.relu(&self.data, &mut result).is_ok() {
    273
    0
                            return Ok(Vector::from_vec(result));
    274
    0
                        }
    275
    0
                    }
    276
    0
                }
    277
            }
    278
    279
    0
            let mut result = vec![0.0; self.len()];
    280
    281
            // Use parallel processing for very large arrays (reduces TLB pressure and improves cache utilization)
    282
            #[cfg(feature = "parallel")]
    283
            {
    284
                const PARALLEL_THRESHOLD: usize = 500_000; // Increased to avoid overhead at smaller sizes
    285
                const CHUNK_SIZE: usize = 65536; // 64K elements = 256KB, cache-friendly
    286
    287
                if self.len() >= PARALLEL_THRESHOLD {
    288
                    use rayon::prelude::*;
    289
    290
                    self.data
    291
                        .par_chunks(CHUNK_SIZE)
    292
                        .zip(result.par_chunks_mut(CHUNK_SIZE))
    293
                        .for_each(|(chunk_in, chunk_out)| {
    294
                            dispatch_unary_op!(self.backend, relu, chunk_in, chunk_out);
    295
                        });
    296
    297
                    return Ok(Vector::from_vec(result)); // Use from_vec to avoid extra copy
    298
                }
    299
            }
    300
    301
            // Sequential processing for small arrays or when parallel feature disabled
    302
    0
            dispatch_unary_op!(self.backend, relu, &self.data, &mut result);
    303
    304
    0
            Ok(Vector::from_vec(result)) // Use from_vec to avoid extra copy
    305
    0
        }
    306
    307
        /// Sigmoid (logistic) activation function
    308
        ///
    309
        /// Computes the element-wise sigmoid: σ(x) = 1 / (1 + e^(-x)).
    310
        /// Sigmoid is a classic activation function that squashes inputs to the range (0, 1).
    311
        ///
    312
        /// # Formula
    313
        ///
    314
        /// ```text
    315
        /// sigmoid(x)[i] = 1 / (1 + exp(-x\[i\]))
    316
        ///               = exp(x\[i\]) / (1 + exp(x\[i\]))
    317
        /// ```
    318
        ///
    319
        /// # Properties
    320
        ///
    321
        /// - **Bounded output**: Maps all inputs to (0, 1) range
    322
        /// - **Smooth**: Infinitely differentiable (C^∞)
    323
        /// - **Symmetric**: σ(-x) = 1 - σ(x)
    324
        /// - **Derivative**: σ'(x) = σ(x) * (1 - σ(x))
    325
        /// - **Interpretable**: Output can be interpreted as probability
    326
        ///
    327
        /// # Applications
    328
        ///
    329
        /// - **Binary classification**: Final layer for binary output (0 or 1)
    330
        /// - **Logistic regression**: Traditional ML algorithm
    331
        /// - **Gating mechanisms**: LSTM/GRU gates (input, forget, output)
    332
        /// - **Attention mechanisms**: Soft attention weights
    333
        ///
    334
        /// # Numerical Considerations
    335
        ///
    336
        /// For very large negative inputs (x < -50), exp(-x) overflows to infinity.
    337
        /// However, sigmoid(x) approaches 0, so we return 0 for numerical stability.
    338
        /// For very large positive inputs (x > 50), exp(-x) underflows to 0,
    339
        /// and sigmoid(x) approaches 1.
    340
        ///
    341
        /// # Performance
    342
        ///
    343
        /// This operation is compute-bound due to the exp() operation. SIMD provides
    344
        /// modest speedups, but the exponential is the bottleneck.
    345
        ///
    346
        /// # Errors
    347
        ///
    348
        /// Returns `EmptyVector` if the input vector is empty.
    349
        ///
    350
        /// # Examples
    351
        ///
    352
        /// ```
    353
        /// use trueno::Vector;
    354
        ///
    355
        /// let v = Vector::from_slice(&[-2.0, 0.0, 2.0]);
    356
        /// let result = v.sigmoid()?;
    357
        ///
    358
        /// // sigmoid(-2) ≈ 0.119, sigmoid(0) = 0.5, sigmoid(2) ≈ 0.881
    359
        /// assert!((result.as_slice()[0] - 0.119).abs() < 0.001);
    360
        /// assert!((result.as_slice()[1] - 0.5).abs() < 0.001);
    361
        /// assert!((result.as_slice()[2] - 0.881).abs() < 0.001);
    362
        /// # Ok::<(), trueno::TruenoError>(())
    363
        /// ```
    364
    0
        pub fn sigmoid(&self) -> Result<Self> {
    365
    0
            if self.data.is_empty() {
    366
    0
                return Err(TruenoError::EmptyVector);
    367
    0
            }
    368
    369
            // OpComplexity::Low - GPU threshold: >100K elements
    370
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    371
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 2-800x slower, see docs/performance-analysis.md
    372
    373
            // Try GPU first for large vectors
    374
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    375
            {
    376
    0
                if self.data.len() >= GPU_THRESHOLD {
    377
                    use crate::backends::gpu::GpuDevice;
    378
    0
                    if GpuDevice::is_available() {
    379
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    380
    0
                        let mut result = vec![0.0; self.data.len()];
    381
    0
                        if gpu.sigmoid(&self.data, &mut result).is_ok() {
    382
    0
                            return Ok(Vector::from_vec(result));
    383
    0
                        }
    384
    0
                    }
    385
    0
                }
    386
            }
    387
    388
    0
            let mut result = vec![0.0; self.len()];
    389
    390
            // Dispatch to appropriate backend
    391
    0
            dispatch_unary_op!(self.backend, sigmoid, &self.data, &mut result);
    392
    393
    0
            Ok(Vector::from_vec(result))
    394
    0
        }
    395
    396
        /// Leaky ReLU activation function
    397
        ///
    398
        /// Computes the element-wise Leaky ReLU with a configurable negative slope.
    399
        /// Leaky ReLU addresses the "dying ReLU" problem by allowing small negative values.
    400
        ///
    401
        /// # Formula
    402
        ///
    403
        /// ```text
    404
        /// leaky_relu(x, α)[i] = max(αx\[i\], x\[i\])
    405
        ///                     = x\[i\]    if x\[i\] > 0
    406
        ///                     = αx\[i\]   if x\[i\] ≤ 0
    407
        /// ```
    408
        ///
    409
        /// # Parameters
    410
        ///
    411
        /// - `negative_slope`: The slope for negative values (typically 0.01)
    412
        ///   - Must be in range [0.0, 1.0)
    413
        ///   - Common values: 0.01 (default), 0.1, 0.2
    414
        ///   - α = 0 reduces to standard ReLU
    415
        ///   - α = 1 reduces to identity function
    416
        ///
    417
        /// # Properties
    418
        ///
    419
        /// - **Fixes dying ReLU**: Neurons can't completely die (always has gradient)
    420
        /// - **Non-zero gradient**: Gradient is α for negative inputs (not zero)
    421
        /// - **Unbounded positive**: No saturation for positive values
    422
        /// - **Parameterized**: Negative slope can be tuned or learned (PReLU)
    423
        ///
    424
        /// # Applications
    425
        ///
    426
        /// - **Deep networks**: Prevents dying neurons in very deep networks
    427
        /// - **GANs**: Often used in generator and discriminator networks
    428
        /// - **Better gradient flow**: Helps with vanishing gradient problem
    429
        /// - **Empirical improvements**: Often outperforms ReLU in practice
    430
        ///
    431
        /// # Performance
    432
        ///
    433
        /// This operation is memory-bound (simple multiplication and comparison).
    434
        /// SIMD provides modest speedups.
    435
        ///
    436
        /// # Errors
    437
        ///
    438
        /// Returns `EmptyVector` if the input vector is empty.
    439
        /// Returns `InvalidInput` if negative_slope is not in [0.0, 1.0).
    440
        ///
    441
        /// # Examples
    442
        ///
    443
        /// ```
    444
        /// use trueno::Vector;
    445
        ///
    446
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    447
        /// let result = v.leaky_relu(0.01)?;
    448
        ///
    449
        /// // Negative values multiplied by 0.01, positive unchanged
    450
        /// assert_eq!(result.as_slice(), &[-0.02, -0.01, 0.0, 1.0, 2.0]);
    451
        /// # Ok::<(), trueno::TruenoError>(())
    452
        /// ```
    453
    0
        pub fn leaky_relu(&self, negative_slope: f32) -> Result<Self> {
    454
    0
            if self.data.is_empty() {
    455
    0
                return Err(TruenoError::EmptyVector);
    456
    0
            }
    457
    458
            // Validate negative_slope parameter
    459
    0
            if !(0.0..1.0).contains(&negative_slope) {
    460
    0
                return Err(TruenoError::InvalidInput(format!(
    461
    0
                    "negative_slope must be in [0.0, 1.0), got {}",
    462
    0
                    negative_slope
    463
    0
                )));
    464
    0
            }
    465
    466
            // OpComplexity::Low - GPU threshold: >100K elements
    467
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    468
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 2-800x slower, see docs/performance-analysis.md
    469
    470
            // Try GPU first for large vectors
    471
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    472
            {
    473
    0
                if self.data.len() >= GPU_THRESHOLD {
    474
                    use crate::backends::gpu::GpuDevice;
    475
    0
                    if GpuDevice::is_available() {
    476
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    477
    0
                        let mut result = vec![0.0; self.data.len()];
    478
    0
                        if gpu
    479
    0
                            .leaky_relu(&self.data, &mut result, negative_slope)
    480
    0
                            .is_ok()
    481
                        {
    482
    0
                            return Ok(Vector::from_vec(result));
    483
    0
                        }
    484
    0
                    }
    485
    0
                }
    486
            }
    487
    488
            // Scalar fallback: leaky_relu(x, α) = x if x > 0, αx otherwise
    489
    0
            let data: Vec<f32> = self
    490
    0
                .data
    491
    0
                .iter()
    492
    0
                .map(|&x| if x > 0.0 { x } else { negative_slope * x })
    493
    0
                .collect();
    494
    495
    0
            Ok(Vector::from_vec(data))
    496
    0
        }
    497
    498
        /// ELU (Exponential Linear Unit) activation function
    499
        ///
    500
        /// Computes the element-wise ELU with a configurable alpha parameter.
    501
        /// ELU pushes mean activations closer to zero, improving learning.
    502
        ///
    503
        /// # Formula
    504
        ///
    505
        /// ```text
    506
        /// elu(x, α)[i] = x\[i\]           if x\[i\] > 0
    507
        ///              = α(e^x\[i\] - 1)  if x\[i\] ≤ 0
    508
        /// ```
    509
        ///
    510
        /// # Parameters
    511
        ///
    512
        /// - `alpha`: Controls the saturation value for negative inputs (typically 1.0)
    513
        ///   - Must be > 0
    514
        ///   - Common value: 1.0 (original ELU paper)
    515
        ///   - Larger α → slower saturation for negative inputs
    516
        ///
    517
        /// # Properties
    518
        ///
    519
        /// - **Smooth**: Unlike ReLU/Leaky ReLU, has smooth gradients everywhere
    520
        /// - **Negative values**: Allows negative outputs (pushes mean closer to zero)
    521
        /// - **Bounded below**: Saturates to -α for very negative inputs
    522
        /// - **Unbounded above**: No saturation for positive values
    523
        /// - **Non-zero gradient**: Has gradient everywhere (no dead neurons)
    524
        ///
    525
        /// # Applications
    526
        ///
    527
        /// - **Deep networks**: Better gradient flow than ReLU
    528
        /// - **Mean activation near zero**: Reduces internal covariate shift
    529
        /// - **Noise robustness**: Smooth activation helps with noisy gradients
    530
        /// - **Empirical improvements**: Often outperforms ReLU and Leaky ReLU
    531
        ///
    532
        /// # Performance
    533
        ///
    534
        /// This operation is compute-bound due to exp() for negative values.
    535
        /// More expensive than ReLU/Leaky ReLU but provides better properties.
    536
        ///
    537
        /// # Errors
    538
        ///
    539
        /// Returns `EmptyVector` if the input vector is empty.
    540
        /// Returns `InvalidInput` if alpha <= 0.
    541
        ///
    542
        /// # Examples
    543
        ///
    544
        /// ```
    545
        /// use trueno::Vector;
    546
        ///
    547
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    548
        /// let result = v.elu(1.0)?;
    549
        ///
    550
        /// // Negative values: α(e^x - 1), positive unchanged
    551
        /// // elu(-2, 1) ≈ -0.865, elu(-1, 1) ≈ -0.632
    552
        /// assert!((result.as_slice()[0] - (-0.865)).abs() < 0.01);
    553
        /// assert!((result.as_slice()[1] - (-0.632)).abs() < 0.01);
    554
        /// assert_eq!(result.as_slice()[2], 0.0);
    555
        /// assert_eq!(result.as_slice()[3], 1.0);
    556
        /// assert_eq!(result.as_slice()[4], 2.0);
    557
        /// # Ok::<(), trueno::TruenoError>(())
    558
        /// ```
    559
    0
        pub fn elu(&self, alpha: f32) -> Result<Self> {
    560
    0
            if self.data.is_empty() {
    561
    0
                return Err(TruenoError::EmptyVector);
    562
    0
            }
    563
    564
            // Validate alpha parameter
    565
    0
            if alpha <= 0.0 {
    566
    0
                return Err(TruenoError::InvalidInput(format!(
    567
    0
                    "alpha must be > 0, got {}",
    568
    0
                    alpha
    569
    0
                )));
    570
    0
            }
    571
    572
            // OpComplexity::Low - GPU threshold: >100K elements
    573
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    574
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 2-800x slower, see docs/performance-analysis.md
    575
    576
            // Try GPU first for large vectors
    577
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    578
            {
    579
    0
                if self.data.len() >= GPU_THRESHOLD {
    580
                    use crate::backends::gpu::GpuDevice;
    581
    0
                    if GpuDevice::is_available() {
    582
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    583
    0
                        let mut result = vec![0.0; self.data.len()];
    584
    0
                        if gpu.elu(&self.data, &mut result, alpha).is_ok() {
    585
    0
                            return Ok(Vector::from_vec(result));
    586
    0
                        }
    587
    0
                    }
    588
    0
                }
    589
            }
    590
    591
            // Scalar fallback: elu(x, α) = x if x > 0, α(e^x - 1) otherwise
    592
    0
            let data: Vec<f32> = self
    593
    0
                .data
    594
    0
                .iter()
    595
    0
                .map(|&x| if x > 0.0 { x } else { alpha * (x.exp() - 1.0) })
    596
    0
                .collect();
    597
    598
    0
            Ok(Vector::from_vec(data))
    599
    0
        }
    600
    601
        /// GELU (Gaussian Error Linear Unit) activation function
    602
        ///
    603
        /// Computes the element-wise GELU activation using the tanh approximation.
    604
        /// GELU is the activation function used in transformers (BERT, GPT, etc.).
    605
        ///
    606
        /// # Formula
    607
        ///
    608
        /// ```text
    609
        /// gelu(x) ≈ 0.5 * x * (1 + tanh(√(2/π) * (x + 0.044715 * x³)))
    610
        /// ```
    611
        ///
    612
        /// This is the tanh approximation which is faster than the exact form
    613
        /// involving the error function (erf).
    614
        ///
    615
        /// # Properties
    616
        ///
    617
        /// - **Smooth**: Infinitely differentiable everywhere
    618
        /// - **Non-monotonic**: Unlike ReLU variants, has slight non-monotonicity near zero
    619
        /// - **Stochastic regularizer**: Can be viewed as adaptive dropout
    620
        /// - **Zero-centered**: Mean activation close to zero
    621
        /// - **Bounded below**: Approaches 0 as x → -∞
    622
        /// - **Unbounded above**: Linear growth for large positive x
    623
        ///
    624
        /// # Applications
    625
        ///
    626
        /// - **Transformers**: BERT, GPT-2, GPT-3, GPT-4 (default activation)
    627
        /// - **Vision transformers**: ViT, DINO, MAE
    628
        /// - **Modern architectures**: State-of-the-art NLP and vision models
    629
        /// - **Better than ReLU**: Empirically outperforms ReLU in many tasks
    630
        ///
    631
        /// # Performance
    632
        ///
    633
        /// This operation is compute-intensive (tanh, x³ calculations).
    634
        /// More expensive than ReLU but comparable to ELU.
    635
        ///
    636
        /// # Errors
    637
        ///
    638
        /// Returns `EmptyVector` if the input vector is empty.
    639
        ///
    640
        /// # Examples
    641
        ///
    642
        /// ```
    643
        /// use trueno::Vector;
    644
        ///
    645
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    646
        /// let result = v.gelu()?;
    647
        ///
    648
        /// // GELU is smooth and non-monotonic near zero
    649
        /// assert!(result.as_slice()[0] < 0.0); // Negative inputs → small negative outputs
    650
        /// assert_eq!(result.as_slice()[2], 0.0); // gelu(0) = 0
    651
        /// assert!(result.as_slice()[4] > 1.5); // Large positive → ~linear
    652
        /// # Ok::<(), trueno::TruenoError>(())
    653
        /// ```
    654
    0
        pub fn gelu(&self) -> Result<Self> {
    655
    0
            if self.data.is_empty() {
    656
    0
                return Err(TruenoError::EmptyVector);
    657
    0
            }
    658
    659
            // OpComplexity::Low - GPU threshold: >100K elements
    660
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    661
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 2-800x slower, see docs/performance-analysis.md
    662
    663
            // Try GPU first for large vectors
    664
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    665
            {
    666
    0
                if self.data.len() >= GPU_THRESHOLD {
    667
                    use crate::backends::gpu::GpuDevice;
    668
    0
                    if GpuDevice::is_available() {
    669
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    670
    0
                        let mut result = vec![0.0; self.data.len()];
    671
    0
                        if gpu.gelu(&self.data, &mut result).is_ok() {
    672
    0
                            return Ok(Vector::from_vec(result));
    673
    0
                        }
    674
    0
                    }
    675
    0
                }
    676
            }
    677
    678
    0
            let mut result = vec![0.0; self.len()];
    679
    680
            // Dispatch to appropriate backend
    681
    0
            dispatch_unary_op!(self.backend, gelu, &self.data, &mut result);
    682
    683
    0
            Ok(Vector::from_vec(result))
    684
    0
        }
    685
    686
        /// Swish activation function (also known as SiLU - Sigmoid Linear Unit)
    687
        ///
    688
        /// Applies the Swish activation element-wise: swish(x) = x * sigmoid(x) = x / (1 + e^(-x)).
    689
        ///
    690
        /// Swish is a smooth, non-monotonic activation function that consistently matches or
    691
        /// outperforms ReLU in deep networks. It's used in EfficientNet, MobileNet v3, and
    692
        /// many modern architectures. The function is self-gated: it adaptively gates the
    693
        /// input based on its value.
    694
        ///
    695
        /// Properties:
    696
        /// - Smooth and differentiable everywhere
    697
        /// - Non-monotonic: has a slight "dip" for negative values
    698
        /// - swish(0) = 0
    699
        /// - swish(x) ≈ x for large positive x (linear)
    700
        /// - swish(x) ≈ 0 for large negative x
    701
        /// - Unbounded above, bounded below by ≈ -0.278 at x ≈ -1.278
    702
        ///
    703
        /// # Performance
    704
        ///
    705
        /// Compute-bound operation requiring exponential and division.
    706
        /// Future SIMD optimizations planned for Phase 9 (GPU backend).
    707
        ///
    708
        /// # Examples
    709
        ///
    710
        /// ```
    711
        /// use trueno::Vector;
    712
        ///
    713
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    714
        /// let result = v.swish()?;
    715
        ///
    716
        /// // Swish is smooth and self-gated
    717
        /// assert!(result.as_slice()[0] < 0.0); // Negative inputs → small negative outputs
    718
        /// assert_eq!(result.as_slice()[2], 0.0); // swish(0) = 0
    719
        /// assert!(result.as_slice()[4] > 1.5); // Large positive → ~linear
    720
        /// # Ok::<(), trueno::TruenoError>(())
    721
        /// ```
    722
        ///
    723
        /// # Errors
    724
        ///
    725
        /// Returns `EmptyVector` if the input vector is empty.
    726
        ///
    727
        /// # References
    728
        ///
    729
        /// - Ramachandran et al. (2017): "Searching for Activation Functions"
    730
        /// - Also known as SiLU (Sigmoid Linear Unit): Elfwing et al. (2018)
    731
    0
        pub fn swish(&self) -> Result<Self> {
    732
    0
            if self.data.is_empty() {
    733
    0
                return Err(TruenoError::EmptyVector);
    734
    0
            }
    735
    736
            // OpComplexity::Low - GPU threshold: >100K elements
    737
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    738
            const GPU_THRESHOLD: usize = usize::MAX; // GPU DISABLED - 2-800x slower, see docs/performance-analysis.md
    739
    740
            // Try GPU first for large vectors
    741
            #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))]
    742
            {
    743
    0
                if self.data.len() >= GPU_THRESHOLD {
    744
                    use crate::backends::gpu::GpuDevice;
    745
    0
                    if GpuDevice::is_available() {
    746
    0
                        let gpu = GpuDevice::new().map_err(TruenoError::InvalidInput)?;
    747
    0
                        let mut result = vec![0.0; self.data.len()];
    748
    0
                        if gpu.swish(&self.data, &mut result).is_ok() {
    749
    0
                            return Ok(Vector::from_vec(result));
    750
    0
                        }
    751
    0
                    }
    752
    0
                }
    753
            }
    754
    755
    0
            let mut result = vec![0.0; self.len()];
    756
    757
            // Dispatch to appropriate SIMD backend
    758
    0
            dispatch_unary_op!(self.backend, swish, &self.data, &mut result);
    759
    760
    0
            Ok(Vector::from_vec(result))
    761
    0
        }
    762
    763
        /// Hard Swish activation function
    764
        ///
    765
        /// Applies the hardswish activation element-wise: hardswish(x) = x * relu6(x + 3) / 6
    766
        ///
    767
        /// Hardswish is a piece-wise linear approximation to swish, designed for efficient
    768
        /// computation in mobile neural networks. It's used in MobileNetV3 and avoids the
    769
        /// expensive sigmoid computation of standard swish.
    770
        ///
    771
        /// Properties:
    772
        /// - Piece-wise linear: efficient to compute
    773
        /// - hardswish(x) = 0 for x ≤ -3
    774
        /// - hardswish(x) = x for x ≥ 3
    775
        /// - hardswish(x) = x * (x + 3) / 6 for -3 < x < 3
    776
        /// - hardswish(0) = 0
    777
        /// - Smooth transitions at boundaries
    778
        ///
    779
        /// # Performance
    780
        ///
    781
        /// More efficient than swish as it uses only multiply/divide operations
    782
        /// instead of expensive exponential functions. Ideal for inference on
    783
        /// resource-constrained devices.
    784
        ///
    785
        /// # Examples
    786
        ///
    787
        /// ```
    788
        /// use trueno::Vector;
    789
        ///
    790
        /// let v = Vector::from_slice(&[-4.0, -3.0, 0.0, 3.0, 4.0]);
    791
        /// let result = v.hardswish()?;
    792
        ///
    793
        /// // Piece-wise linear behavior
    794
        /// assert_eq!(result.as_slice()[0], 0.0); // x ≤ -3 → 0
    795
        /// assert_eq!(result.as_slice()[1], 0.0); // x = -3 → 0
    796
        /// assert_eq!(result.as_slice()[2], 0.0); // x = 0 → 0
    797
        /// assert_eq!(result.as_slice()[3], 3.0); // x = 3 → x
    798
        /// assert_eq!(result.as_slice()[4], 4.0); // x ≥ 3 → x
    799
        /// # Ok::<(), trueno::TruenoError>(())
    800
        /// ```
    801
        ///
    802
        /// # Errors
    803
        ///
    804
        /// Returns `EmptyVector` if the input vector is empty.
    805
        ///
    806
        /// # References
    807
        ///
    808
        /// - Howard et al. (2019): "Searching for MobileNetV3"
    809
    0
        pub fn hardswish(&self) -> Result<Self> {
    810
    0
            if self.data.is_empty() {
    811
    0
                return Err(TruenoError::EmptyVector);
    812
    0
            }
    813
    814
            // Scalar implementation: hardswish(x) = x * relu6(x + 3) / 6
    815
            // Simplified piece-wise:
    816
            // - x <= -3: 0
    817
            // - x >= 3: x
    818
            // - else: x * (x + 3) / 6
    819
    0
            let data: Vec<f32> = self
    820
    0
                .data
    821
    0
                .iter()
    822
    0
                .map(|&x| {
    823
    0
                    if x <= -3.0 {
    824
    0
                        0.0
    825
    0
                    } else if x >= 3.0 {
    826
    0
                        x
    827
                    } else {
    828
    0
                        x * (x + 3.0) / 6.0
    829
                    }
    830
    0
                })
    831
    0
                .collect();
    832
    833
    0
            Ok(Vector::from_vec(data))
    834
    0
        }
    835
    836
        /// Mish activation function
    837
        ///
    838
        /// Applies the mish activation element-wise: mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + e^x))
    839
        ///
    840
        /// Mish is a self-regularizing non-monotonic activation function that often outperforms
    841
        /// ReLU and swish in computer vision tasks. It's used in YOLOv4 and many modern architectures.
    842
        ///
    843
        /// Properties:
    844
        /// - Smooth and non-monotonic (similar to swish)
    845
        /// - Self-regularizing: prevents dying neurons
    846
        /// - mish(0) ≈ 0 (small positive value)
    847
        /// - mish(x) ≈ x for large positive x (nearly linear)
    848
        /// - mish(x) ≈ 0 for large negative x
    849
        /// - Bounded below by ≈ -0.31 at x ≈ -1.19
    850
        ///
    851
        /// # Performance
    852
        ///
    853
        /// Compute-bound operation requiring exponential, logarithm, and tanh.
    854
        /// More expensive than ReLU/swish but often provides better accuracy.
    855
        ///
    856
        /// # Examples
    857
        ///
    858
        /// ```
    859
        /// use trueno::Vector;
    860
        ///
    861
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    862
        /// let result = v.mish()?;
    863
        ///
    864
        /// // Mish is smooth and self-gated
    865
        /// assert!(result.as_slice()[0] < 0.0); // Small negative output for negative inputs
    866
        /// assert!(result.as_slice()[2].abs() < 1e-5); // mish(0) = 0
    867
        /// assert!(result.as_slice()[4] > 1.5); // Large positive → near linear
    868
        /// # Ok::<(), trueno::TruenoError>(())
    869
        /// ```
    870
        ///
    871
        /// # Errors
    872
        ///
    873
        /// Returns `EmptyVector` if the input vector is empty.
    874
        ///
    875
        /// # References
    876
        ///
    877
        /// - Misra (2019): "Mish: A Self Regularized Non-Monotonic Neural Activation Function"
    878
    0
        pub fn mish(&self) -> Result<Self> {
    879
    0
            if self.data.is_empty() {
    880
    0
                return Err(TruenoError::EmptyVector);
    881
    0
            }
    882
    883
            // Scalar implementation: mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + e^x))
    884
    0
            let data: Vec<f32> = self
    885
    0
                .data
    886
    0
                .iter()
    887
    0
                .map(|&x| {
    888
                    // Handle extreme values for numerical stability
    889
    0
                    if x < -20.0 {
    890
                        // For very negative x: softplus ≈ 0, tanh(0) ≈ 0, so mish ≈ 0
    891
    0
                        0.0
    892
    0
                    } else if x > 20.0 {
    893
                        // For very positive x: softplus ≈ x, tanh(x) ≈ 1, so mish ≈ x
    894
    0
                        x
    895
                    } else {
    896
                        // Normal case: x * tanh(ln(1 + e^x))
    897
    0
                        let softplus = (1.0 + x.exp()).ln();
    898
    0
                        x * softplus.tanh()
    899
                    }
    900
    0
                })
    901
    0
                .collect();
    902
    903
    0
            Ok(Vector::from_vec(data))
    904
    0
        }
    905
    906
        /// SELU (Scaled Exponential Linear Unit) activation function
    907
        ///
    908
        /// Computes selu(x) = λ * (x if x > 0 else α * (exp(x) - 1))
    909
        /// where λ ≈ 1.0507 and α ≈ 1.6733
    910
        ///
    911
        /// # Properties
    912
        ///
    913
        /// - **Self-normalizing**: Activations converge to zero mean and unit variance
    914
        /// - **Vanishing gradient prevention**: Non-zero gradient for negative inputs
    915
        /// - **Automatic normalization**: Reduces need for batch normalization
    916
        ///
    917
        /// # Performance
    918
        ///
    919
        /// Uses scalar implementation (GPU disabled for element-wise ops).
    920
        ///
    921
        /// # Examples
    922
        ///
    923
        /// ```
    924
        /// use trueno::Vector;
    925
        ///
    926
        /// let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    927
        /// let result = v.selu()?;
    928
        ///
    929
        /// // Positive values scaled by λ ≈ 1.0507
    930
        /// assert!((result.as_slice()[3] - 1.0507).abs() < 0.001);
    931
        /// assert!((result.as_slice()[4] - 2.1014).abs() < 0.001);
    932
        ///
    933
        /// // Zero stays zero
    934
        /// assert!(result.as_slice()[2].abs() < 1e-5);
    935
        ///
    936
        /// // Negative values use ELU-like formula
    937
        /// assert!(result.as_slice()[0] < 0.0);
    938
        /// # Ok::<(), trueno::TruenoError>(())
    939
        /// ```
    940
        ///
    941
        /// # Errors
    942
        ///
    943
        /// Returns `EmptyVector` if the input vector is empty.
    944
        ///
    945
        /// # References
    946
        ///
    947
        /// - Klambauer et al. (2017): "Self-Normalizing Neural Networks"
    948
    0
        pub fn selu(&self) -> Result<Self> {
    949
    0
            if self.data.is_empty() {
    950
    0
                return Err(TruenoError::EmptyVector);
    951
    0
            }
    952
    953
            // SELU constants from Klambauer et al. (2017)
    954
            // These specific values ensure self-normalizing property
    955
            const LAMBDA: f32 = 1.0507009873554804934193349852946;
    956
            const ALPHA: f32 = 1.6732632423543772848170429916717;
    957
    958
            // Scalar implementation: selu(x) = λ * (x if x > 0 else α * (exp(x) - 1))
    959
    0
            let data: Vec<f32> = self
    960
    0
                .data
    961
    0
                .iter()
    962
    0
                .map(|&x| {
    963
    0
                    if x > 0.0 {
    964
    0
                        LAMBDA * x
    965
                    } else {
    966
    0
                        LAMBDA * ALPHA * (x.exp() - 1.0)
    967
                    }
    968
    0
                })
    969
    0
                .collect();
    970
    971
    0
            Ok(Vector::from_vec(data))
    972
    0
        }
    973
    }
    974
    975
    #[cfg(test)]
    976
    mod tests {
    977
        use super::*;
    978
    979
        // ========== Softmax ==========
    980
    981
        #[test]
    982
        fn test_softmax_basic() {
    983
            let v = Vector::from_slice(&[1.0, 2.0, 3.0]);
    984
            let result = v.softmax().unwrap();
    985
            // Check sum = 1
    986
            let sum: f32 = result.as_slice().iter().sum();
    987
            assert!((sum - 1.0).abs() < 1e-5);
    988
            // Check all values in [0, 1]
    989
            for &val in result.as_slice() {
    990
                assert!(val >= 0.0 && val <= 1.0);
    991
            }
    992
            // Check highest input has highest probability
    993
            assert!(result.as_slice()[2] > result.as_slice()[1]);
    994
            assert!(result.as_slice()[1] > result.as_slice()[0]);
    995
        }
    996
    997
        #[test]
    998
        fn test_softmax_empty() {
    999
            let v = Vector::<f32>::from_slice(&[]);
    1000
            assert!(matches!(v.softmax(), Err(TruenoError::EmptyVector)));
    1001
        }
    1002
    1003
        #[test]
    1004
        fn test_softmax_single() {
    1005
            let v = Vector::from_slice(&[5.0]);
    1006
            let result = v.softmax().unwrap();
    1007
            assert!((result.as_slice()[0] - 1.0).abs() < 1e-6);
    1008
        }
    1009
    1010
        #[test]
    1011
        fn test_softmax_uniform() {
    1012
            let v = Vector::from_slice(&[1.0, 1.0, 1.0, 1.0]);
    1013
            let result = v.softmax().unwrap();
    1014
            // All equal inputs should give equal outputs
    1015
            for &val in result.as_slice() {
    1016
                assert!((val - 0.25).abs() < 1e-6);
    1017
            }
    1018
        }
    1019
    1020
        #[test]
    1021
        fn test_softmax_large_values() {
    1022
            // Test numerical stability with large values
    1023
            let v = Vector::from_slice(&[1000.0, 1001.0, 1002.0]);
    1024
            let result = v.softmax().unwrap();
    1025
            let sum: f32 = result.as_slice().iter().sum();
    1026
            assert!((sum - 1.0).abs() < 1e-5);
    1027
        }
    1028
    1029
        // ========== Log Softmax ==========
    1030
    1031
        #[test]
    1032
        fn test_log_softmax_basic() {
    1033
            let v = Vector::from_slice(&[1.0, 2.0, 3.0]);
    1034
            let result = v.log_softmax().unwrap();
    1035
            // All log probabilities should be <= 0
    1036
            for &val in result.as_slice() {
    1037
                assert!(val <= 0.0);
    1038
            }
    1039
        }
    1040
    1041
        #[test]
    1042
        fn test_log_softmax_empty() {
    1043
            let v = Vector::<f32>::from_slice(&[]);
    1044
            assert!(matches!(v.log_softmax(), Err(TruenoError::EmptyVector)));
    1045
        }
    1046
    1047
        #[test]
    1048
        fn test_log_softmax_single() {
    1049
            let v = Vector::from_slice(&[5.0]);
    1050
            let result = v.log_softmax().unwrap();
    1051
            // log(1) = 0
    1052
            assert!((result.as_slice()[0] - 0.0).abs() < 1e-6);
    1053
        }
    1054
    1055
        // ========== ReLU ==========
    1056
    1057
        #[test]
    1058
        fn test_relu_basic() {
    1059
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1060
            let result = v.relu().unwrap();
    1061
            assert!((result.as_slice()[0] - 0.0).abs() < 1e-6);
    1062
            assert!((result.as_slice()[1] - 0.0).abs() < 1e-6);
    1063
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-6);
    1064
            assert!((result.as_slice()[3] - 1.0).abs() < 1e-6);
    1065
            assert!((result.as_slice()[4] - 2.0).abs() < 1e-6);
    1066
        }
    1067
    1068
        #[test]
    1069
        fn test_relu_empty() {
    1070
            let v = Vector::<f32>::from_slice(&[]);
    1071
            assert!(matches!(v.relu(), Err(TruenoError::EmptyVector)));
    1072
        }
    1073
    1074
        #[test]
    1075
        fn test_relu_all_negative() {
    1076
            let v = Vector::from_slice(&[-5.0, -3.0, -1.0]);
    1077
            let result = v.relu().unwrap();
    1078
            for &val in result.as_slice() {
    1079
                assert!((val - 0.0).abs() < 1e-6);
    1080
            }
    1081
        }
    1082
    1083
        #[test]
    1084
        fn test_relu_all_positive() {
    1085
            let v = Vector::from_slice(&[1.0, 2.0, 3.0]);
    1086
            let result = v.relu().unwrap();
    1087
            assert!((result.as_slice()[0] - 1.0).abs() < 1e-6);
    1088
            assert!((result.as_slice()[1] - 2.0).abs() < 1e-6);
    1089
            assert!((result.as_slice()[2] - 3.0).abs() < 1e-6);
    1090
        }
    1091
    1092
        // ========== Sigmoid ==========
    1093
    1094
        #[test]
    1095
        fn test_sigmoid_basic() {
    1096
            let v = Vector::from_slice(&[-10.0, 0.0, 10.0]);
    1097
            let result = v.sigmoid().unwrap();
    1098
            // sigmoid(-10) ≈ 0
    1099
            assert!(result.as_slice()[0] < 0.001);
    1100
            // sigmoid(0) = 0.5
    1101
            assert!((result.as_slice()[1] - 0.5).abs() < 1e-6);
    1102
            // sigmoid(10) ≈ 1
    1103
            assert!(result.as_slice()[2] > 0.999);
    1104
        }
    1105
    1106
        #[test]
    1107
        fn test_sigmoid_empty() {
    1108
            let v = Vector::<f32>::from_slice(&[]);
    1109
            assert!(matches!(v.sigmoid(), Err(TruenoError::EmptyVector)));
    1110
        }
    1111
    1112
        #[test]
    1113
        fn test_sigmoid_range() {
    1114
            let v = Vector::from_slice(&[-100.0, -1.0, 0.0, 1.0, 100.0]);
    1115
            let result = v.sigmoid().unwrap();
    1116
            for &val in result.as_slice() {
    1117
                assert!(val >= 0.0 && val <= 1.0);
    1118
            }
    1119
        }
    1120
    1121
        // ========== Leaky ReLU ==========
    1122
    1123
        #[test]
    1124
        fn test_leaky_relu_basic() {
    1125
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1126
            let result = v.leaky_relu(0.01).unwrap();
    1127
            assert!((result.as_slice()[0] - (-0.02)).abs() < 1e-6);
    1128
            assert!((result.as_slice()[1] - (-0.01)).abs() < 1e-6);
    1129
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-6);
    1130
            assert!((result.as_slice()[3] - 1.0).abs() < 1e-6);
    1131
            assert!((result.as_slice()[4] - 2.0).abs() < 1e-6);
    1132
        }
    1133
    1134
        #[test]
    1135
        fn test_leaky_relu_empty() {
    1136
            let v = Vector::<f32>::from_slice(&[]);
    1137
            assert!(matches!(v.leaky_relu(0.01), Err(TruenoError::EmptyVector)));
    1138
        }
    1139
    1140
        #[test]
    1141
        fn test_leaky_relu_different_slopes() {
    1142
            let v = Vector::from_slice(&[-1.0]);
    1143
            // slope 0.1
    1144
            let result = v.leaky_relu(0.1).unwrap();
    1145
            assert!((result.as_slice()[0] - (-0.1)).abs() < 1e-6);
    1146
            // slope 0.2
    1147
            let result = v.leaky_relu(0.2).unwrap();
    1148
            assert!((result.as_slice()[0] - (-0.2)).abs() < 1e-6);
    1149
        }
    1150
    1151
        // ========== ELU ==========
    1152
    1153
        #[test]
    1154
        fn test_elu_basic() {
    1155
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1156
            let result = v.elu(1.0).unwrap();
    1157
            // Positive values unchanged
    1158
            assert!((result.as_slice()[3] - 1.0).abs() < 1e-6);
    1159
            assert!((result.as_slice()[4] - 2.0).abs() < 1e-6);
    1160
            // Zero stays zero
    1161
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-6);
    1162
            // Negative values: alpha * (exp(x) - 1)
    1163
            assert!(result.as_slice()[0] < 0.0);
    1164
            assert!(result.as_slice()[1] < 0.0);
    1165
        }
    1166
    1167
        #[test]
    1168
        fn test_elu_empty() {
    1169
            let v = Vector::<f32>::from_slice(&[]);
    1170
            assert!(matches!(v.elu(1.0), Err(TruenoError::EmptyVector)));
    1171
        }
    1172
    1173
        // ========== GELU ==========
    1174
    1175
        #[test]
    1176
        fn test_gelu_basic() {
    1177
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1178
            let result = v.gelu().unwrap();
    1179
            // GELU(0) = 0
    1180
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-5);
    1181
            // GELU is approximately linear for positive values
    1182
            assert!(result.as_slice()[3] > 0.5);
    1183
            assert!(result.as_slice()[4] > 1.5);
    1184
            // Negative values are small but not zero
    1185
            assert!(result.as_slice()[0].abs() < 0.1);
    1186
        }
    1187
    1188
        #[test]
    1189
        fn test_gelu_empty() {
    1190
            let v = Vector::<f32>::from_slice(&[]);
    1191
            assert!(matches!(v.gelu(), Err(TruenoError::EmptyVector)));
    1192
        }
    1193
    1194
        // ========== Swish ==========
    1195
    1196
        #[test]
    1197
        fn test_swish_basic() {
    1198
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1199
            let result = v.swish().unwrap();
    1200
            // Swish(0) = 0
    1201
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-6);
    1202
            // Swish(x) = x * sigmoid(x)
    1203
            // Swish(1) ≈ 0.731
    1204
            assert!((result.as_slice()[3] - 0.731).abs() < 0.01);
    1205
        }
    1206
    1207
        #[test]
    1208
        fn test_swish_empty() {
    1209
            let v = Vector::<f32>::from_slice(&[]);
    1210
            assert!(matches!(v.swish(), Err(TruenoError::EmptyVector)));
    1211
        }
    1212
    1213
        // ========== Hardswish ==========
    1214
    1215
        #[test]
    1216
        fn test_hardswish_basic() {
    1217
            let v = Vector::from_slice(&[-4.0, -3.0, 0.0, 3.0, 4.0]);
    1218
            let result = v.hardswish().unwrap();
    1219
            // x <= -3: hardswish(x) = 0
    1220
            assert!((result.as_slice()[0] - 0.0).abs() < 1e-6);
    1221
            assert!((result.as_slice()[1] - 0.0).abs() < 1e-6);
    1222
            // x >= 3: hardswish(x) = x
    1223
            assert!((result.as_slice()[3] - 3.0).abs() < 1e-6);
    1224
            assert!((result.as_slice()[4] - 4.0).abs() < 1e-6);
    1225
            // x = 0: hardswish(0) = 0
    1226
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-6);
    1227
        }
    1228
    1229
        #[test]
    1230
        fn test_hardswish_empty() {
    1231
            let v = Vector::<f32>::from_slice(&[]);
    1232
            assert!(matches!(v.hardswish(), Err(TruenoError::EmptyVector)));
    1233
        }
    1234
    1235
        // ========== Mish ==========
    1236
    1237
        #[test]
    1238
        fn test_mish_basic() {
    1239
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1240
            let result = v.mish().unwrap();
    1241
            // Mish(0) = 0
    1242
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-6);
    1243
            // Mish is smooth and non-monotonic for negative values
    1244
            assert!(result.as_slice()[0] < 0.0);
    1245
        }
    1246
    1247
        #[test]
    1248
        fn test_mish_empty() {
    1249
            let v = Vector::<f32>::from_slice(&[]);
    1250
            assert!(matches!(v.mish(), Err(TruenoError::EmptyVector)));
    1251
        }
    1252
    1253
        // ========== SELU ==========
    1254
    1255
        #[test]
    1256
        fn test_selu_basic() {
    1257
            let v = Vector::from_slice(&[-2.0, -1.0, 0.0, 1.0, 2.0]);
    1258
            let result = v.selu().unwrap();
    1259
            // SELU(0) = 0
    1260
            assert!((result.as_slice()[2] - 0.0).abs() < 1e-5);
    1261
            // Positive values scaled by λ ≈ 1.0507
    1262
            assert!((result.as_slice()[3] - 1.0507).abs() < 0.001);
    1263
            assert!((result.as_slice()[4] - 2.1014).abs() < 0.001);
    1264
            // Negative values use ELU-like formula
    1265
            assert!(result.as_slice()[0] < 0.0);
    1266
            assert!(result.as_slice()[1] < 0.0);
    1267
        }
    1268
    1269
        #[test]
    1270
        fn test_selu_empty() {
    1271
            let v = Vector::<f32>::from_slice(&[]);
    1272
            assert!(matches!(v.selu(), Err(TruenoError::EmptyVector)));
    1273
        }
    1274
    1275
        // ========== Backend Tests ==========
    1276
    1277
        #[test]
    1278
        fn test_relu_scalar_backend() {
    1279
            let v = Vector::from_slice_with_backend(&[-1.0, 0.0, 1.0], Backend::Scalar);
    1280
            let result = v.relu().unwrap();
    1281
            assert!((result.as_slice()[0] - 0.0).abs() < 1e-6);
    1282
            assert!((result.as_slice()[2] - 1.0).abs() < 1e-6);
    1283
        }
    1284
    1285
        #[test]
    1286
        #[cfg(target_arch = "x86_64")]
    1287
        fn test_relu_sse2_backend() {
    1288
            let v = Vector::from_slice_with_backend(&[-1.0, 0.0, 1.0, 2.0], Backend::SSE2);
    1289
            let result = v.relu().unwrap();
    1290
            assert!((result.as_slice()[0] - 0.0).abs() < 1e-6);
    1291
            assert!((result.as_slice()[2] - 1.0).abs() < 1e-6);
    1292
        }
    1293
    1294
        #[test]
    1295
        fn test_sigmoid_scalar_backend() {
    1296
            let v = Vector::from_slice_with_backend(&[0.0], Backend::Scalar);
    1297
            let result = v.sigmoid().unwrap();
    1298
            assert!((result.as_slice()[0] - 0.5).abs() < 1e-6);
    1299
        }
    1300
    1301
        // ========== Large Array Tests ==========
    1302
    1303
        #[test]
    1304
        fn test_relu_large() {
    1305
            let v = Vector::from_slice(&[-1.0; 1000]);
    1306
            let result = v.relu().unwrap();
    1307
            for &val in result.as_slice() {
    1308
                assert!((val - 0.0).abs() < 1e-6);
    1309
            }
    1310
        }
    1311
    1312
        #[test]
    1313
        fn test_sigmoid_large() {
    1314
            let v = Vector::from_slice(&[0.0; 1000]);
    1315
            let result = v.sigmoid().unwrap();
    1316
            for &val in result.as_slice() {
    1317
                assert!((val - 0.5).abs() < 1e-6);
    1318
            }
    1319
        }
    1320
    1321
        #[test]
    1322
        fn test_softmax_large() {
    1323
            let v = Vector::from_slice(&[1.0; 100]);
    1324
            let result = v.softmax().unwrap();
    1325
            let sum: f32 = result.as_slice().iter().sum();
    1326
            assert!((sum - 1.0).abs() < 1e-4);
    1327
            // All equal inputs should give equal probabilities
    1328
            for &val in result.as_slice() {
    1329
                assert!((val - 0.01).abs() < 1e-4);
    1330
            }
    1331
        }
    1332
    }