## Coverage Report

#### Created: 2025-09-08 21:26

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/ruchy/src/frontend/lexer.rs

    Line
    Count
    Source
    1
    //! Lexical analysis and tokenization
    2
    3
    use crate::frontend::ast::Span;
    4
    use logos::{Lexer, Logos};
    5
    6
    /// Process a basic escape character
    7
    0
    fn process_basic_escape(ch: char) -> Option<char> {
    8
    0
        match ch {
    9
    0
            'n' => Some('\n'),
    10
    0
            't' => Some('\t'),
    11
    0
            'r' => Some('\r'),
    12
    0
            '\\' => Some('\\'),
    13
    0
            '"' => Some('"'),
    14
    0
            '\'' => Some('\''),
    15
    0
            '0' => Some('\0'),
    16
    0
            _ => None,
    17
        }
    18
    0
    }
    19
    20
    /// Process a Unicode escape sequence
    21
    0
    fn process_unicode_escape(chars: &mut std::str::Chars) -> String {
    22
    0
        chars.next(); // consume '{'
    23
    24
    0
        let mut hex = String::new();
    25
    0
        for hex_char in chars.by_ref() {
    26
    0
            if hex_char == '}' {
    27
    0
                break;
    28
    0
            }
    29
    0
            hex.push(hex_char);
    30
        }
    31
    32
    0
        if let Ok(code_point) = u32::from_str_radix(&hex, 16) {
    33
    0
            if let Some(unicode_char) = char::from_u32(code_point) {
    34
    0
                return unicode_char.to_string();
    35
    0
            }
    36
    0
        }
    37
    38
        // Invalid code point or hex, keep as literal
    39
    0
        format!("\\u{{{hex}}}")
    40
    0
    }
    41
    42
    /// Process escape sequences in a string literal
    43
    0
    fn process_escapes(s: &str) -> String {
    44
    0
        let mut result = String::new();
    45
    0
        let mut chars = s.chars();
    46
    47
    0
        while let Some(ch) = chars.next() {
    48
    0
            if ch == '\\' {
    49
    0
                match chars.next() {
    50
    0
                    None => result.push('\\'), // End of string
    51
    0
                    Some('u') if chars.as_str().starts_with('{') => {
    52
    0
                        result.push_str(&process_unicode_escape(&mut chars));
    53
    0
                    }
    54
    0
                    Some(escape_ch) => {
    55
    0
                        if let Some(escaped) = process_basic_escape(escape_ch) {
    56
    0
                            result.push(escaped);
    57
    0
                        } else {
    58
    0
                            // Unknown escape sequence, keep as literal
    59
    0
                            result.push('\\');
    60
    0
                            result.push(escape_ch);
    61
    0
                        }
    62
                    }
    63
                }
    64
    0
            } else {
    65
    0
                result.push(ch);
    66
    0
            }
    67
        }
    68
    69
    0
        result
    70
    0
    }
    71
    72
    0
    #[derive(Logos, Debug, PartialEq, Clone)]
    73
    #[logos(skip r"[ \t\n\f]+")]
    74
    #[logos(skip r"//[^\n]*")]
    75
    #[logos(skip r"/\*([^*]|\*[^/])*\*/")]
    76
    pub enum Token {
    77
        // Literals
    78
        #[regex(r"[0-9]+(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?", |lex| {
    79
            let slice = lex.slice();
    80
            // Strip type suffix for parsing the numeric value
    81
    0
            let (num_part, _type_suffix) = if let Some(pos) = slice.find(|c: char| c.is_alphabetic()) {
    82
                (&slice[..pos], Some(&slice[pos..]))
    83
            } else {
    84
                (slice, None)
    85
            };
    86
            num_part.parse::<i64>().ok()
    87
        })]
    88
        Integer(i64),
    89
    90
        #[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", |lex| lex.slice().parse::<f64>().ok())]
    91
        Float(f64),
    92
    93
        #[regex(r#""([^"\\]|\\.)*""#, |lex| {
    94
            let s = lex.slice();
    95
            let inner = &s[1..s.len()-1];
    96
            Some(process_escapes(inner))
    97
        })]
    98
        String(String),
    99
    100
        #[regex(r#"f"([^"\\]|\\.)*""#, |lex| {
    101
            let s = lex.slice();
    102
            // Remove f" prefix and " suffix
    103
            let inner = &s[2..s.len()-1];
    104
            Some(process_escapes(inner))
    105
        })]
    106
        FString(String),
    107
    108
        #[regex(r#"r"([^"])*""#, |lex| {
    109
            let s = lex.slice();
    110
            // Remove r" prefix and " suffix - no escape processing for raw strings
    111
            Some(s[2..s.len()-1].to_string())
    112
        })]
    113
        RawString(String),
    114
    115
        #[regex(r"'([^'\\]|\\.)'", |lex| {
    116
            let s = lex.slice();
    117
            let inner = &s[1..s.len()-1];
    118
            if inner.len() == 1 {
    119
                inner.chars().next()
    120
            } else if inner.starts_with('\\') && inner.len() == 2 {
    121
                match inner.chars().nth(1) {
    122
                    Some('n') => Some('\n'),
    123
                    Some('t') => Some('\t'),
    124
                    Some('r') => Some('\r'),
    125
                    Some('\\') => Some('\\'),
    126
                    Some('\'') => Some('\''),
    127
                    Some('0') => Some('\0'),
    128
                    _ => None,
    129
                }
    130
            } else {
    131
                None
    132
            }
    133
        })]
    134
        Char(char),
    135
    136
        #[token("true", |_| true)]
    137
        #[token("false", |_| false)]
    138
        Bool(bool),
    139
    140
        // Keywords
    141
        #[token("fun")]
    142
        Fun,
    143
        #[token("fn")]
    144
        Fn,
    145
        #[token("let")]
    146
        Let,
    147
        #[token("var")]
    148
        Var,
    149
        #[token("mod")]
    150
        Mod,
    151
        #[token("if")]
    152
        If,
    153
        #[token("else")]
    154
        Else,
    155
        #[token("match")]
    156
        Match,
    157
        #[token("for")]
    158
        For,
    159
        #[token("in")]
    160
        In,
    161
        #[token("while")]
    162
        While,
    163
        #[token("loop")]
    164
        Loop,
    165
        #[token("async")]
    166
        Async,
    167
        #[token("await")]
    168
        Await,
    169
        #[token("throw")]
    170
        Throw,
    171
        #[token("try")]
    172
        Try,
    173
        #[token("catch")]
    174
        Catch,
    175
        #[token("finally")]
    176
        Finally,
    177
        #[token("return")]
    178
        Return,
    179
        #[token("command")]
    180
        Command,
    181
        #[token("Ok")]
    182
        Ok,
    183
        #[token("Err")]
    184
        Err,
    185
        #[token("Some")]
    186
        Some,
    187
        #[token("None")]
    188
        None,
    189
        #[token("null")]
    190
        Null,
    191
        #[token("Result")]
    192
        Result,
    193
        #[token("Option")]
    194
        Option,
    195
        #[token("break")]
    196
        Break,
    197
        #[token("continue")]
    198
        Continue,
    199
        #[token("struct")]
    200
        Struct,
    201
        #[token("enum")]
    202
        Enum,
    203
        #[token("impl")]
    204
        Impl,
    205
        #[token("trait")]
    206
        Trait,
    207
        #[token("extend")]
    208
        Extend,
    209
        #[token("actor")]
    210
        Actor,
    211
        #[token("state")]
    212
        State,
    213
        #[token("receive")]
    214
        Receive,
    215
        #[token("send")]
    216
        Send,
    217
        #[token("ask")]
    218
        Ask,
    219
        #[token("type")]
    220
        Type,
    221
        #[token("where")]
    222
        Where,
    223
        #[token("const")]
    224
        Const,
    225
        #[token("static")]
    226
        Static,
    227
        #[token("mut")]
    228
        Mut,
    229
        #[token("pub")]
    230
        Pub,
    231
        #[token("import")]
    232
        Import,
    233
        #[token("use")]
    234
        Use,
    235
        #[token("as")]
    236
        As,
    237
        #[token("module")]
    238
        Module,
    239
        #[token("export")]
    240
        Export,
    241
        #[token("df", priority = 2)]
    242
        DataFrame,
    243
    244
        // Identifiers (lower priority than keywords)
    245
        #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
    246
        Identifier(String),
    247
    248
        // Operators
    249
        #[token("+")]
    250
        Plus,
    251
        #[token("-")]
    252
        Minus,
    253
        #[token("*")]
    254
        Star,
    255
        #[token("/")]
    256
        Slash,
    257
        #[token("%")]
    258
        Percent,
    259
        #[token("**")]
    260
        Power,
    261
    262
        #[token("==")]
    263
        EqualEqual,
    264
        #[token("!=")]
    265
        NotEqual,
    266
        #[token("<?")]
    267
        ActorQuery,
    268
        #[token("<-")]
    269
        LeftArrow,
    270
        #[token("<")]
    271
        Less,
    272
        #[token("<=")]
    273
        LessEqual,
    274
        #[token(">")]
    275
        Greater,
    276
        #[token(">=")]
    277
        GreaterEqual,
    278
    279
        #[token("&&")]
    280
        AndAnd,
    281
        #[token("||")]
    282
        OrOr,
    283
        #[token("!")]
    284
        Bang,
    285
    286
        #[token("&")]
    287
        Ampersand,
    288
        #[token("|")]
    289
        Pipe,
    290
        #[token("^")]
    291
        Caret,
    292
        #[token("~")]
    293
        Tilde,
    294
        #[token("\\")]
    295
        Backslash,
    296
        #[token("<<")]
    297
        LeftShift,
    298
    299
        #[token("=")]
    300
        Equal,
    301
        #[token("+=")]
    302
        PlusEqual,
    303
        #[token("-=")]
    304
        MinusEqual,
    305
        #[token("*=")]
    306
        StarEqual,
    307
        #[token("/=")]
    308
        SlashEqual,
    309
        #[token("%=")]
    310
        PercentEqual,
    311
        #[token("**=")]
    312
        PowerEqual,
    313
        #[token("&=")]
    314
        AmpersandEqual,
    315
        #[token("|=")]
    316
        PipeEqual,
    317
        #[token("^=")]
    318
        CaretEqual,
    319
        #[token("<<=")]
    320
        LeftShiftEqual,
    321
    322
        #[token("++")]
    323
        Increment,
    324
        #[token("--")]
    325
        Decrement,
    326
    327
        #[token("|>")]
    328
        Pipeline,
    329
        #[token("->")]
    330
        Arrow,
    331
        #[token("=>")]
    332
        FatArrow,
    333
        #[token("..")]
    334
        DotDot,
    335
        #[token("..=")]
    336
        DotDotEqual,
    337
        #[token("...")]
    338
        DotDotDot,
    339
        #[token("??")]
    340
        NullCoalesce,
    341
        #[token("?")]
    342
        Question,
    343
        #[token("?.")]
    344
        SafeNav,
    345
    346
        // Delimiters
    347
        #[token("(")]
    348
        LeftParen,
    349
        #[token(")")]
    350
        RightParen,
    351
        #[token("[")]
    352
        LeftBracket,
    353
        #[token("]")]
    354
        RightBracket,
    355
        #[token("{")]
    356
        LeftBrace,
    357
        #[token("}")]
    358
        RightBrace,
    359
    360
        // Punctuation
    361
        #[token(",")]
    362
        Comma,
    363
        #[token(".")]
    364
        Dot,
    365
        #[token(":")]
    366
        Colon,
    367
        #[token("::")]
    368
        ColonColon,
    369
        #[token(";")]
    370
        Semicolon,
    371
        #[token("_", priority = 2)]
    372
        Underscore,
    373
    374
        // Attribute support
    375
        #[token("#")]
    376
        Hash,
    377
    }
    378
    379
    impl Token {
    380
        #[must_use]
    381
    0
        pub fn is_binary_op(&self) -> bool {
    382
    0
            matches!(
    383
    0
                self,
    384
                Token::Plus
    385
                    | Token::Minus
    386
                    | Token::Star
    387
                    | Token::Slash
    388
                    | Token::Percent
    389
                    | Token::Power
    390
                    | Token::EqualEqual
    391
                    | Token::NotEqual
    392
                    | Token::Less
    393
                    | Token::LessEqual
    394
                    | Token::Greater
    395
                    | Token::GreaterEqual
    396
                    | Token::AndAnd
    397
                    | Token::OrOr
    398
                    | Token::Ampersand
    399
                    | Token::Pipe
    400
                    | Token::Caret
    401
                    | Token::LeftShift
    402
            )
    403
    0
        }
    404
    405
        #[must_use]
    406
    0
        pub fn is_unary_op(&self) -> bool {
    407
    0
            matches!(
    408
    0
                self,
    409
                Token::Bang | Token::Minus | Token::Tilde | Token::Ampersand
    410
            )
    411
    0
        }
    412
    413
        #[must_use]
    414
    0
        pub fn is_assignment_op(&self) -> bool {
    415
    0
            matches!(
    416
    0
                self,
    417
                Token::Equal
    418
                    | Token::PlusEqual
    419
                    | Token::MinusEqual
    420
                    | Token::StarEqual
    421
                    | Token::SlashEqual
    422
                    | Token::PercentEqual
    423
                    | Token::PowerEqual
    424
                    | Token::AmpersandEqual
    425
                    | Token::PipeEqual
    426
                    | Token::CaretEqual
    427
                    | Token::LeftShiftEqual
    428
            )
    429
    0
        }
    430
    }
    431
    432
    pub struct TokenStream<'a> {
    433
        lexer: Lexer<'a, Token>,
    434
        peeked: Option<(Token, Span)>,
    435
    }
    436
    437
    /// Saved position in the token stream for backtracking
    438
    #[derive(Clone)]
    439
    pub struct TokenStreamPosition<'a> {
    440
        lexer: Lexer<'a, Token>,
    441
        peeked: Option<(Token, Span)>,
    442
    }
    443
    444
    impl<'a> TokenStream<'a> {
    445
        #[must_use]
    446
    0
        pub fn new(input: &'a str) -> Self {
    447
    0
            Self {
    448
    0
                lexer: Token::lexer(input),
    449
    0
                peeked: None,
    450
    0
            }
    451
    0
        }
    452
    453
        /// Save the current position for later restoration
    454
        #[must_use]
    455
    0
        pub fn position(&self) -> TokenStreamPosition<'a> {
    456
    0
            TokenStreamPosition {
    457
    0
                lexer: self.lexer.clone(),
    458
    0
                peeked: self.peeked.clone(),
    459
    0
            }
    460
    0
        }
    461
    462
        /// Restore a previously saved position
    463
    0
        pub fn set_position(&mut self, pos: TokenStreamPosition<'a>) {
    464
    0
            self.lexer = pos.lexer;
    465
    0
            self.peeked = pos.peeked;
    466
    0
        }
    467
    468
        #[allow(clippy::should_implement_trait)]
    469
    0
        pub fn next(&mut self) -> Option<(Token, Span)> {
    470
    0
            if let Some(peeked) = self.peeked.take() {
    471
    0
                return Some(peeked);
    472
    0
            }
    473
    474
    0
            self.lexer.next().map(|result| {
    475
    0
                let token = result.unwrap_or(Token::Bang); // Error recovery
    476
    0
                let span = Span::new(self.lexer.span().start, self.lexer.span().end);
    477
    0
                (token, span)
    478
    0
            })
    479
    0
        }
    480
    481
    0
        pub fn peek(&mut self) -> Option<&(Token, Span)> {
    482
    0
            if self.peeked.is_none() {
    483
    0
                self.peeked = self.next();
    484
    0
            }
    485
    0
            self.peeked.as_ref()
    486
    0
        }
    487
    488
    0
        pub fn peek_nth(&mut self, n: usize) -> Option<(Token, Span)> {
    489
            // For simplicity, we'll only support peek_nth(1) by cloning the lexer
    490
    0
            if n == 1 {
    491
    0
                let saved_peeked = self.peeked.clone();
    492
    0
                let saved_lexer = self.lexer.clone();
    493
    494
                // Get first token
    495
    0
                let _first = self.peek();
    496
    0
                self.advance();
    497
    498
                // Get second token
    499
    0
                let result = self.peek().cloned();
    500
    501
                // Restore state
    502
    0
                self.lexer = saved_lexer;
    503
    0
                self.peeked = saved_peeked;
    504
    505
    0
                result
    506
            } else {
    507
    0
                None // Not supported for n > 1
    508
            }
    509
    0
        }
    510
    511
    0
        pub fn peek_nth_is_colon(&mut self, n: usize) -> bool {
    512
    0
            if n == 0 {
    513
    0
                self.peek().is_some_and(|(t, _)| matches!(t, Token::Colon))
    514
            } else {
    515
    0
                self.peek_nth(n)
    516
    0
                    .is_some_and(|(t, _)| matches!(t, Token::Colon))
    517
            }
    518
    0
        }
    519
    520
        /// Expect a specific token and return its span
    521
        ///
    522
        /// # Errors
    523
        ///
    524
        /// Returns an error if the next token doesn't match the expected token or if we reached EOF
    525
    0
        pub fn expect(&mut self, expected: &Token) -> anyhow::Result<Span> {
    526
    0
            match self.next() {
    527
    0
                Some((token, span)) if token == *expected => Ok(span),
    528
    0
                Some((token, _)) => anyhow::bail!("Expected {:?}, found {:?}", expected, token),
    529
    0
                None => anyhow::bail!("Expected {:?}, found EOF", expected),
    530
            }
    531
    0
        }
    532
    533
        // Alias for next() to avoid clippy warning about Iterator trait
    534
    0
        pub fn advance(&mut self) -> Option<(Token, Span)> {
    535
    0
            self.next()
    536
    0
        }
    537
    }
    538
    539
    #[cfg(test)]
    540
    #[allow(clippy::unwrap_used)]
    541
    #[allow(clippy::panic)]
    542
    mod tests {
    543
        use super::*;
    544
        use proptest::prelude::*;
    545
    546
        #[test]
    547
        #[allow(clippy::approx_constant)] // Intentional literal for test
    548
        fn test_tokenize_basic() {
    549
            let mut stream = TokenStream::new("let x = 42 + 3.14");
    550
    551
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Let));
    552
            assert_eq!(
    553
                stream.next().map(|(t, _)| t),
    554
                Some(Token::Identifier("x".to_string()))
    555
            );
    556
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Equal));
    557
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Integer(42)));
    558
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Plus));
    559
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Float(3.14))); // Intentional literal for test
    560
            assert_eq!(stream.next().map(|(t, _)| t), None);
    561
        }
    562
    563
        #[test]
    564
        fn test_tokenize_pipeline() {
    565
            let mut stream = TokenStream::new("[1, 2, 3] >> map(|x| x * 2)");
    566
    567
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::LeftBracket));
    568
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Integer(1)));
    569
            // ... rest of tokens
    570
        }
    571
    572
        #[test]
    573
        fn test_tokenize_comments() {
    574
            let mut stream = TokenStream::new("x // comment\n+ /* block */ y");
    575
    576
            assert_eq!(
    577
                stream.next().map(|(t, _)| t),
    578
                Some(Token::Identifier("x".to_string()))
    579
            );
    580
            assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Plus));
    581
            assert_eq!(
    582
                stream.next().map(|(t, _)| t),
    583
                Some(Token::Identifier("y".to_string()))
    584
            );
    585
        }
    586
    587
        proptest! {
    588
            #[test]
    589
            fn test_tokenize_identifiers(s in "[a-zA-Z_][a-zA-Z0-9_]{0,100}") {
    590
                // Skip reserved keywords that should tokenize as their respective tokens
    591
                let reserved_keywords = [
    592
                    "true", "false", "fun", "fn", "let", "var", "mod", "if", "else", "match",
    593
                    "for", "in", "while", "loop", "async", "await", "throw", "try", "catch",
    594
                    "return", "command", "Ok", "Err", "Some", "None", "null", "Result", "Option",
    595
                    "break", "continue", "struct", "enum", "impl", "trait", "extend", "actor",
    596
                    "state", "receive", "send", "ask", "type", "where", "const", "static",
    597
                    "mut", "pub", "import", "use", "as", "module", "export", "df"
    598
                ];
    599
    600
                if reserved_keywords.contains(&s.as_str()) {
    601
                    return Ok(()); // Skip test for reserved keywords
    602
                }
    603
    604
                let mut stream = TokenStream::new(&s);
    605
                match stream.advance() {
    606
                    Some((Token::Identifier(id), _)) => prop_assert_eq!(id, s),
    607
                    Some((Token::Underscore, _)) if s == "_" => {}, // Special case for underscore
    608
                    _ => panic!("Failed to tokenize identifier: {s}"),
    609
                }
    610
            }
    611
    612
            #[test]
    613
            fn test_tokenize_integers(n in 0i64..1_000_000) {
    614
                let s = n.to_string();
    615
                let mut stream = TokenStream::new(&s);
    616
                match stream.advance() {
    617
                    Some((Token::Integer(i), _)) => prop_assert_eq!(i, n),
    618
                    _ => panic!("Failed to tokenize integer"),
    619
                }
    620
            }
    621
        }
    622
    }