/home/noah/src/ruchy/src/frontend/lexer.rs

Source
//! Lexical analysis and tokenization

use crate::frontend::ast::Span;
use logos::{Lexer, Logos};

/// Process escape sequences in a string literal
fn process_escapes(s: &str) -> String {
    let mut result = String::new();
    let mut chars = s.chars();

    while let Some(ch1.55k) = chars.next() {
        if ch == '\\' {
            match chars.next() {
                Some('n') => result.push('\n'),
                Some('t') => result.push('\t'),
                Some('r') => result.push('\r'),
                Some('\\') | None => result.push('\\'), // Backslash or end of string
                Some('"') => result.push('"'),
                Some('\'') => result.push('\''),
                Some('0') => result.push('\0'),
                Some('u') if chars.as_str().starts_with('{') => {
                    // Unicode escape: \u{XXXX}
                    chars.next(); // consume '{'
                    let mut hex = String::new();
                    for hex_char in chars.by_ref() {
                        if hex_char == '}' {
                            break;
                        }
                        hex.push(hex_char);
                    }

                    if let Ok(code_point) = u32::from_str_radix(&hex, 16) {
                        if let Some(unicode_char) = char::from_u32(code_point) {
                            result.push(unicode_char);
                        } else {
                            // Invalid Unicode code point, keep as literal
                            result.push_str("\\u{");
                            result.push_str(&hex);
                            result.push('}');
                        }
                    } else {
                        // Invalid hex, keep as literal
                        result.push_str("\\u{");
                        result.push_str(&hex);
                        result.push('}');
                    }
                }
                Some(other) => {
                    // Unknown escape sequence, keep as literal
                    result.push('\\');
                    result.push(other);
                }
            }
        } else {
            result.push(ch);
        }
    }

    result
}

#[derive(Logos, Debug, PartialEq, Clone)]
#[logos(skip r"[ \t\n\f]+")]
#[logos(skip r"//[^\n]*")]
#[logos(skip r"/\*([^*]|\*[^/])*\*/")]
pub enum Token {
    // Literals
    #[regex(r"[0-9]+(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?", |lex| {
        let slice = lex.slice();
        // Strip type suffix for parsing the numeric value
        let (num_part, _type_suffix) = if let Some(pos) = slice.find(|c: char| c.is_alphabetic()) {
            (&slice[..pos], Some(&slice[pos..]))
        } else {
            (slice, None)
        };
        num_part.parse::<i64>().ok()
    })]
    Integer(i64),

    #[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", |lex| lex.slice().parse::<f64>().ok())]
    Float(f64),

    #[regex(r#""([^"\\]|\\.)*""#, |lex| {
        let s = lex.slice();
        let inner = &s[1..s.len()-1];
        Some(process_escapes(inner))
    })]
    String(String),

    #[regex(r#"f"([^"\\]|\\.)*""#, |lex| {
        let s = lex.slice();
        // Remove f" prefix and " suffix
        let inner = &s[2..s.len()-1];
        Some(process_escapes(inner))
    })]
    FString(String),

    #[regex(r#"r"([^"])*""#, |lex| {
        let s = lex.slice();
        // Remove r" prefix and " suffix - no escape processing for raw strings
        Some(s[2..s.len()-1].to_string())
    })]
    RawString(String),

    #[regex(r"'([^'\\]|\\.)'", |lex| {
        let s = lex.slice();
        let inner = &s[1..s.len()-1];
        if inner.len() == 1 {
            inner.chars().next()
        } else if inner.starts_with('\\') && inner.len() == 2 {
            match inner.chars().nth(1) {
                Some('n') => Some('\n'),
                Some('t') => Some('\t'),
                Some('r') => Some('\r'),
                Some('\\') => Some('\\'),
                Some('\'') => Some('\''),
                Some('0') => Some('\0'),
                _ => None,
            }
        } else {
            None
        }
    })]
    Char(char),

    #[token("true", |_| true)]
    #[token("false", |_| false)]
    Bool(bool),

    // Keywords
    #[token("fun")]
    Fun,
    #[token("fn")]
    Fn,
    #[token("let")]
    Let,
    #[token("var")]
    Var,
    #[token("mod")]
    Mod,
    #[token("if")]
    If,
    #[token("else")]
    Else,
    #[token("match")]
    Match,
    #[token("for")]
    For,
    #[token("in")]
    In,
    #[token("while")]
    While,
    #[token("loop")]
    Loop,
    #[token("async")]
    Async,
    #[token("await")]
    Await,
    #[token("throw")]
    Throw,
    #[token("try")]
    Try,
    #[token("catch")]
    Catch,
    #[token("return")]
    Return,
    #[token("command")]
    Command,
    #[token("Ok")]
    Ok,
    #[token("Err")]
    Err,
    #[token("Some")]
    Some,
    #[token("None")]
    None,
    #[token("null")]
    Null,
    #[token("Result")]
    Result,
    #[token("Option")]
    Option,
    #[token("break")]
    Break,
    #[token("continue")]
    Continue,
    #[token("struct")]
    Struct,
    #[token("enum")]
    Enum,
    #[token("impl")]
    Impl,
    #[token("trait")]
    Trait,
    #[token("extend")]
    Extend,
    #[token("actor")]
    Actor,
    #[token("state")]
    State,
    #[token("receive")]
    Receive,
    #[token("send")]
    Send,
    #[token("ask")]
    Ask,
    #[token("type")]
    Type,
    #[token("where")]
    Where,
    #[token("const")]
    Const,
    #[token("static")]
    Static,
    #[token("mut")]
    Mut,
    #[token("pub")]
    Pub,
    #[token("import")]
    Import,
    #[token("use")]
    Use,
    #[token("as")]
    As,
    #[token("module")]
    Module,
    #[token("export")]
    Export,
    #[token("df", priority = 2)]
    DataFrame,

    // Identifiers (lower priority than keywords)
    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
    Identifier(String),

    // Operators
    #[token("+")]
    Plus,
    #[token("-")]
    Minus,
    #[token("*")]
    Star,
    #[token("/")]
    Slash,
    #[token("%")]
    Percent,
    #[token("**")]
    Power,

    #[token("==")]
    EqualEqual,
    #[token("!=")]
    NotEqual,
    #[token("<?")]
    ActorQuery,
    #[token("<-")]
    LeftArrow,
    #[token("<")]
    Less,
    #[token("<=")]
    LessEqual,
    #[token(">")]
    Greater,
    #[token(">=")]
    GreaterEqual,

    #[token("&&")]
    AndAnd,
    #[token("||")]
    OrOr,
    #[token("!")]
    Bang,

    #[token("&")]
    Ampersand,
    #[token("|")]
    Pipe,
    #[token("^")]
    Caret,
    #[token("~")]
    Tilde,
    #[token("\\")]
    Backslash,
    #[token("<<")]
    LeftShift,

    #[token("=")]
    Equal,
    #[token("+=")]
    PlusEqual,
    #[token("-=")]
    MinusEqual,
    #[token("*=")]
    StarEqual,
    #[token("/=")]
    SlashEqual,
    #[token("%=")]
    PercentEqual,
    #[token("**=")]
    PowerEqual,
    #[token("&=")]
    AmpersandEqual,
    #[token("|=")]
    PipeEqual,
    #[token("^=")]
    CaretEqual,
    #[token("<<=")]
    LeftShiftEqual,

    #[token("++")]
    Increment,
    #[token("--")]
    Decrement,

    #[token("|>")]
    Pipeline,
    #[token("->")]
    Arrow,
    #[token("=>")]
    FatArrow,
    #[token("..")]
    DotDot,
    #[token("..=")]
    DotDotEqual,
    #[token("...")]
    DotDotDot,
    #[token("??")]
    NullCoalesce,
    #[token("?")]
    Question,
    #[token("?.")]
    SafeNav,

    // Delimiters
    #[token("(")]
    LeftParen,
    #[token(")")]
    RightParen,
    #[token("[")]
    LeftBracket,
    #[token("]")]
    RightBracket,
    #[token("{")]
    LeftBrace,
    #[token("}")]
    RightBrace,

    // Punctuation
    #[token(",")]
    Comma,
    #[token(".")]
    Dot,
    #[token(":")]
    Colon,
    #[token("::")]
    ColonColon,
    #[token(";")]
    Semicolon,
    #[token("_", priority = 2)]
    Underscore,

    // Attribute support
    #[token("#")]
    Hash,
}

impl Token {
    #[must_use]
    pub fn is_binary_op(&self) -> bool {
        matches!(
            self,
            Token::Plus
                | Token::Minus
                | Token::Star
                | Token::Slash
                | Token::Percent
                | Token::Power
                | Token::EqualEqual
                | Token::NotEqual
                | Token::Less
                | Token::LessEqual
                | Token::Greater
                | Token::GreaterEqual
                | Token::AndAnd
                | Token::OrOr
                | Token::Ampersand
                | Token::Pipe
                | Token::Caret
                | Token::LeftShift
        )
    }

    #[must_use]
    pub fn is_unary_op(&self) -> bool {
        matches!(
            self,
            Token::Bang | Token::Minus | Token::Tilde | Token::Ampersand
        )
    }

    #[must_use]
    pub fn is_assignment_op(&self) -> bool {
        matches!(
            self,
            Token::Equal
                | Token::PlusEqual
                | Token::MinusEqual
                | Token::StarEqual
                | Token::SlashEqual
                | Token::PercentEqual
                | Token::PowerEqual
                | Token::AmpersandEqual
                | Token::PipeEqual
                | Token::CaretEqual
                | Token::LeftShiftEqual
        )
    }
}

pub struct TokenStream<'a> {
    lexer: Lexer<'a, Token>,
    peeked: Option<(Token, Span)>,
}

/// Saved position in the token stream for backtracking
#[derive(Clone)]
pub struct TokenStreamPosition<'a> {
    lexer: Lexer<'a, Token>,
    peeked: Option<(Token, Span)>,
}

impl<'a> TokenStream<'a> {
    #[must_use]
    pub fn new(input: &'a str) -> Self {
        Self {
            lexer: Token::lexer(input),
            peeked: None,
        }
    }

    /// Save the current position for later restoration
    #[must_use]
    pub fn position(&self) -> TokenStreamPosition<'a> {
        TokenStreamPosition {
            lexer: self.lexer.clone(),
            peeked: self.peeked.clone(),
        }
    }

    /// Restore a previously saved position
    pub fn set_position(&mut self, pos: TokenStreamPosition<'a>) {
        self.lexer = pos.lexer;
        self.peeked = pos.peeked;
    }

    #[allow(clippy::should_implement_trait)]
    pub fn next(&mut self) -> Option<(Token, Span)> {
        if let Some(peeked4.62k) = self.peeked.take() {
            return Some(peeked);
        }

        self.lexer.next().map(|result| {5.12k
            let token = result.unwrap_or(Token::Bang); // Error recovery
            let span = Span::new(self.lexer.span().start, self.lexer.span().end);
            (token, span)
        })
    }

    pub fn peek(&mut self) -> Option<&(Token, Span)> {
        if self.peeked.is_none() {
            self.peeked = self.next();
        }4.83k
        self.peeked.as_ref()
    }

    pub fn peek_nth(&mut self, n: usize) -> Option<(Token, Span)> {
        // For simplicity, we'll only support peek_nth(1) by cloning the lexer
        if n == 1 {
            let saved_peeked = self.peeked.clone();
            let saved_lexer = self.lexer.clone();

            // Get first token
            let _first = self.peek();
            self.advance();

            // Get second token
            let result = self.peek().cloned();

            // Restore state
            self.lexer = saved_lexer;
            self.peeked = saved_peeked;

            result
        } else {
            None // Not supported for n > 1
        }
    }

    pub fn peek_nth_is_colon(&mut self, n: usize) -> bool {
        if n == 0 {
            self.peek().is_some_and(|(t, _)| matches!(t, Token::Colon))
        } else {
            self.peek_nth(n)
                .is_some_and(|(t, _)| matches!(t, Token::Colon))
        }
    }

    /// Expect a specific token and return its span
    ///
    /// # Errors
    ///
    /// Returns an error if the next token doesn't match the expected token or if we reached EOF
    pub fn expect(&mut self, expected: &Token) -> anyhow::Result<Span> {
        match self.next() {
            Some((token, span)) if token == *expected => Ok(span),
            Some((token, _)) => anyhow::bail!("Expected {:?}, found {:?}", expected, token),
            None => anyhow::bail!("Expected {:?}, found EOF", expected),
        }
    }

    // Alias for next() to avoid clippy warning about Iterator trait
    pub fn advance(&mut self) -> Option<(Token, Span)> {
        self.next()
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used)]
#[allow(clippy::panic)]
mod tests {
    use super::*;
    use proptest::prelude::*;

    #[test]
    #[allow(clippy::approx_constant)] // Intentional literal for test
    fn test_tokenize_basic() {
        let mut stream = TokenStream::new("let x = 42 + 3.14");

        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Let));
        assert_eq!(
            stream.next().map(|(t, _)| t),
            Some(Token::Identifier("x".to_string()))
        );
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Equal));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Integer(42)));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Plus));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Float(3.14))); // Intentional literal for test
        assert_eq!(stream.next().map(|(t, _)| t), None);
    }

    #[test]
    fn test_tokenize_pipeline() {
        let mut stream = TokenStream::new("[1, 2, 3] >> map(|x| x * 2)");

        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::LeftBracket));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Integer(1)));
        // ... rest of tokens
    }

    #[test]
    fn test_tokenize_comments() {
        let mut stream = TokenStream::new("x // comment\n+ /* block */ y");

        assert_eq!(
            stream.next().map(|(t, _)| t),
            Some(Token::Identifier("x".to_string()))
        );
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Plus));
        assert_eq!(
            stream.next().map(|(t, _)| t),
            Some(Token::Identifier("y".to_string()))
        );
    }

    proptest! {
        #[test]
        fn test_tokenize_identifiers(s in "[a-zA-Z_][a-zA-Z0-9_]{0,100}") {
            // Skip reserved keywords that should tokenize as their respective tokens
            let reserved_keywords = [
                "true", "false", "fun", "fn", "let", "var", "mod", "if", "else", "match",
                "for", "in", "while", "loop", "async", "await", "throw", "try", "catch",
                "return", "command", "Ok", "Err", "Some", "None", "null", "Result", "Option",
                "break", "continue", "struct", "enum", "impl", "trait", "extend", "actor",
                "state", "receive", "send", "ask", "type", "where", "const", "static",
                "mut", "pub", "import", "use", "as", "module", "export", "df"
            ];
            
            if reserved_keywords.contains(&s.as_str()) {
                return Ok(()); // Skip test for reserved keywords
            }
            
            let mut stream = TokenStream::new(&s);
            match stream.advance() {
                Some((Token::Identifier(id), _)) => prop_assert_eq!(id, s),
                Some((Token::Underscore, _)) if s == "_" => {}, // Special case for underscore
                _ => panic!("Failed to tokenize identifier: {s}"),
            }
        }

        #[test]
        fn test_tokenize_integers(n in 0i64..1_000_000) {
            let s = n.to_string();
            let mut stream = TokenStream::new(&s);
            match stream.advance() {
                Some((Token::Integer(i), _)) => prop_assert_eq!(i, n),
                _ => panic!("Failed to tokenize integer"),
            }
        }
    }
}

Coverage Report

Created: 2025-09-05 15:26

Line	Count	Source
1		//! Lexical analysis and tokenization
2
3		use crate::frontend::ast::Span;
4		use logos::{Lexer, Logos};
5
6		/// Process escape sequences in a string literal
7	110	fn process_escapes(s: &str) -> String {
8	110	let mut result = String::new();
9	110	let mut chars = s.chars();
10
11	1.66k	while let Some( ch1.55k ) = chars.next() {
12	1.55k	if ch == '\\' {
13	1	match chars.next() {
14	1	Some('n') => result.push('\n'),
15	0	Some('t') => result.push('\t'),
16	0	Some('r') => result.push('\r'),
17	0	Some('\\') \| None => result.push('\\'), // Backslash or end of string
18	0	Some('"') => result.push('"'),
19	0	Some('\'') => result.push('\''),
20	0	Some('0') => result.push('\0'),
21	0	Some('u') if chars.as_str().starts_with('{') => {
22		// Unicode escape: \u{XXXX}
23	0	chars.next(); // consume '{'
24	0	let mut hex = String::new();
25	0	for hex_char in chars.by_ref() {
26	0	if hex_char == '}' {
27	0	break;
28	0	}
29	0	hex.push(hex_char);
30		}
31
32	0	if let Ok(code_point) = u32::from_str_radix(&hex, 16) {
33	0	if let Some(unicode_char) = char::from_u32(code_point) {
34	0	result.push(unicode_char);
35	0	} else {
36	0	// Invalid Unicode code point, keep as literal
37	0	result.push_str("\\u{");
38	0	result.push_str(&hex);
39	0	result.push('}');
40	0	}
41	0	} else {
42	0	// Invalid hex, keep as literal
43	0	result.push_str("\\u{");
44	0	result.push_str(&hex);
45	0	result.push('}');
46	0	}
47		}
48	0	Some(other) => {
49	0	// Unknown escape sequence, keep as literal
50	0	result.push('\\');
51	0	result.push(other);
52	0	}
53		}
54	1.55k	} else {
55	1.55k	result.push(ch);
56	1.55k	}
57		}
58
59	110	result
60	110	}
61
62	1.73k	#[derive(Logos, Debug, PartialEq, Clone)]
63		#[logos(skip r"[ \t\n\f]+")]
64		#[logos(skip r"//[^\n]*")]
65		#[logos(skip r"/\([^]\|\[^/])\*/")]
66		pub enum Token {
67		// Literals
68		#[regex(r"[0-9]+(?:i8\|i16\|i32\|i64\|i128\|isize\|u8\|u16\|u32\|u64\|u128\|usize)?", \|lex\| {
69		let slice = lex.slice();
70		// Strip type suffix for parsing the numeric value
71	1.18k	let (num_part, _type_suffix) = if let Some(pos) = slice.find(\|c: char\| c.is_alphabetic()) {
72		(&slice[..pos], Some(&slice[pos..]))
73		} else {
74		(slice, None)
75		};
76		num_part.parse::<i64>().ok()
77		})]
78		Integer(i64),
79
80		#[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", \|lex\| lex.slice().parse::<f64>().ok())]
81		Float(f64),
82
83		#[regex(r#""([^"\\]\|\\.)*""#, \|lex\| {
84		let s = lex.slice();
85		let inner = &s[1..s.len()-1];
86		Some(process_escapes(inner))
87		})]
88		String(String),
89
90		#[regex(r#"f"([^"\\]\|\\.)*""#, \|lex\| {
91		let s = lex.slice();
92		// Remove f" prefix and " suffix
93		let inner = &s[2..s.len()-1];
94		Some(process_escapes(inner))
95		})]
96		FString(String),
97
98		#[regex(r#"r"([^"])*""#, \|lex\| {
99		let s = lex.slice();
100		// Remove r" prefix and " suffix - no escape processing for raw strings
101		Some(s[2..s.len()-1].to_string())
102		})]
103		RawString(String),
104
105		#[regex(r"'([^'\\]\|\\.)'", \|lex\| {
106		let s = lex.slice();
107		let inner = &s[1..s.len()-1];
108		if inner.len() == 1 {
109		inner.chars().next()
110		} else if inner.starts_with('\\') && inner.len() == 2 {
111		match inner.chars().nth(1) {
112		Some('n') => Some('\n'),
113		Some('t') => Some('\t'),
114		Some('r') => Some('\r'),
115		Some('\\') => Some('\\'),
116		Some('\'') => Some('\''),
117		Some('0') => Some('\0'),
118		_ => None,
119		}
120		} else {
121		None
122		}
123		})]
124		Char(char),
125
126		#[token("true", \|_\| true)]
127		#[token("false", \|_\| false)]
128		Bool(bool),
129
130		// Keywords
131		#[token("fun")]
132		Fun,
133		#[token("fn")]
134		Fn,
135		#[token("let")]
136		Let,
137		#[token("var")]
138		Var,
139		#[token("mod")]
140		Mod,
141		#[token("if")]
142		If,
143		#[token("else")]
144		Else,
145		#[token("match")]
146		Match,
147		#[token("for")]
148		For,
149		#[token("in")]
150		In,
151		#[token("while")]
152		While,
153		#[token("loop")]
154		Loop,
155		#[token("async")]
156		Async,
157		#[token("await")]
158		Await,
159		#[token("throw")]
160		Throw,
161		#[token("try")]
162		Try,
163		#[token("catch")]
164		Catch,
165		#[token("return")]
166		Return,
167		#[token("command")]
168		Command,
169		#[token("Ok")]
170		Ok,
171		#[token("Err")]
172		Err,
173		#[token("Some")]
174		Some,
175		#[token("None")]
176		None,
177		#[token("null")]
178		Null,
179		#[token("Result")]
180		Result,
181		#[token("Option")]
182		Option,
183		#[token("break")]
184		Break,
185		#[token("continue")]
186		Continue,
187		#[token("struct")]
188		Struct,
189		#[token("enum")]
190		Enum,
191		#[token("impl")]
192		Impl,
193		#[token("trait")]
194		Trait,
195		#[token("extend")]
196		Extend,
197		#[token("actor")]
198		Actor,
199		#[token("state")]
200		State,
201		#[token("receive")]
202		Receive,
203		#[token("send")]
204		Send,
205		#[token("ask")]
206		Ask,
207		#[token("type")]
208		Type,
209		#[token("where")]
210		Where,
211		#[token("const")]
212		Const,
213		#[token("static")]
214		Static,
215		#[token("mut")]
216		Mut,
217		#[token("pub")]
218		Pub,
219		#[token("import")]
220		Import,
221		#[token("use")]
222		Use,
223		#[token("as")]
224		As,
225		#[token("module")]
226		Module,
227		#[token("export")]
228		Export,
229		#[token("df", priority = 2)]
230		DataFrame,
231
232		// Identifiers (lower priority than keywords)
233		#[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", \|lex\| lex.slice().to_string(), priority = 1)]
234		Identifier(String),
235
236		// Operators
237		#[token("+")]
238		Plus,
239		#[token("-")]
240		Minus,
241		#[token("*")]
242		Star,
243		#[token("/")]
244		Slash,
245		#[token("%")]
246		Percent,
247		#[token("**")]
248		Power,
249
250		#[token("==")]
251		EqualEqual,
252		#[token("!=")]
253		NotEqual,
254		#[token("<?")]
255		ActorQuery,
256		#[token("<-")]
257		LeftArrow,
258		#[token("<")]
259		Less,
260		#[token("<=")]
261		LessEqual,
262		#[token(">")]
263		Greater,
264		#[token(">=")]
265		GreaterEqual,
266
267		#[token("&&")]
268		AndAnd,
269		#[token("\|\|")]
270		OrOr,
271		#[token("!")]
272		Bang,
273
274		#[token("&")]
275		Ampersand,
276		#[token("\|")]
277		Pipe,
278		#[token("^")]
279		Caret,
280		#[token("~")]
281		Tilde,
282		#[token("\\")]
283		Backslash,
284		#[token("<<")]
285		LeftShift,
286
287		#[token("=")]
288		Equal,
289		#[token("+=")]
290		PlusEqual,
291		#[token("-=")]
292		MinusEqual,
293		#[token("*=")]
294		StarEqual,
295		#[token("/=")]
296		SlashEqual,
297		#[token("%=")]
298		PercentEqual,
299		#[token("**=")]
300		PowerEqual,
301		#[token("&=")]
302		AmpersandEqual,
303		#[token("\|=")]
304		PipeEqual,
305		#[token("^=")]
306		CaretEqual,
307		#[token("<<=")]
308		LeftShiftEqual,
309
310		#[token("++")]
311		Increment,
312		#[token("--")]
313		Decrement,
314
315		#[token("\|>")]
316		Pipeline,
317		#[token("->")]
318		Arrow,
319		#[token("=>")]
320		FatArrow,
321		#[token("..")]
322		DotDot,
323		#[token("..=")]
324		DotDotEqual,
325		#[token("...")]
326		DotDotDot,
327		#[token("??")]
328		NullCoalesce,
329		#[token("?")]
330		Question,
331		#[token("?.")]
332		SafeNav,
333
334		// Delimiters
335		#[token("(")]
336		LeftParen,
337		#[token(")")]
338		RightParen,
339		#[token("[")]
340		LeftBracket,
341		#[token("]")]
342		RightBracket,
343		#[token("{")]
344		LeftBrace,
345		#[token("}")]
346		RightBrace,
347
348		// Punctuation
349		#[token(",")]
350		Comma,
351		#[token(".")]
352		Dot,
353		#[token(":")]
354		Colon,
355		#[token("::")]
356		ColonColon,
357		#[token(";")]
358		Semicolon,
359		#[token("_", priority = 2)]
360		Underscore,
361
362		// Attribute support
363		#[token("#")]
364		Hash,
365		}
366
367		impl Token {
368		#[must_use]
369	1.14k	pub fn is_binary_op(&self) -> bool {
370	566	matches!(
371	1.14k	self,
372		Token::Plus
373		\| Token::Minus
374		\| Token::Star
375		\| Token::Slash
376		\| Token::Percent
377		\| Token::Power
378		\| Token::EqualEqual
379		\| Token::NotEqual
380		\| Token::Less
381		\| Token::LessEqual
382		\| Token::Greater
383		\| Token::GreaterEqual
384		\| Token::AndAnd
385		\| Token::OrOr
386		\| Token::Ampersand
387		\| Token::Pipe
388		\| Token::Caret
389		\| Token::LeftShift
390		)
391	1.14k	}
392
393		#[must_use]
394	0	pub fn is_unary_op(&self) -> bool {
395	0	matches!(
396	0	self,
397		Token::Bang \| Token::Minus \| Token::Tilde \| Token::Ampersand
398		)
399	0	}
400
401		#[must_use]
402	646	pub fn is_assignment_op(&self) -> bool {
403	646	matches!(
404	646	self,
405		Token::Equal
406		\| Token::PlusEqual
407		\| Token::MinusEqual
408		\| Token::StarEqual
409		\| Token::SlashEqual
410		\| Token::PercentEqual
411		\| Token::PowerEqual
412		\| Token::AmpersandEqual
413		\| Token::PipeEqual
414		\| Token::CaretEqual
415		\| Token::LeftShiftEqual
416		)
417	646	}
418		}
419
420		pub struct TokenStream<'a> {
421		lexer: Lexer<'a, Token>,
422		peeked: Option<(Token, Span)>,
423		}
424
425		/// Saved position in the token stream for backtracking
426		#[derive(Clone)]
427		pub struct TokenStreamPosition<'a> {
428		lexer: Lexer<'a, Token>,
429		peeked: Option<(Token, Span)>,
430		}
431
432		impl<'a> TokenStream<'a> {
433		#[must_use]
434	1.09k	pub fn new(input: &'a str) -> Self {
435	1.09k	Self {
436	1.09k	lexer: Token::lexer(input),
437	1.09k	peeked: None,
438	1.09k	}
439	1.09k	}
440
441		/// Save the current position for later restoration
442		#[must_use]
443	54	pub fn position(&self) -> TokenStreamPosition<'a> {
444	54	TokenStreamPosition {
445	54	lexer: self.lexer.clone(),
446	54	peeked: self.peeked.clone(),
447	54	}
448	54	}
449
450		/// Restore a previously saved position
451	49	pub fn set_position(&mut self, pos: TokenStreamPosition<'a>) {
452	49	self.lexer = pos.lexer;
453	49	self.peeked = pos.peeked;
454	49	}
455
456		#[allow(clippy::should_implement_trait)]
457	12.4k	pub fn next(&mut self) -> Option<(Token, Span)> {
458	12.4k	if let Some( peeked4.62k ) = self.peeked.take() {
459	4.62k	return Some(peeked);
460	7.78k	}
461
462	7.78k	self.lexer.next().map(\|result\| {5.12k
463	5.12k	let token = result.unwrap_or(Token::Bang); // Error recovery
464	5.12k	let span = Span::new(self.lexer.span().start, self.lexer.span().end);
465	5.12k	(token, span)
466	5.12k	})
467	12.4k	}
468
469	12.4k	pub fn peek(&mut self) -> Option<&(Token, Span)> {
470	12.4k	if self.peeked.is_none() {
471	7.66k	self.peeked = self.next();
472	7.66k	}4.83k
473	12.4k	self.peeked.as_ref()
474	12.4k	}
475
476	0	pub fn peek_nth(&mut self, n: usize) -> Option<(Token, Span)> {
477		// For simplicity, we'll only support peek_nth(1) by cloning the lexer
478	0	if n == 1 {
479	0	let saved_peeked = self.peeked.clone();
480	0	let saved_lexer = self.lexer.clone();
481
482		// Get first token
483	0	let _first = self.peek();
484	0	self.advance();
485
486		// Get second token
487	0	let result = self.peek().cloned();
488
489		// Restore state
490	0	self.lexer = saved_lexer;
491	0	self.peeked = saved_peeked;
492
493	0	result
494		} else {
495	0	None // Not supported for n > 1
496		}
497	0	}
498
499	0	pub fn peek_nth_is_colon(&mut self, n: usize) -> bool {
500	0	if n == 0 {
501	0	self.peek().is_some_and(\|(t, _)\| matches!(t, Token::Colon))
502		} else {
503	0	self.peek_nth(n)
504	0	.is_some_and(\|(t, _)\| matches!(t, Token::Colon))
505		}
506	0	}
507
508		/// Expect a specific token and return its span
509		///
510		/// # Errors
511		///
512		/// Returns an error if the next token doesn't match the expected token or if we reached EOF
513	804	pub fn expect(&mut self, expected: &Token) -> anyhow::Result<Span> {
514	804	match self.next() {
515	804	Some((token, span)) if token == *expected => Ok(span),
516	0	Some((token, _)) => anyhow::bail!("Expected {:?}, found {:?}", expected, token),
517	0	None => anyhow::bail!("Expected {:?}, found EOF", expected),
518		}
519	804	}
520
521		// Alias for next() to avoid clippy warning about Iterator trait
522	3.93k	pub fn advance(&mut self) -> Option<(Token, Span)> {
523	3.93k	self.next()
524	3.93k	}
525		}
526
527		#[cfg(test)]
528		#[allow(clippy::unwrap_used)]
529		#[allow(clippy::panic)]
530		mod tests {
531		use super::*;
532		use proptest::prelude::*;
533
534		#[test]
535		#[allow(clippy::approx_constant)] // Intentional literal for test
536	1	fn test_tokenize_basic() {
537	1	let mut stream = TokenStream::new("let x = 42 + 3.14");
538
539	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Let));
540	1	assert_eq!(
541	1	stream.next().map(\|(t, _)\| t),
542	1	Some(Token::Identifier("x".to_string()))
543		);
544	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Equal));
545	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Integer(42)));
546	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Plus));
547	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Float(3.14))); // Intentional literal for test
548	1	assert_eq!(stream.next().map(\|(t, _)\| t), None);
549	1	}
550
551		#[test]
552	1	fn test_tokenize_pipeline() {
553	1	let mut stream = TokenStream::new("[1, 2, 3] >> map(\|x\| x * 2)");
554
555	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::LeftBracket));
556	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Integer(1)));
557		// ... rest of tokens
558	1	}
559
560		#[test]
561	1	fn test_tokenize_comments() {
562	1	let mut stream = TokenStream::new("x // comment\n+ /* block */ y");
563
564	1	assert_eq!(
565	1	stream.next().map(\|(t, _)\| t),
566	1	Some(Token::Identifier("x".to_string()))
567		);
568	1	assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Plus));
569	1	assert_eq!(
570	1	stream.next().map(\|(t, _)\| t),
571	1	Some(Token::Identifier("y".to_string()))
572		);
573	1	}
574
575		proptest! {
576		#[test]
577		fn test_tokenize_identifiers(s in "[a-zA-Z_][a-zA-Z0-9_]{0,100}") {
578		// Skip reserved keywords that should tokenize as their respective tokens
579		let reserved_keywords = [
580		"true", "false", "fun", "fn", "let", "var", "mod", "if", "else", "match",
581		"for", "in", "while", "loop", "async", "await", "throw", "try", "catch",
582		"return", "command", "Ok", "Err", "Some", "None", "null", "Result", "Option",
583		"break", "continue", "struct", "enum", "impl", "trait", "extend", "actor",
584		"state", "receive", "send", "ask", "type", "where", "const", "static",
585		"mut", "pub", "import", "use", "as", "module", "export", "df"
586		];
587
588		if reserved_keywords.contains(&s.as_str()) {
589		return Ok(()); // Skip test for reserved keywords
590		}
591
592		let mut stream = TokenStream::new(&s);
593		match stream.advance() {
594		Some((Token::Identifier(id), _)) => prop_assert_eq!(id, s),
595		Some((Token::Underscore, _)) if s == "_" => {}, // Special case for underscore
596		_ => panic!("Failed to tokenize identifier: {s}"),
597		}
598		}
599
600		#[test]
601		fn test_tokenize_integers(n in 0i64..1_000_000) {
602		let s = n.to_string();
603		let mut stream = TokenStream::new(&s);
604		match stream.advance() {
605		Some((Token::Integer(i), _)) => prop_assert_eq!(i, n),
606		_ => panic!("Failed to tokenize integer"),
607		}
608		}
609		}
610		}