/home/noah/src/ruchy/src/frontend/lexer.rs

Source
//! Lexical analysis and tokenization

use crate::frontend::ast::Span;
use logos::{Lexer, Logos};

/// Process a basic escape character
fn process_basic_escape(ch: char) -> Option<char> {
    match ch {
        'n' => Some('\n'),
        't' => Some('\t'),
        'r' => Some('\r'),
        '\\' => Some('\\'),
        '"' => Some('"'),
        '\'' => Some('\''),
        '0' => Some('\0'),
        _ => None,
    }
}

/// Process a Unicode escape sequence
fn process_unicode_escape(chars: &mut std::str::Chars) -> String {
    chars.next(); // consume '{'
    
    let mut hex = String::new();
    for hex_char in chars.by_ref() {
        if hex_char == '}' {
            break;
        }
        hex.push(hex_char);
    }

    if let Ok(code_point) = u32::from_str_radix(&hex, 16) {
        if let Some(unicode_char) = char::from_u32(code_point) {
            return unicode_char.to_string();
        }
    }
    
    // Invalid code point or hex, keep as literal
    format!("\\u{{{hex}}}")
}

/// Process escape sequences in a string literal
fn process_escapes(s: &str) -> String {
    let mut result = String::new();
    let mut chars = s.chars();

    while let Some(ch) = chars.next() {
        if ch == '\\' {
            match chars.next() {
                None => result.push('\\'), // End of string
                Some('u') if chars.as_str().starts_with('{') => {
                    result.push_str(&process_unicode_escape(&mut chars));
                }
                Some(escape_ch) => {
                    if let Some(escaped) = process_basic_escape(escape_ch) {
                        result.push(escaped);
                    } else {
                        // Unknown escape sequence, keep as literal
                        result.push('\\');
                        result.push(escape_ch);
                    }
                }
            }
        } else {
            result.push(ch);
        }
    }

    result
}

#[derive(Logos, Debug, PartialEq, Clone)]
#[logos(skip r"[ \t\n\f]+")]
#[logos(skip r"//[^\n]*")]
#[logos(skip r"/\*([^*]|\*[^/])*\*/")]
pub enum Token {
    // Literals
    #[regex(r"[0-9]+(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?", |lex| {
        let slice = lex.slice();
        // Strip type suffix for parsing the numeric value
        let (num_part, _type_suffix) = if let Some(pos) = slice.find(|c: char| c.is_alphabetic()) {
            (&slice[..pos], Some(&slice[pos..]))
        } else {
            (slice, None)
        };
        num_part.parse::<i64>().ok()
    })]
    Integer(i64),

    #[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", |lex| lex.slice().parse::<f64>().ok())]
    Float(f64),

    #[regex(r#""([^"\\]|\\.)*""#, |lex| {
        let s = lex.slice();
        let inner = &s[1..s.len()-1];
        Some(process_escapes(inner))
    })]
    String(String),

    #[regex(r#"f"([^"\\]|\\.)*""#, |lex| {
        let s = lex.slice();
        // Remove f" prefix and " suffix
        let inner = &s[2..s.len()-1];
        Some(process_escapes(inner))
    })]
    FString(String),

    #[regex(r#"r"([^"])*""#, |lex| {
        let s = lex.slice();
        // Remove r" prefix and " suffix - no escape processing for raw strings
        Some(s[2..s.len()-1].to_string())
    })]
    RawString(String),

    #[regex(r"'([^'\\]|\\.)'", |lex| {
        let s = lex.slice();
        let inner = &s[1..s.len()-1];
        if inner.len() == 1 {
            inner.chars().next()
        } else if inner.starts_with('\\') && inner.len() == 2 {
            match inner.chars().nth(1) {
                Some('n') => Some('\n'),
                Some('t') => Some('\t'),
                Some('r') => Some('\r'),
                Some('\\') => Some('\\'),
                Some('\'') => Some('\''),
                Some('0') => Some('\0'),
                _ => None,
            }
        } else {
            None
        }
    })]
    Char(char),

    #[token("true", |_| true)]
    #[token("false", |_| false)]
    Bool(bool),

    // Keywords
    #[token("fun")]
    Fun,
    #[token("fn")]
    Fn,
    #[token("let")]
    Let,
    #[token("var")]
    Var,
    #[token("mod")]
    Mod,
    #[token("if")]
    If,
    #[token("else")]
    Else,
    #[token("match")]
    Match,
    #[token("for")]
    For,
    #[token("in")]
    In,
    #[token("while")]
    While,
    #[token("loop")]
    Loop,
    #[token("async")]
    Async,
    #[token("await")]
    Await,
    #[token("throw")]
    Throw,
    #[token("try")]
    Try,
    #[token("catch")]
    Catch,
    #[token("finally")]
    Finally,
    #[token("return")]
    Return,
    #[token("command")]
    Command,
    #[token("Ok")]
    Ok,
    #[token("Err")]
    Err,
    #[token("Some")]
    Some,
    #[token("None")]
    None,
    #[token("null")]
    Null,
    #[token("Result")]
    Result,
    #[token("Option")]
    Option,
    #[token("break")]
    Break,
    #[token("continue")]
    Continue,
    #[token("struct")]
    Struct,
    #[token("enum")]
    Enum,
    #[token("impl")]
    Impl,
    #[token("trait")]
    Trait,
    #[token("extend")]
    Extend,
    #[token("actor")]
    Actor,
    #[token("state")]
    State,
    #[token("receive")]
    Receive,
    #[token("send")]
    Send,
    #[token("ask")]
    Ask,
    #[token("type")]
    Type,
    #[token("where")]
    Where,
    #[token("const")]
    Const,
    #[token("static")]
    Static,
    #[token("mut")]
    Mut,
    #[token("pub")]
    Pub,
    #[token("import")]
    Import,
    #[token("use")]
    Use,
    #[token("as")]
    As,
    #[token("module")]
    Module,
    #[token("export")]
    Export,
    #[token("df", priority = 2)]
    DataFrame,

    // Identifiers (lower priority than keywords)
    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
    Identifier(String),

    // Operators
    #[token("+")]
    Plus,
    #[token("-")]
    Minus,
    #[token("*")]
    Star,
    #[token("/")]
    Slash,
    #[token("%")]
    Percent,
    #[token("**")]
    Power,

    #[token("==")]
    EqualEqual,
    #[token("!=")]
    NotEqual,
    #[token("<?")]
    ActorQuery,
    #[token("<-")]
    LeftArrow,
    #[token("<")]
    Less,
    #[token("<=")]
    LessEqual,
    #[token(">")]
    Greater,
    #[token(">=")]
    GreaterEqual,

    #[token("&&")]
    AndAnd,
    #[token("||")]
    OrOr,
    #[token("!")]
    Bang,

    #[token("&")]
    Ampersand,
    #[token("|")]
    Pipe,
    #[token("^")]
    Caret,
    #[token("~")]
    Tilde,
    #[token("\\")]
    Backslash,
    #[token("<<")]
    LeftShift,

    #[token("=")]
    Equal,
    #[token("+=")]
    PlusEqual,
    #[token("-=")]
    MinusEqual,
    #[token("*=")]
    StarEqual,
    #[token("/=")]
    SlashEqual,
    #[token("%=")]
    PercentEqual,
    #[token("**=")]
    PowerEqual,
    #[token("&=")]
    AmpersandEqual,
    #[token("|=")]
    PipeEqual,
    #[token("^=")]
    CaretEqual,
    #[token("<<=")]
    LeftShiftEqual,

    #[token("++")]
    Increment,
    #[token("--")]
    Decrement,

    #[token("|>")]
    Pipeline,
    #[token("->")]
    Arrow,
    #[token("=>")]
    FatArrow,
    #[token("..")]
    DotDot,
    #[token("..=")]
    DotDotEqual,
    #[token("...")]
    DotDotDot,
    #[token("??")]
    NullCoalesce,
    #[token("?")]
    Question,
    #[token("?.")]
    SafeNav,

    // Delimiters
    #[token("(")]
    LeftParen,
    #[token(")")]
    RightParen,
    #[token("[")]
    LeftBracket,
    #[token("]")]
    RightBracket,
    #[token("{")]
    LeftBrace,
    #[token("}")]
    RightBrace,

    // Punctuation
    #[token(",")]
    Comma,
    #[token(".")]
    Dot,
    #[token(":")]
    Colon,
    #[token("::")]
    ColonColon,
    #[token(";")]
    Semicolon,
    #[token("_", priority = 2)]
    Underscore,

    // Attribute support
    #[token("#")]
    Hash,
}

impl Token {
    #[must_use]
    pub fn is_binary_op(&self) -> bool {
        matches!(
            self,
            Token::Plus
                | Token::Minus
                | Token::Star
                | Token::Slash
                | Token::Percent
                | Token::Power
                | Token::EqualEqual
                | Token::NotEqual
                | Token::Less
                | Token::LessEqual
                | Token::Greater
                | Token::GreaterEqual
                | Token::AndAnd
                | Token::OrOr
                | Token::Ampersand
                | Token::Pipe
                | Token::Caret
                | Token::LeftShift
        )
    }

    #[must_use]
    pub fn is_unary_op(&self) -> bool {
        matches!(
            self,
            Token::Bang | Token::Minus | Token::Tilde | Token::Ampersand
        )
    }

    #[must_use]
    pub fn is_assignment_op(&self) -> bool {
        matches!(
            self,
            Token::Equal
                | Token::PlusEqual
                | Token::MinusEqual
                | Token::StarEqual
                | Token::SlashEqual
                | Token::PercentEqual
                | Token::PowerEqual
                | Token::AmpersandEqual
                | Token::PipeEqual
                | Token::CaretEqual
                | Token::LeftShiftEqual
        )
    }
}

pub struct TokenStream<'a> {
    lexer: Lexer<'a, Token>,
    peeked: Option<(Token, Span)>,
}

/// Saved position in the token stream for backtracking
#[derive(Clone)]
pub struct TokenStreamPosition<'a> {
    lexer: Lexer<'a, Token>,
    peeked: Option<(Token, Span)>,
}

impl<'a> TokenStream<'a> {
    #[must_use]
    pub fn new(input: &'a str) -> Self {
        Self {
            lexer: Token::lexer(input),
            peeked: None,
        }
    }

    /// Save the current position for later restoration
    #[must_use]
    pub fn position(&self) -> TokenStreamPosition<'a> {
        TokenStreamPosition {
            lexer: self.lexer.clone(),
            peeked: self.peeked.clone(),
        }
    }

    /// Restore a previously saved position
    pub fn set_position(&mut self, pos: TokenStreamPosition<'a>) {
        self.lexer = pos.lexer;
        self.peeked = pos.peeked;
    }

    #[allow(clippy::should_implement_trait)]
    pub fn next(&mut self) -> Option<(Token, Span)> {
        if let Some(peeked) = self.peeked.take() {
            return Some(peeked);
        }

        self.lexer.next().map(|result| {
            let token = result.unwrap_or(Token::Bang); // Error recovery
            let span = Span::new(self.lexer.span().start, self.lexer.span().end);
            (token, span)
        })
    }

    pub fn peek(&mut self) -> Option<&(Token, Span)> {
        if self.peeked.is_none() {
            self.peeked = self.next();
        }
        self.peeked.as_ref()
    }

    pub fn peek_nth(&mut self, n: usize) -> Option<(Token, Span)> {
        // For simplicity, we'll only support peek_nth(1) by cloning the lexer
        if n == 1 {
            let saved_peeked = self.peeked.clone();
            let saved_lexer = self.lexer.clone();

            // Get first token
            let _first = self.peek();
            self.advance();

            // Get second token
            let result = self.peek().cloned();

            // Restore state
            self.lexer = saved_lexer;
            self.peeked = saved_peeked;

            result
        } else {
            None // Not supported for n > 1
        }
    }

    pub fn peek_nth_is_colon(&mut self, n: usize) -> bool {
        if n == 0 {
            self.peek().is_some_and(|(t, _)| matches!(t, Token::Colon))
        } else {
            self.peek_nth(n)
                .is_some_and(|(t, _)| matches!(t, Token::Colon))
        }
    }

    /// Expect a specific token and return its span
    ///
    /// # Errors
    ///
    /// Returns an error if the next token doesn't match the expected token or if we reached EOF
    pub fn expect(&mut self, expected: &Token) -> anyhow::Result<Span> {
        match self.next() {
            Some((token, span)) if token == *expected => Ok(span),
            Some((token, _)) => anyhow::bail!("Expected {:?}, found {:?}", expected, token),
            None => anyhow::bail!("Expected {:?}, found EOF", expected),
        }
    }

    // Alias for next() to avoid clippy warning about Iterator trait
    pub fn advance(&mut self) -> Option<(Token, Span)> {
        self.next()
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used)]
#[allow(clippy::panic)]
mod tests {
    use super::*;
    use proptest::prelude::*;

    #[test]
    #[allow(clippy::approx_constant)] // Intentional literal for test
    fn test_tokenize_basic() {
        let mut stream = TokenStream::new("let x = 42 + 3.14");

        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Let));
        assert_eq!(
            stream.next().map(|(t, _)| t),
            Some(Token::Identifier("x".to_string()))
        );
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Equal));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Integer(42)));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Plus));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Float(3.14))); // Intentional literal for test
        assert_eq!(stream.next().map(|(t, _)| t), None);
    }

    #[test]
    fn test_tokenize_pipeline() {
        let mut stream = TokenStream::new("[1, 2, 3] >> map(|x| x * 2)");

        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::LeftBracket));
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Integer(1)));
        // ... rest of tokens
    }

    #[test]
    fn test_tokenize_comments() {
        let mut stream = TokenStream::new("x // comment\n+ /* block */ y");

        assert_eq!(
            stream.next().map(|(t, _)| t),
            Some(Token::Identifier("x".to_string()))
        );
        assert_eq!(stream.next().map(|(t, _)| t), Some(Token::Plus));
        assert_eq!(
            stream.next().map(|(t, _)| t),
            Some(Token::Identifier("y".to_string()))
        );
    }

    proptest! {
        #[test]
        fn test_tokenize_identifiers(s in "[a-zA-Z_][a-zA-Z0-9_]{0,100}") {
            // Skip reserved keywords that should tokenize as their respective tokens
            let reserved_keywords = [
                "true", "false", "fun", "fn", "let", "var", "mod", "if", "else", "match",
                "for", "in", "while", "loop", "async", "await", "throw", "try", "catch",
                "return", "command", "Ok", "Err", "Some", "None", "null", "Result", "Option",
                "break", "continue", "struct", "enum", "impl", "trait", "extend", "actor",
                "state", "receive", "send", "ask", "type", "where", "const", "static",
                "mut", "pub", "import", "use", "as", "module", "export", "df"
            ];
            
            if reserved_keywords.contains(&s.as_str()) {
                return Ok(()); // Skip test for reserved keywords
            }
            
            let mut stream = TokenStream::new(&s);
            match stream.advance() {
                Some((Token::Identifier(id), _)) => prop_assert_eq!(id, s),
                Some((Token::Underscore, _)) if s == "_" => {}, // Special case for underscore
                _ => panic!("Failed to tokenize identifier: {s}"),
            }
        }

        #[test]
        fn test_tokenize_integers(n in 0i64..1_000_000) {
            let s = n.to_string();
            let mut stream = TokenStream::new(&s);
            match stream.advance() {
                Some((Token::Integer(i), _)) => prop_assert_eq!(i, n),
                _ => panic!("Failed to tokenize integer"),
            }
        }
    }
}

Coverage Report

Created: 2025-09-08 21:26

Line	Count	Source
1		//! Lexical analysis and tokenization
2
3		use crate::frontend::ast::Span;
4		use logos::{Lexer, Logos};
5
6		/// Process a basic escape character
7	0	fn process_basic_escape(ch: char) -> Option<char> {
8	0	match ch {
9	0	'n' => Some('\n'),
10	0	't' => Some('\t'),
11	0	'r' => Some('\r'),
12	0	'\\' => Some('\\'),
13	0	'"' => Some('"'),
14	0	'\'' => Some('\''),
15	0	'0' => Some('\0'),
16	0	_ => None,
17		}
18	0	}
19
20		/// Process a Unicode escape sequence
21	0	fn process_unicode_escape(chars: &mut std::str::Chars) -> String {
22	0	chars.next(); // consume '{'
23
24	0	let mut hex = String::new();
25	0	for hex_char in chars.by_ref() {
26	0	if hex_char == '}' {
27	0	break;
28	0	}
29	0	hex.push(hex_char);
30		}
31
32	0	if let Ok(code_point) = u32::from_str_radix(&hex, 16) {
33	0	if let Some(unicode_char) = char::from_u32(code_point) {
34	0	return unicode_char.to_string();
35	0	}
36	0	}
37
38		// Invalid code point or hex, keep as literal
39	0	format!("\\u{{{hex}}}")
40	0	}
41
42		/// Process escape sequences in a string literal
43	0	fn process_escapes(s: &str) -> String {
44	0	let mut result = String::new();
45	0	let mut chars = s.chars();
46
47	0	while let Some(ch) = chars.next() {
48	0	if ch == '\\' {
49	0	match chars.next() {
50	0	None => result.push('\\'), // End of string
51	0	Some('u') if chars.as_str().starts_with('{') => {
52	0	result.push_str(&process_unicode_escape(&mut chars));
53	0	}
54	0	Some(escape_ch) => {
55	0	if let Some(escaped) = process_basic_escape(escape_ch) {
56	0	result.push(escaped);
57	0	} else {
58	0	// Unknown escape sequence, keep as literal
59	0	result.push('\\');
60	0	result.push(escape_ch);
61	0	}
62		}
63		}
64	0	} else {
65	0	result.push(ch);
66	0	}
67		}
68
69	0	result
70	0	}
71
72	0	#[derive(Logos, Debug, PartialEq, Clone)]
73		#[logos(skip r"[ \t\n\f]+")]
74		#[logos(skip r"//[^\n]*")]
75		#[logos(skip r"/\([^]\|\[^/])\*/")]
76		pub enum Token {
77		// Literals
78		#[regex(r"[0-9]+(?:i8\|i16\|i32\|i64\|i128\|isize\|u8\|u16\|u32\|u64\|u128\|usize)?", \|lex\| {
79		let slice = lex.slice();
80		// Strip type suffix for parsing the numeric value
81	0	let (num_part, _type_suffix) = if let Some(pos) = slice.find(\|c: char\| c.is_alphabetic()) {
82		(&slice[..pos], Some(&slice[pos..]))
83		} else {
84		(slice, None)
85		};
86		num_part.parse::<i64>().ok()
87		})]
88		Integer(i64),
89
90		#[regex(r"[0-9]+\.[0-9]+([eE][+-]?[0-9]+)?", \|lex\| lex.slice().parse::<f64>().ok())]
91		Float(f64),
92
93		#[regex(r#""([^"\\]\|\\.)*""#, \|lex\| {
94		let s = lex.slice();
95		let inner = &s[1..s.len()-1];
96		Some(process_escapes(inner))
97		})]
98		String(String),
99
100		#[regex(r#"f"([^"\\]\|\\.)*""#, \|lex\| {
101		let s = lex.slice();
102		// Remove f" prefix and " suffix
103		let inner = &s[2..s.len()-1];
104		Some(process_escapes(inner))
105		})]
106		FString(String),
107
108		#[regex(r#"r"([^"])*""#, \|lex\| {
109		let s = lex.slice();
110		// Remove r" prefix and " suffix - no escape processing for raw strings
111		Some(s[2..s.len()-1].to_string())
112		})]
113		RawString(String),
114
115		#[regex(r"'([^'\\]\|\\.)'", \|lex\| {
116		let s = lex.slice();
117		let inner = &s[1..s.len()-1];
118		if inner.len() == 1 {
119		inner.chars().next()
120		} else if inner.starts_with('\\') && inner.len() == 2 {
121		match inner.chars().nth(1) {
122		Some('n') => Some('\n'),
123		Some('t') => Some('\t'),
124		Some('r') => Some('\r'),
125		Some('\\') => Some('\\'),
126		Some('\'') => Some('\''),
127		Some('0') => Some('\0'),
128		_ => None,
129		}
130		} else {
131		None
132		}
133		})]
134		Char(char),
135
136		#[token("true", \|_\| true)]
137		#[token("false", \|_\| false)]
138		Bool(bool),
139
140		// Keywords
141		#[token("fun")]
142		Fun,
143		#[token("fn")]
144		Fn,
145		#[token("let")]
146		Let,
147		#[token("var")]
148		Var,
149		#[token("mod")]
150		Mod,
151		#[token("if")]
152		If,
153		#[token("else")]
154		Else,
155		#[token("match")]
156		Match,
157		#[token("for")]
158		For,
159		#[token("in")]
160		In,
161		#[token("while")]
162		While,
163		#[token("loop")]
164		Loop,
165		#[token("async")]
166		Async,
167		#[token("await")]
168		Await,
169		#[token("throw")]
170		Throw,
171		#[token("try")]
172		Try,
173		#[token("catch")]
174		Catch,
175		#[token("finally")]
176		Finally,
177		#[token("return")]
178		Return,
179		#[token("command")]
180		Command,
181		#[token("Ok")]
182		Ok,
183		#[token("Err")]
184		Err,
185		#[token("Some")]
186		Some,
187		#[token("None")]
188		None,
189		#[token("null")]
190		Null,
191		#[token("Result")]
192		Result,
193		#[token("Option")]
194		Option,
195		#[token("break")]
196		Break,
197		#[token("continue")]
198		Continue,
199		#[token("struct")]
200		Struct,
201		#[token("enum")]
202		Enum,
203		#[token("impl")]
204		Impl,
205		#[token("trait")]
206		Trait,
207		#[token("extend")]
208		Extend,
209		#[token("actor")]
210		Actor,
211		#[token("state")]
212		State,
213		#[token("receive")]
214		Receive,
215		#[token("send")]
216		Send,
217		#[token("ask")]
218		Ask,
219		#[token("type")]
220		Type,
221		#[token("where")]
222		Where,
223		#[token("const")]
224		Const,
225		#[token("static")]
226		Static,
227		#[token("mut")]
228		Mut,
229		#[token("pub")]
230		Pub,
231		#[token("import")]
232		Import,
233		#[token("use")]
234		Use,
235		#[token("as")]
236		As,
237		#[token("module")]
238		Module,
239		#[token("export")]
240		Export,
241		#[token("df", priority = 2)]
242		DataFrame,
243
244		// Identifiers (lower priority than keywords)
245		#[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", \|lex\| lex.slice().to_string(), priority = 1)]
246		Identifier(String),
247
248		// Operators
249		#[token("+")]
250		Plus,
251		#[token("-")]
252		Minus,
253		#[token("*")]
254		Star,
255		#[token("/")]
256		Slash,
257		#[token("%")]
258		Percent,
259		#[token("**")]
260		Power,
261
262		#[token("==")]
263		EqualEqual,
264		#[token("!=")]
265		NotEqual,
266		#[token("<?")]
267		ActorQuery,
268		#[token("<-")]
269		LeftArrow,
270		#[token("<")]
271		Less,
272		#[token("<=")]
273		LessEqual,
274		#[token(">")]
275		Greater,
276		#[token(">=")]
277		GreaterEqual,
278
279		#[token("&&")]
280		AndAnd,
281		#[token("\|\|")]
282		OrOr,
283		#[token("!")]
284		Bang,
285
286		#[token("&")]
287		Ampersand,
288		#[token("\|")]
289		Pipe,
290		#[token("^")]
291		Caret,
292		#[token("~")]
293		Tilde,
294		#[token("\\")]
295		Backslash,
296		#[token("<<")]
297		LeftShift,
298
299		#[token("=")]
300		Equal,
301		#[token("+=")]
302		PlusEqual,
303		#[token("-=")]
304		MinusEqual,
305		#[token("*=")]
306		StarEqual,
307		#[token("/=")]
308		SlashEqual,
309		#[token("%=")]
310		PercentEqual,
311		#[token("**=")]
312		PowerEqual,
313		#[token("&=")]
314		AmpersandEqual,
315		#[token("\|=")]
316		PipeEqual,
317		#[token("^=")]
318		CaretEqual,
319		#[token("<<=")]
320		LeftShiftEqual,
321
322		#[token("++")]
323		Increment,
324		#[token("--")]
325		Decrement,
326
327		#[token("\|>")]
328		Pipeline,
329		#[token("->")]
330		Arrow,
331		#[token("=>")]
332		FatArrow,
333		#[token("..")]
334		DotDot,
335		#[token("..=")]
336		DotDotEqual,
337		#[token("...")]
338		DotDotDot,
339		#[token("??")]
340		NullCoalesce,
341		#[token("?")]
342		Question,
343		#[token("?.")]
344		SafeNav,
345
346		// Delimiters
347		#[token("(")]
348		LeftParen,
349		#[token(")")]
350		RightParen,
351		#[token("[")]
352		LeftBracket,
353		#[token("]")]
354		RightBracket,
355		#[token("{")]
356		LeftBrace,
357		#[token("}")]
358		RightBrace,
359
360		// Punctuation
361		#[token(",")]
362		Comma,
363		#[token(".")]
364		Dot,
365		#[token(":")]
366		Colon,
367		#[token("::")]
368		ColonColon,
369		#[token(";")]
370		Semicolon,
371		#[token("_", priority = 2)]
372		Underscore,
373
374		// Attribute support
375		#[token("#")]
376		Hash,
377		}
378
379		impl Token {
380		#[must_use]
381	0	pub fn is_binary_op(&self) -> bool {
382	0	matches!(
383	0	self,
384		Token::Plus
385		\| Token::Minus
386		\| Token::Star
387		\| Token::Slash
388		\| Token::Percent
389		\| Token::Power
390		\| Token::EqualEqual
391		\| Token::NotEqual
392		\| Token::Less
393		\| Token::LessEqual
394		\| Token::Greater
395		\| Token::GreaterEqual
396		\| Token::AndAnd
397		\| Token::OrOr
398		\| Token::Ampersand
399		\| Token::Pipe
400		\| Token::Caret
401		\| Token::LeftShift
402		)
403	0	}
404
405		#[must_use]
406	0	pub fn is_unary_op(&self) -> bool {
407	0	matches!(
408	0	self,
409		Token::Bang \| Token::Minus \| Token::Tilde \| Token::Ampersand
410		)
411	0	}
412
413		#[must_use]
414	0	pub fn is_assignment_op(&self) -> bool {
415	0	matches!(
416	0	self,
417		Token::Equal
418		\| Token::PlusEqual
419		\| Token::MinusEqual
420		\| Token::StarEqual
421		\| Token::SlashEqual
422		\| Token::PercentEqual
423		\| Token::PowerEqual
424		\| Token::AmpersandEqual
425		\| Token::PipeEqual
426		\| Token::CaretEqual
427		\| Token::LeftShiftEqual
428		)
429	0	}
430		}
431
432		pub struct TokenStream<'a> {
433		lexer: Lexer<'a, Token>,
434		peeked: Option<(Token, Span)>,
435		}
436
437		/// Saved position in the token stream for backtracking
438		#[derive(Clone)]
439		pub struct TokenStreamPosition<'a> {
440		lexer: Lexer<'a, Token>,
441		peeked: Option<(Token, Span)>,
442		}
443
444		impl<'a> TokenStream<'a> {
445		#[must_use]
446	0	pub fn new(input: &'a str) -> Self {
447	0	Self {
448	0	lexer: Token::lexer(input),
449	0	peeked: None,
450	0	}
451	0	}
452
453		/// Save the current position for later restoration
454		#[must_use]
455	0	pub fn position(&self) -> TokenStreamPosition<'a> {
456	0	TokenStreamPosition {
457	0	lexer: self.lexer.clone(),
458	0	peeked: self.peeked.clone(),
459	0	}
460	0	}
461
462		/// Restore a previously saved position
463	0	pub fn set_position(&mut self, pos: TokenStreamPosition<'a>) {
464	0	self.lexer = pos.lexer;
465	0	self.peeked = pos.peeked;
466	0	}
467
468		#[allow(clippy::should_implement_trait)]
469	0	pub fn next(&mut self) -> Option<(Token, Span)> {
470	0	if let Some(peeked) = self.peeked.take() {
471	0	return Some(peeked);
472	0	}
473
474	0	self.lexer.next().map(\|result\| {
475	0	let token = result.unwrap_or(Token::Bang); // Error recovery
476	0	let span = Span::new(self.lexer.span().start, self.lexer.span().end);
477	0	(token, span)
478	0	})
479	0	}
480
481	0	pub fn peek(&mut self) -> Option<&(Token, Span)> {
482	0	if self.peeked.is_none() {
483	0	self.peeked = self.next();
484	0	}
485	0	self.peeked.as_ref()
486	0	}
487
488	0	pub fn peek_nth(&mut self, n: usize) -> Option<(Token, Span)> {
489		// For simplicity, we'll only support peek_nth(1) by cloning the lexer
490	0	if n == 1 {
491	0	let saved_peeked = self.peeked.clone();
492	0	let saved_lexer = self.lexer.clone();
493
494		// Get first token
495	0	let _first = self.peek();
496	0	self.advance();
497
498		// Get second token
499	0	let result = self.peek().cloned();
500
501		// Restore state
502	0	self.lexer = saved_lexer;
503	0	self.peeked = saved_peeked;
504
505	0	result
506		} else {
507	0	None // Not supported for n > 1
508		}
509	0	}
510
511	0	pub fn peek_nth_is_colon(&mut self, n: usize) -> bool {
512	0	if n == 0 {
513	0	self.peek().is_some_and(\|(t, _)\| matches!(t, Token::Colon))
514		} else {
515	0	self.peek_nth(n)
516	0	.is_some_and(\|(t, _)\| matches!(t, Token::Colon))
517		}
518	0	}
519
520		/// Expect a specific token and return its span
521		///
522		/// # Errors
523		///
524		/// Returns an error if the next token doesn't match the expected token or if we reached EOF
525	0	pub fn expect(&mut self, expected: &Token) -> anyhow::Result<Span> {
526	0	match self.next() {
527	0	Some((token, span)) if token == *expected => Ok(span),
528	0	Some((token, _)) => anyhow::bail!("Expected {:?}, found {:?}", expected, token),
529	0	None => anyhow::bail!("Expected {:?}, found EOF", expected),
530		}
531	0	}
532
533		// Alias for next() to avoid clippy warning about Iterator trait
534	0	pub fn advance(&mut self) -> Option<(Token, Span)> {
535	0	self.next()
536	0	}
537		}
538
539		#[cfg(test)]
540		#[allow(clippy::unwrap_used)]
541		#[allow(clippy::panic)]
542		mod tests {
543		use super::*;
544		use proptest::prelude::*;
545
546		#[test]
547		#[allow(clippy::approx_constant)] // Intentional literal for test
548		fn test_tokenize_basic() {
549		let mut stream = TokenStream::new("let x = 42 + 3.14");
550
551		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Let));
552		assert_eq!(
553		stream.next().map(\|(t, _)\| t),
554		Some(Token::Identifier("x".to_string()))
555		);
556		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Equal));
557		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Integer(42)));
558		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Plus));
559		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Float(3.14))); // Intentional literal for test
560		assert_eq!(stream.next().map(\|(t, _)\| t), None);
561		}
562
563		#[test]
564		fn test_tokenize_pipeline() {
565		let mut stream = TokenStream::new("[1, 2, 3] >> map(\|x\| x * 2)");
566
567		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::LeftBracket));
568		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Integer(1)));
569		// ... rest of tokens
570		}
571
572		#[test]
573		fn test_tokenize_comments() {
574		let mut stream = TokenStream::new("x // comment\n+ /* block */ y");
575
576		assert_eq!(
577		stream.next().map(\|(t, _)\| t),
578		Some(Token::Identifier("x".to_string()))
579		);
580		assert_eq!(stream.next().map(\|(t, _)\| t), Some(Token::Plus));
581		assert_eq!(
582		stream.next().map(\|(t, _)\| t),
583		Some(Token::Identifier("y".to_string()))
584		);
585		}
586
587		proptest! {
588		#[test]
589		fn test_tokenize_identifiers(s in "[a-zA-Z_][a-zA-Z0-9_]{0,100}") {
590		// Skip reserved keywords that should tokenize as their respective tokens
591		let reserved_keywords = [
592		"true", "false", "fun", "fn", "let", "var", "mod", "if", "else", "match",
593		"for", "in", "while", "loop", "async", "await", "throw", "try", "catch",
594		"return", "command", "Ok", "Err", "Some", "None", "null", "Result", "Option",
595		"break", "continue", "struct", "enum", "impl", "trait", "extend", "actor",
596		"state", "receive", "send", "ask", "type", "where", "const", "static",
597		"mut", "pub", "import", "use", "as", "module", "export", "df"
598		];
599
600		if reserved_keywords.contains(&s.as_str()) {
601		return Ok(()); // Skip test for reserved keywords
602		}
603
604		let mut stream = TokenStream::new(&s);
605		match stream.advance() {
606		Some((Token::Identifier(id), _)) => prop_assert_eq!(id, s),
607		Some((Token::Underscore, _)) if s == "_" => {}, // Special case for underscore
608		_ => panic!("Failed to tokenize identifier: {s}"),
609		}
610		}
611
612		#[test]
613		fn test_tokenize_integers(n in 0i64..1_000_000) {
614		let s = n.to_string();
615		let mut stream = TokenStream::new(&s);
616		match stream.advance() {
617		Some((Token::Integer(i), _)) => prop_assert_eq!(i, n),
618		_ => panic!("Failed to tokenize integer"),
619		}
620		}
621		}
622		}