//! Lexer to tokenize SQL-like syntax into lexemes

use std::str::FromStr;
use std::sync::LazyLock;

use regex::Regex;

use crate::field::Field;
use crate::function::Function;

#[derive(Clone, PartialEq, Debug)]
pub enum Lexeme {
    Select,
    RawString(String),
    Comma,
    From,
    Where,
    Operator(String),
    String(String),
    Open,
    Close,
    CurlyOpen,
    CurlyClose,
    ArithmeticOperator(String),
    And,
    Or,
    Not,
    Order,
    By,
    DescendingOrder,
    Limit,
    Into,
}

#[derive(Debug, PartialEq)]
enum LexingMode {
    Undefined,
    RawString,
    Comma,
    Operator,
    ArithmeticOperator,
    SingleQuotedString,
    DoubleQuotedString,
    BackticksQuotedString,
    Open,
    Close,
}

#[derive(Clone)]
struct LexerState {
    first_lexeme: bool,
    before_from: bool,
    possible_search_root: bool,
    after_open: bool,
    after_where: bool,
    after_operator: bool,
}

impl LexerState {
    fn new() -> Self {
        LexerState {
            first_lexeme: true,
            before_from: true,
            possible_search_root: false,
            after_open: false,
            after_where: false,
            after_operator: false,
        }
    }
}

#[derive(Clone)]
pub struct Lexer {
    input: Vec<String>,
    input_index: usize,
    char_index: isize,
    state: Box<LexerState>,
    state_history: Vec<Box<LexerState>>,
}

impl Lexer {
    pub fn new(input: Vec<String>) -> Lexer {
        Lexer {
            input,
            input_index: 0,
            char_index: 0,
            state: Box::new(LexerState::new()),
            state_history: vec![],
        }
    }

    pub fn get_input_string(&self) -> String {
        self.input.join(" ")
    }

    pub fn push_state(&mut self) {
        self.state_history.push(self.state.clone());
        self.state = Box::new(LexerState::new());
    }

    pub fn pop_state(&mut self) {
        if let Some(state) = self.state_history.pop() {
            self.state = state;
        } else {
            self.state = Box::new(LexerState::new());
        }
    }

    pub fn next_lexeme(&mut self) -> Option<Lexeme> {
        let mut s = String::new();
        let mut mode = LexingMode::Undefined;

        loop {
            let input_part = self.input.get(self.input_index);
            if input_part.is_none() {
                break;
            }
            let input_part = input_part.unwrap();
            
            let c;
            if self.char_index == -1 {
                c = ' ';
            } else {
                let input_char = input_part.chars().nth(self.char_index as usize);
                if input_char.is_none() {
                    self.input_index += 1;
                    self.char_index = -1;
                    self.state.possible_search_root = false;
                    continue;
                }
                c = input_char.unwrap();
            }
            
            match mode {
                LexingMode::Comma | LexingMode::Open | LexingMode::Close => break,
                LexingMode::SingleQuotedString => {
                    self.char_index += 1;
                    if c == '\'' {
                        break;
                    }
                    s.push(c);
                }
                LexingMode::DoubleQuotedString => {
                    self.char_index += 1;
                    if c == '"' {
                        break;
                    }
                    s.push(c);
                }
                LexingMode::BackticksQuotedString => {
                    self.char_index += 1;
                    if c == '`' {
                        break;
                    }
                    s.push(c);
                }
                LexingMode::Operator => {
                    if !self.is_op_char(c) {
                        break;
                    }

                    self.char_index += 1;
                    s.push(c);
                }
                LexingMode::ArithmeticOperator => {
                    break;
                }
                LexingMode::RawString => {
                    let is_date = c == '-' && looks_like_date(&s);
                    if !is_date {
                        if self.is_arithmetic_op_char(c) {
                            let maybe_expr = looks_like_expression(&s);
                            if maybe_expr {
                                break;
                            }
                        } else if (self.input.len() == 1 
                                || (self.input.len() > 1 && !self.state.possible_search_root))
                            && (c == ' ' || c == ',' || is_paren_char(c) || self.is_op_char(c)) {
                            break;
                        }
                    }

                    self.char_index += 1;
                    s.push(c);
                }
                LexingMode::Undefined => {
                    self.char_index += 1;
                    match c {
                        ' ' => {}
                        '\'' => mode = LexingMode::SingleQuotedString,
                        '"' => mode = LexingMode::DoubleQuotedString,
                        '`' => mode = LexingMode::BackticksQuotedString,
                        ',' => mode = LexingMode::Comma,
                        '(' | '{' => {
                            s.push(c);
                            mode = LexingMode::Open
                        }
                        ')' | '}' => {
                            s.push(c);
                            mode = LexingMode::Close
                        }
                        _ => {
                            mode = if self.is_op_char(c) {
                                LexingMode::Operator
                            } else if self.is_arithmetic_op_char(c) {
                                LexingMode::ArithmeticOperator
                            } else {
                                LexingMode::RawString
                            };
                            s.push(c);
                        }
                    }

                    self.state.after_open = mode == LexingMode::Open;
                }
            }
        }

        let lexeme = match mode {
            LexingMode::SingleQuotedString => Some(Lexeme::String(s)),
            LexingMode::DoubleQuotedString => Some(Lexeme::String(s)),
            LexingMode::BackticksQuotedString => Some(Lexeme::String(s)),
            LexingMode::Operator => Some(Lexeme::Operator(s)),
            LexingMode::ArithmeticOperator => Some(Lexeme::ArithmeticOperator(s)),
            LexingMode::Comma => Some(Lexeme::Comma),
            LexingMode::Open if &s == "(" => {
                s.clear();
                Some(Lexeme::Open)
            }
            LexingMode::Open if &s == "{" => {
                s.clear();
                Some(Lexeme::CurlyOpen)
            }
            LexingMode::Close if &s == ")" => {
                s.clear();
                Some(Lexeme::Close)
            }
            LexingMode::Close if &s == "}" => {
                s.clear();
                Some(Lexeme::CurlyClose)
            }
            LexingMode::RawString => match s.to_lowercase().as_str() {
                "select" => {
                    Some(Lexeme::Select)
                }
                "from" => {
                    self.state.before_from = false;
                    self.state.after_where = false;
                    Some(Lexeme::From)
                }
                "where" => {
                    self.state.after_where = true;
                    Some(Lexeme::Where)
                }
                "or" => Some(Lexeme::Or),
                "and" => Some(Lexeme::And),
                "not" if self.state.after_where => Some(Lexeme::Not),
                "order" => Some(Lexeme::Order),
                "by" => Some(Lexeme::By),
                "asc" => self.next_lexeme(),
                "desc" => Some(Lexeme::DescendingOrder),
                "limit" => Some(Lexeme::Limit),
                "into" => Some(Lexeme::Into),
                "eq" | "ne" | "gt" | "lt" | "ge" | "le" | "gte" | "lte" | "regexp" | "rx"
                | "like" | "between" | "in" | "exists" => Some(Lexeme::Operator(s.to_lowercase())),
                "mul" | "div" | "mod" | "plus" | "minus" => Some(Lexeme::ArithmeticOperator(s)),
                _ => Some(Lexeme::RawString(s)),
            },
            _ => None,
        };

        self.state.first_lexeme = false;
        self.state.possible_search_root = matches!(lexeme, Some(Lexeme::From))
                || (matches!(lexeme, Some(Lexeme::Comma)) && !self.state.before_from && !self.state.after_where);
        self.state.after_operator = matches!(lexeme, Some(Lexeme::Operator(_)));

        lexeme
    }

    fn is_arithmetic_op_char(&self, c: char) -> bool {
        match c {
            '+' | '-' => self.state.before_from || self.state.after_where,
            '*' | '/' | '%' => {
                (self.state.before_from || self.state.after_where) && !self.state.after_open && !self.state.after_operator
            }
            _ => false,
        }
    }

    fn is_op_char(&self, c: char) -> bool {
        if !self.state.before_from && !self.state.after_where {
            return false;
        }

        matches!(c, '=' | '!' | '<' | '>' | '~')
    }
}

fn is_paren_char(c: char) -> bool {
    c == '(' || c == ')' || c == '{' || c == '}'
}

static DATE_ALIKE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new("(\\d{4})-?(\\d{2})?").unwrap()
});

fn looks_like_expression(s: &str) -> bool {
    !s.split(|c: char| !c.is_ascii_alphanumeric()).any(|s| {
        Field::parse_field(s).is_err() && Function::from_str(s).is_err() && s.parse::<i64>().is_err()
    })
}

fn looks_like_date(s: &str) -> bool {
    match DATE_ALIKE_REGEX.captures(s) {
        Some(cap) => {
            let year = cap[1].parse::<i32>();
            let year_ok = match year {
                Ok(year) => (1970..3000).contains(&year), // optimistic assumption
                _ => false,
            };

            if !year_ok {
                return false;
            }

            match cap.get(2) {
                Some(month) => {
                    let month = month.as_str().parse::<i32>();

                    match month {
                        Ok(month) => (1..=12).contains(&month),
                        _ => false,
                    }
                }
                _ => true,
            }
        }
        _ => false,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    macro_rules! lexer {
        ($($str:literal),+) => {
            {
                let quote = vec![$($str.to_string()),+];
                Lexer::new(quote)
            }
        }
    }

    #[test]
    fn looks_like_date_test() {
        assert!(looks_like_date("2018"));
        assert!(looks_like_date("2018-01"));
    }

    #[test]
    fn lexemes() {
        let mut lexer = lexer!("select name, path ,size , fsize from /test depth 2, /test2 archives,/test3 depth 3 archives , /test4 ,'/test5' where name != 123 AND ( size gt 456 or fsize lte 758) or name = 'xxx' order by 1 ,3 desc , path asc limit 50");
        
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("fsize")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("depth")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("2")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test2")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("archives")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test3")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("depth")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("3")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("archives")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test4")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::String(String::from("/test5")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("!=")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("123")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::And));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("gt")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("456")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Or));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("fsize")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("lte")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("758")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Or));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Operator(String::from("="))));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::String(String::from("xxx"))));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Order));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::By));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("1")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("3")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::DescendingOrder));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Limit));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("50")))
        );
    }

    #[test]
    fn spaces() {
        let lexer = lexer!("path,size from . where size=0");
        assert_spaces(lexer);

        let lexer = lexer!("path,size from . where size =0");
        assert_spaces(lexer);

        let lexer = lexer!("path,size from . where size= 0");
        assert_spaces(lexer);

        let lexer = lexer!("path,size from . where size = 0");
        assert_spaces(lexer);

        let lexer = lexer!("path,size from . where size   =     0");
        assert_spaces(lexer);
    }

    fn assert_spaces(mut lexer: Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Operator(String::from("="))));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("0")))
        );
    }

    #[test]
    fn func_calls() {
        let mut lexer = lexer!("name, length(name),UPPER( name ) from .");
        assert_func_calls_lexemes(&mut lexer);
    }

    #[test]
    fn func_calls_with_multiple_input_parts() {
        let mut lexer = lexer!("name,", "length(name),UPPER(", "name", ")", "from", ".");
        assert_func_calls_lexemes(&mut lexer);
    }

    fn assert_func_calls_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("length")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("UPPER")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
    }

    #[test]
    fn func_calls2() {
        let mut lexer = lexer!("select name, upper(name) from . depth 1");
        assert_func_calls2_lexemes(&mut lexer);
    }

    #[test]
    fn func_calls2_with_multiple_input_parts() {
        let mut lexer = lexer!("select", "name,", "upper(name)", "from", ".", "depth", "1");
        assert_func_calls2_lexemes(&mut lexer);
    }

    fn assert_func_calls2_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("upper")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("depth")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("1")))
        );
    }

    #[test]
    fn func_calls3() {
        let mut lexer = lexer!("select name, rand() from . depth 1 order by rand() limit 10");
        assert_func_calls3_lexemes(&mut lexer);
    }

    #[test]
    fn func_calls3_with_multiple_input_parts() {
        let mut lexer = lexer!(
            "select", "name,", "rand()", "from", ".", "depth", "1", "order", "by", "rand()", "limit", "10"
        );
        assert_func_calls3_lexemes(&mut lexer);
    }

    fn assert_func_calls3_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("rand")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("depth")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("1")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Order));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::By));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("rand")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Limit));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("10")))
        );
    }

    #[test]
    fn agg_func_calls() {
        let mut lexer = lexer!("COUNT(*), MIN(size), AVG(size), MAX(size) from .");
        assert_agg_func_calls_lexemes(&mut lexer);
    }

    #[test]
    fn agg_func_calls_with_multiple_input_parts() {
        let mut lexer = lexer!("COUNT(*),", "MIN(size),", "AVG(size),", "MAX(size)", "from", ".");
        assert_agg_func_calls_lexemes(&mut lexer);
    }

    fn assert_agg_func_calls_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("COUNT")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("*")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("MIN")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("AVG")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("MAX")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
    }

    #[test]
    fn arithmetic_operators() {
        let mut lexer = lexer!("width + height, width-height, width mul height, path from .");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("width")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::ArithmeticOperator(String::from("+")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("height")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("width")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::ArithmeticOperator(String::from("-")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("height")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("width")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::ArithmeticOperator(String::from("mul")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("height")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
    }

    #[test]
    fn context_sensitive_date_string() {
        let mut lexer = lexer!("size,modified,path from . where modified gt 2018-08-01 and name='*.txt' order by modified");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("modified")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("modified")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("gt")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("2018-08-01")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::And));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Operator(String::from("="))));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::String(String::from("*.txt")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Order));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::By));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("modified")))
        );
    }

    #[test]
    fn root_with_dashes() {
        let mut lexer = lexer!("path from ./foo-bar");
        assert_root_with_dashes_lexemes(&mut lexer);
    }

    #[test]
    fn root_with_dashes_with_multiple_input_parts() {
        let mut lexer = lexer!("path", "from", "./foo-bar");
        assert_root_with_dashes_lexemes(&mut lexer);
    }

    fn assert_root_with_dashes_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("./foo-bar")))
        );
    }

    #[test]
    fn another_workaround_for_raw_paths() {
        let mut lexer = lexer!("name, size where path eq \\*some/stuff-inside/\\*.rs");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("path")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("eq")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(
                "\\*some/stuff-inside/\\*.rs"
            )))
        );
    }

    #[test]
    fn mime_types() {
        let mut lexer = lexer!("mime from . where mime = application/pkcs8+pem");
        assert_mime_types_lexemes(&mut lexer);
    }

    #[test]
    fn mime_types_with_multiple_input_parts() {
        let mut lexer = lexer!("mime", "from", ".", "where", "mime", "=", "application/pkcs8+pem");
        assert_mime_types_lexemes(&mut lexer);
    }

    fn assert_mime_types_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("mime")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from(".")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("mime")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Operator(String::from("="))));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("application/pkcs8+pem")))
        );
    }

    #[test]
    fn raw_abs_path_after_eq() {
        let mut lexer = lexer!("abspath,absdir,name where absdir = /home/user/docs");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("abspath")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Operator(String::from("="))));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/home/user/docs")))
        );

        let mut lexer = lexer!("abspath,absdir,name where absdir == /home/user/docs");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("abspath")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("==")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/home/user/docs")))
        );

        let mut lexer = lexer!("abspath,absdir,name where absdir === /home/user/docs");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("abspath")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("===")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/home/user/docs")))
        );

        let mut lexer = lexer!("abspath,absdir,name where absdir eq /home/user/docs");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("abspath")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("absdir")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("eq")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/home/user/docs")))
        );
    }

    #[test]
    fn star_field() {
        let mut lexer = lexer!("select modified,* from /test");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("modified")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::ArithmeticOperator(String::from("*")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test")))
        );

        let mut lexer = lexer!("select modified, * from /test limit 10");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("modified")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::ArithmeticOperator(String::from("*")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test")))
        );
    }

    #[test]
    fn lexer_with_multiple_input_parts() {
        let mut lexer = lexer!("select", "modified,*", "from", "/test");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("modified")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Comma));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::ArithmeticOperator(String::from("*")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test")))
        );
    }

    #[test]
    fn group_by() {
        let mut lexer = lexer!("select AVG(size) from /test group by mime");
        assert_group_by_lexemes(&mut lexer);
    }

    #[test]
    fn group_by_with_multiple_input_parts() {
        let mut lexer = lexer!("select", "AVG(size)", "from", "/test", "group", "by", "mime");
        assert_group_by_lexemes(&mut lexer);
    }

    fn assert_group_by_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("AVG")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Open));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Close));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/test")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::RawString("group".to_owned())));
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::By));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("mime")))
        );
    }

    #[test]
    fn between_op() {
        let mut lexer = lexer!("select name from /home/user where size between 1000000 and 2000000");
        assert_between_op_lexemes(&mut lexer);
    }

    #[test]
    fn between_op_with_multiple_input_parts() {
        let mut lexer = lexer!(
            "select", "name", "from", "/home/user", "where", "size", "between", "1000000", "and", "2000000"
        );
        assert_between_op_lexemes(&mut lexer);
    }

    fn assert_between_op_lexemes(lexer: &mut Lexer) {
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("/home/user")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::Where));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("size")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Operator(String::from("between")))
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("1000000")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::And));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("2000000")))
        );
    }

    #[test]
    fn spaces_in_path_with_single_quotes() {
        let mut lexer = lexer!("select name from '/home/user/foo bar/'");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::String(String::from("/home/user/foo bar/")))
        );
    }

    #[test]
    fn spaces_in_path_with_double_quotes() {
        let mut lexer = lexer!("select name from \"/home/user/foo bar/\"");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::String(String::from("/home/user/foo bar/")))
        );
    }

    #[test]
    fn spaces_in_path_with_backticks() {
        let mut lexer = lexer!("select name from `/home/user/foo bar/`");

        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::Select)
        );
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::RawString(String::from("name")))
        );
        assert_eq!(lexer.next_lexeme(), Some(Lexeme::From));
        assert_eq!(
            lexer.next_lexeme(),
            Some(Lexeme::String(String::from("/home/user/foo bar/")))
        );
    }
}
