// Copyright (c) 2025 Redglyph (@gmail.com). All Rights Reserved.

#![cfg(test)]

use std::collections::HashSet;
use std::io::Read;
use lexigram_lib::char_reader::{UTF8_HIGH_MIN, UTF8_LOW_MAX, UTF8_MAX};
use lexigram_lib::lexer::StateId;
use lexigram_lib::lexer::Lexer;
use lexigram_lib::segments::Segments;

const TXT1: &str = r#"
    lexicon A;
    fragment ALPHANUM: [0-9A-Za-z_];
    fragment N_ALPHANUM: ~ALPHANUM;
    A: 'a:' ALPHANUM+;
    B: 'b:' (~N_ALPHANUM)+;
    C: 'c:' ([0-9] | [ABC-Z] | 'a'..'z' | '_')+;
    D: 'd:' \w+;
    E: 'e:' [\w]+;
    F: 'f' ':' (\d | [a-zA-Z_])+;
    G: 'g:' ([\d] | [A-Za-z_])+;
    H: 'h:' .;
    I: 'i:' [a-z] [0-9]*? ('0' | '.');
    J: 'j:' [a-z] [0-9]*  ('0' | '.');
    K: 'k:' [a-z] [0-9]+? ('0' | '.');
    L: 'l:' [a-z] [0-9]+  ('0' | '.');
    M: 'm:' [a-z] [0-9]?;
    WHITESPACE: [ \n\r\t]+ -> skip;
"#;

const TXT2: &str = r#"
    lexicon B;
    channels { REF }

    OPEN_TAG: '<' -> push(TAG_MODE), skip;
    OPEN_SPEC: '<?' -> push(SPEC_MODE), skip;
    OPEN_ARRAY: '[' -> mode(ARRAY_MODE), more;

    mode TAG_MODE;
    CLOSE_TAG: '>' -> pop, skip;
    SEP_TAG: [ \t,] -> skip;
    TAG: \w+;
    REF_TAG: '@' \w+ -> channel(REF);

    mode SPEC_MODE;
    CLOSE_SPEC: '?>' -> pop, skip;
    SEP_SPEC: [ \t,] -> skip;
    SPEC: \w+;

    mode ARRAY_MODE;
    CLOSE_ARRAY: ']' -> mode(DEFAULT_MODE), type(ARRAY);
    NUM_ARRAY: \d+ | ',' -> more;
    SEP_ARRAY: [ \t\n] -> skip;
"#;

const TXT3: &str = r#"
    lexicon C;
    A:   'a';
    B:   'b';
    ID1: [a-z]+ -> type(ID);
    ID:  [A-Z]+;
"#;

const TXT4: &str = r#"
    lexicon D;
    A: 'a' -> type(D);
    B: 'b';
    C: 'c' -> skip;
    D: 'd';
    E: 'e' -> type(E);
    F: 'f';
"#;

const TXT5: &str = r#"
    lexicon E;
    fragment X1: 'abc';
    R: 'a' -> type(P);
    Q: 'b' -> skip;
    P: 'c' -> type(R);
    O: 'd' -> skip;
    N: 'e' -> type(O);
    M: 'f' -> type(O);
    L: 'g' -> type(M);
    K: 'h';
    J: 'i' -> mode(ONE), skip;
    I: 'j' -> type(E);
    fragment X0: 'def';
    H: 'k';

    mode ONE;
    fragment Y0: 'ghi';
    G: 'l';
    F: 'm' -> type(I);
    E: 'n' -> type(H);
    D: 'o' -> type(C);
    C: 'p' -> type(A);
    fragment Y1: 'jkl';
    B: 'q' -> mode(DEFAULT_MODE);
    A: 'r' -> skip;

    grammar E;

    other things that shouldn't be scanned or parsed <>
"#;

const TXT6: &str = r#"
    lexicon F;
    Test1: '?' -> type(Test);
    (If): 'if' -> hook;
    (Else): 'else' -> hook;
    End: 'end';
    (Print) -> hook;
    (Test) -> hook;
    (Return);
    Return2: '=' -> type(Return);
    Id: [a-zA-Z][a-zA-Z_0-9]* -> hook;

    grammar F;
    top: Test1 | End | If Id;
"#;

/// Returns the set of characters that are valid at the given lexer state.
///
/// Note: This function can be quite slow, as it must test every possibility.
pub fn get_valid_segments<'a, R: Read>(lexer: &Lexer<'a, R>, state: StateId) -> Segments {
    if state >= lexer.first_end_state {
        Segments::dot()     // accepting state
    } else {
        let mut result = Segments::empty();
        let groups = (0..lexer.nbr_groups)
            .filter_map(|g| if lexer.state_table[lexer.nbr_groups as usize * state + g as usize] < lexer.nbr_states { Some(g) } else { None })
            .collect::<HashSet<_>>();
        // examines all the ASCII codes:
        for c in 0..128 {
            if groups.contains(&lexer.ascii_to_group[c]) {
                result.insert_utf8(c as u32, c as u32);
            }
        }
        // examines all the valid UTF8 codepoints:
        for range in [128..=UTF8_LOW_MAX, UTF8_HIGH_MIN..=UTF8_MAX] {
            for utf in range {
                let char = char::from_u32(utf).unwrap();
                if let Some(group) = lexer.utf8_to_group.get(&char) {
                    if groups.contains(group) {
                        result.insert_utf8(utf, utf); // determine start-stop ranges instead of inserting all codes?
                    }
                }
            }
        }
        // examines all the remaining segments:
        for (seg, group) in &lexer.seg_to_group {
            if groups.contains(group) {
                result.insert(*seg);
            }
        }
        result
    }
}

mod listener {
    use crate::lexi::listener::decode_str;

    #[test]
    fn decode_str_is_ok() {
        let tests = vec![
            ("abc", (true, "abc")),
            ("\\n\\r\\t\\'\\\\", (true, "\n\r\t'\\")),
            ("012顠abc©345𠃐ab", (true, "012顠abc©345𠃐ab")),
            ("\\u{20}\\u{00A9}", (true, " ©")),
            ("\\", (false, "'\\' incomplete escape code in string literal '\\'")),
            ("\\a", (false, "unknown escape code '\\a' in string literal '\\a'")),
            ("\\u", (false, "malformed unicode literal in string literal '\\u' (missing '{')")),
            ("\\u{20", (false, "malformed unicode literal in string literal '\\u{20' (missing '}')")),
            ("\\u{2x0}", (false, "'2x0' isn't a valid hexadecimal value")),
            ("\\u{d800}", (false, "'d800' isn't a valid unicode hexadecimal value")),
        ];
        for (input, (ok, s)) in tests {
            let result = match decode_str(input) {
                Ok(s) => (true, s),
                Err(s) => (false, s)
            };
            assert_eq!(result, (ok, s.to_string()), "failed for input '{input}'");
        }
    }
}

mod simple {
    use std::collections::BTreeMap;
    use iter_index::IndexerIterator;
    use lexigram_lib::{branch, btreemap, term};
    use lexigram_lib::build::{BuildFrom, BuildInto};
    use lexigram_lib::dfa::{tree_to_string, ReType};
    use lexigram_lib::char_reader::CharReader;
    use lexigram_lib::lexer::{ActionOption, LexerError, Pos};
    use lexigram_lib::lexergen::{LexerGen, LexerTables};
    use lexigram_lib::CollectJoin;
    use lexigram_lib::log::{LogReader, LogStatus};
    use crate::Lexi;
    use crate::lexi::listener::RuleType;
    use crate::lexi::SymbolicDfa;
    use super::*;

    #[test]
    fn lexiparser() {
        let tests = vec![
            (
                TXT1, vec![],
                vec![
                    ("A", None),
                    ("B", None),
                    ("C", None),
                    ("D", None),
                    ("E", None),
                    ("F", None),
                    ("G", None),
                    ("H", None),
                    ("I", None),
                    ("J", None),
                    ("K", None),
                    ("L", None),
                    ("M", None),
                ], btreemap![
                    0 => branch!('\t'-'\n', '\r', ' ' => 33, 'a' => 1, 'b' => 17, 'c' => 18, 'd' => 19, 'e' => 20, 'f' => 21, 'g' => 22, 'h' => 23, 'i' => 24, 'j' => 29, 'k' => 30, 'l' => 32, 'm' => 25),
                    1 => branch!(':' => 2),
                    2 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 34),
                    3 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 35),
                    4 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 36),
                    5 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 37),
                    6 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 38),
                    7 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 39),
                    8 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 40),
                    9 => branch!(DOT => 41),
                    10 => branch!('a'-'z' => 11),
                    11 => branch!('.' => 42, '0' => 43, '1'-'9' => 11),
                    12 => branch!('.' => 44, '0' => 45, '1'-'9' => 12),
                    13 => branch!('0'-'9' => 14),
                    14 => branch!('.' => 46, '0' => 47, '1'-'9' => 14),
                    15 => branch!('.' => 48, '0' => 49, '1'-'9' => 15),
                    16 => branch!('a'-'z' => 50),
                    17 => branch!(':' => 3),
                    18 => branch!(':' => 4),
                    19 => branch!(':' => 5),
                    20 => branch!(':' => 6),
                    21 => branch!(':' => 7),
                    22 => branch!(':' => 8),
                    23 => branch!(':' => 9),
                    24 => branch!(':' => 10),
                    25 => branch!(':' => 16),
                    26 => branch!('a'-'z' => 12),
                    27 => branch!('a'-'z' => 13),
                    28 => branch!('0'-'9' => 15),
                    29 => branch!(':' => 26),
                    30 => branch!(':' => 27),
                    31 => branch!('a'-'z' => 28),
                    32 => branch!(':' => 31),
                    33 => branch!('\t'-'\n', '\r', ' ' => 33), // <skip>
                    34 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 34), // <end:0>
                    35 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 35), // <end:1>
                    36 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 36), // <end:2>
                    37 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 37), // <end:3>
                    38 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 38), // <end:4>
                    39 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 39), // <end:5>
                    40 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 40), // <end:6>
                    41 => branch!(), // <end:7>
                    42 => branch!(), // <end:8>
                    43 => branch!(), // <end:8>
                    44 => branch!(), // <end:9>
                    45 => branch!('.' => 44, '0' => 45, '1'-'9' => 12), // <end:9>
                    46 => branch!(), // <end:10>
                    47 => branch!(), // <end:10>
                    48 => branch!(), // <end:11>
                    49 => branch!('.' => 48, '0' => 49, '1'-'9' => 15), // <end:11>
                    50 => branch!('0'-'9' => 51), // <end:12>
                    51 => branch!(), // <end:12>
                ], btreemap![
                    33 => term!(skip), 34 => term!(=0), 35 => term!(=1), 36 => term!(=2), 37 => term!(=3), 38 => term!(=4), 39 => term!(=5),
                    40 => term!(=6), 41 => term!(=7), 42 => term!(=8), 43 => term!(=8), 44 => term!(=9), 45 => term!(=9), 46 => term!(=10),
                    47 => term!(=10), 48 => term!(=11), 49 => term!(=11), 50 => term!(=12), 51 => term!(=12)
                ], vec![
                (
                    "a:aA_0 \t\nb:bB_1 c:cC_2 d:dD_3 e:eE_4 f:fF_5 g:gG_6 h:α i:i123. i:i120 j:j123. j:j1200 j:j120 k:k123. k:k120 l:l123. l:l1200 l:l120. m:m1",
                    vec![
                        (0, 0, "a:aA_0"), (0, 1, "b:bB_1"), (0, 2, "c:cC_2"), (0, 3, "d:dD_3"), (0, 4, "e:eE_4"), (0, 5, "f:fF_5"), (0, 6, "g:gG_6"), (0, 7, "h:α"),
                        (0, 8, "i:i123."), (0, 8, "i:i120"), (0, 9, "j:j123."), (0, 9, "j:j1200"), (0, 9, "j:j120"), (0, 10, "k:k123."), (0, 10, "k:k120"),
                        (0, 11, "l:l123."), (0, 11, "l:l1200"), (0, 11, "l:l120."), (0, 12, "m:m1")
                    ]
                )
            ],
                None,
            ),
            (
                TXT2, vec![],
                vec![
                    ("TAG", None),
                    ("REF_TAG", None),
                    ("SPEC", None),
                    ("ARRAY", Some("]")),
                ], btreemap![
                    0 => branch!('<' => 6, '[' => 7),
                    1 => branch!('\t', ' ', ',' => 9, '0'-'9', 'A'-'Z', '_', 'a'-'z' => 10, '>' => 11, '@' => 2),
                    2 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 12),
                    3 => branch!('\t', ' ', ',' => 13, '0'-'9', 'A'-'Z', '_', 'a'-'z' => 14, '?' => 4),
                    4 => branch!('>' => 15),
                    5 => branch!('\t'-'\n', ' ' => 16, ',' => 17, '0'-'9' => 18, ']' => 19),
                    6 => branch!('?' => 8), // <skip,push(1,state 1)>
                    7 => branch!(), // <more,mode(3,state 5)>
                    8 => branch!(), // <skip,push(2,state 3)>
                    9 => branch!(), // <skip>
                    10 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 10), // <end:5>
                    11 => branch!(), // <skip,pop>
                    12 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 12), // <end:6,ch 1>
                    13 => branch!(), // <skip>
                    14 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 14), // <end:9>
                    15 => branch!(), // <skip,pop>
                    16 => branch!(), // <skip>
                    17 => branch!(), // <more>
                    18 => branch!('0'-'9' => 18), // <more>
                    19 => branch!(), // <end:10,mode(0,state 0)>
                ], btreemap![
                    6 => term!(skip) + term!(push 1) + term!(pushst 1), 7 => term!(more) + term!(mode 3) + term!(pushst 5),
                    8 => term!(skip) + term!(push 2) + term!(pushst 3), 9 => term!(skip), 10 => term!(=0), 11 => term!(skip) + term!(pop),
                    12 => term!(=1) + term!(#1), 13 => term!(skip), 14 => term!(=2), 15 => term!(skip) + term!(pop), 16 => term!(skip),
                    17 => term!(more), 18 => term!(more), 19 => term!(=3) + term!(mode 0) + term!(pushst 0)
               ], vec![
                (
                    "<a,A,\t_, 0><b><><?c,C,\t_, 1?><?d?><??>[0,\t1, 2][3][]",
                    vec![
                        (0, 0, "a"), (0, 0, "A"), (0, 0, "_"), (0, 0, "0"), (0, 0, "b"), (0, 2, "c"), (0, 2, "C"),
                        (0, 2, "_"), (0, 2, "1"), (0, 2, "d"), (0, 3, "[0,1,2]"), (0, 3, "[3]"), (0, 3, "[]")
                    ]
                ),
                (
                    "<a@1,b@my_b_1>",
                    vec![(0, 0, "a"), (1, 1, "@1"), (0, 0, "b"), (1, 1, "@my_b_1")]
                )
                ],
                None,
            ),
            (
                TXT3, vec![],
                vec![
                    ("A", Some("a")),
                    ("B", Some("b")),
                    ("ID", None),
                ], btreemap![
                    0 => branch!('A'-'Z' => 1, 'a' => 2, 'b' => 3, 'c'-'z' => 4),
                    1 => branch!('A'-'Z' => 1), // <end:2>
                    2 => branch!('a'-'z' => 4), // <end:0>
                    3 => branch!('a'-'z' => 4), // <end:1>
                    4 => branch!('a'-'z' => 4), // <end:2>
                ], btreemap![
                    1 => term!(=2), 2 => term!(=0), 3 => term!(=1), 4 => term!(=2)
                ], vec![],
                None,
            ),
            (
                TXT4, vec![],
                vec![
                    ("B", Some("b")),
                    ("D", None),
                    ("E", Some("e")),
                    ("F", Some("f")),
                ], btreemap![
                    0 => branch!('a' => 1, 'b' => 2, 'c' => 3, 'd' => 4, 'e' => 5, 'f' => 6),
                    1 => branch!(), // <end:1>
                    2 => branch!(), // <end:0>
                    3 => branch!(), // <skip>
                    4 => branch!(), // <end:1>
                    5 => branch!(), // <end:2>
                    6 => branch!(), // <end:3>
                ], btreemap![
                    1 => term!(=1), 2 => term!(=0), 3 => term!(skip), 4 => term!(=1), 5 => term!(=2), 6 => term!(=3)
                ], vec![],
                None,
            ),
            (
                TXT5, vec![],
                vec![
                    ("R", Some("c")),
                    ("P", Some("a")),
                    ("O", None),
                    ("M", Some("g")),
                    ("K", Some("h")),
                    ("I", Some("m")),
                    ("H", None),
                    ("G", Some("l")),
                    ("E", Some("j")),
                    ("C", Some("o")),
                    ("B", Some("q")),
                    ("A", Some("p")),
                ], btreemap![
                    0 => branch!('a' => 2, 'b' => 3, 'c' => 4, 'd' => 5, 'e' => 6, 'f' => 7, 'g' => 8, 'h' => 9, 'i' => 10, 'j' => 11, 'k' => 12),
                    1 => branch!('l' => 13, 'm' => 14, 'n' => 15, 'o' => 16, 'p' => 17, 'q' => 18, 'r' => 19),
                    2 => branch!(), // <end:1>
                    3 => branch!(), // <skip>
                    4 => branch!(), // <end:0>
                    5 => branch!(), // <skip>
                    6 => branch!(), // <end:2>
                    7 => branch!(), // <end:2>
                    8 => branch!(), // <end:3>
                    9 => branch!(), // <end:4>
                    10 => branch!(), // <skip,mode(1,state 1)>
                    11 => branch!(), // <end:8>
                    12 => branch!(), // <end:6>
                    13 => branch!(), // <end:7>
                    14 => branch!(), // <end:5>
                    15 => branch!(), // <end:6>
                    16 => branch!(), // <end:9>
                    17 => branch!(), // <end:11>
                    18 => branch!(), // <end:10,mode(0,state 0)>
                    19 => branch!(), // <skip>
                ], btreemap![
                    2 => term!(=1), 3 => term!(skip), 4 => term!(=0), 5 => term!(skip), 6 => term!(=2), 7 => term!(=2),
                    8 => term!(=3), 9 => term!(=4), 10 => term!(skip) + term!(mode 1) + term!(pushst 1), 11 => term!(=8),
                    12 => term!(=6), 13 => term!(=7), 14 => term!(=5), 15 => term!(=6), 16 => term!(=9), 17 => term!(=11),
                    18 => term!(=10) + term!(mode 0) + term!(pushst 0), 19 => term!(skip)
                ], vec![],
                Some(Pos(28, 5)),
            ),
            (
                TXT6, vec![0, 1, 3, 4, 6],
                vec![
                    ("If", Some("if")),
                    ("Else", Some("else")),
                    ("End", Some("end")),
                    ("Print", None),
                    ("Test", Some("?")),
                    ("Return", Some("=")),
                    ("Id", None),
                ], btreemap![
                    0 => branch!('=' => 1, '?' => 2, 'A'-'Z', 'a'-'d', 'f'-'z' => 3, 'e' => 4),
                    1 => branch!(), // <end:5>
                    2 => branch!(), // <end:4>
                    3 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 3), // <end:6,hook>
                    4 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'m', 'o'-'z' => 3, 'n' => 5), // <end:6,hook>
                    5 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'c', 'e'-'z' => 3, 'd' => 6), // <end:6,hook>
                    6 => branch!('0'-'9', 'A'-'Z', '_', 'a'-'z' => 3), // <end:2>
                ], btreemap![
                    1 => term!(=5), 2 => term!(=4), 3 => term!(=6), 4 => term!(=6),
                    5 => term!(=6), 6 => term!(=2)
                ], vec![],
                Some(Pos(13, 5))
            ),
            (
                r#"lexicon test14;
                A: 'a' -> mode(A_MODE);
                B: 'b' -> mode(B_MODE);
                mode B_MODE;
                BB: '\n' -> mode(DEFAULT_MODE);
                mode A_MODE;
                AA: '\n' -> mode(DEFAULT_MODE);"#,
                vec![],
                vec![
                    ("A", Some("a")),
                    ("B", Some("b")),
                    ("BB", Some("\n")),
                    ("AA", Some("\n")),
                ], btreemap![
                    0 => branch!('a' => 3, 'b' => 4),
                    1 => branch!('\n' => 5),
                    2 => branch!('\n' => 6),
                    3 => branch!(), // <end:0,mode(1,state 1)>
                    4 => branch!(), // <end:1,mode(2,state 2)>
                    5 => branch!(), // <end:3,mode(0,state 0)>
                    6 => branch!(), // <end:2,mode(0,state 0)>
                ], btreemap![
                    3 => term!(=0) + term!(mode 1) + term!(pushst 1), 4 => term!(=1) + term!(mode 2) + term!(pushst 2),
                    5 => term!(=3) + term!(mode 0) + term!(pushst 0), 6 => term!(=2) + term!(mode 0) + term!(pushst 0)
                ],
                vec![("a\nb\n", vec![(0, 0, "a"), (0, 3, "\n"), (0, 1, "b"), (0, 2, "\n")])],
                None,
            ),
            (   // test 7
                r##"lexicon G;
                    If: 'if';
                    Id: [a-z]+;
                    Space: ' '+;
                "##,
                vec![],
                vec![("If", Some("if")), ("Id", None), ("Space", None)],
                btreemap![
                    0 => branch!(' ' => 1, 'a'-'h', 'j'-'z' => 2, 'i' => 3),
                    1 => branch!(' ' => 1), // <end:2>
                    2 => branch!('a'-'z' => 2), // <end:1>
                    3 => branch!('a'-'e', 'g'-'z' => 2, 'f' => 4), // <end:1>
                    4 => branch!('a'-'z' => 2), // <end:0>
                ], btreemap![
                    1 => term!(=2), 2 => term!(=1), 3 => term!(=1), 4 => term!(=0)
                ],
                vec![
                    ("if iff", vec![(0, 0, "if"), (0, 2, " "), (0, 1, "iff")]),
                ],
                None,
            ),
            (   // test 8
                r##"lexicon H;
                    If: 'if';
                    Id: [a-z]+?;
                    Space: ' '+;
                "##,
                vec![],
                vec![("If", Some("if")), ("Id", None), ("Space", None)],
                btreemap![
                    0 => branch!(' ' => 1, 'a'-'h', 'j'-'z' => 2, 'i' => 3),
                    1 => branch!(' ' => 1), // <end:2>
                    2 => branch!('a'-'z' => 2), // <end:1>
                    3 => branch!('a'-'e', 'g'-'z' => 2, 'f' => 4), // <end:1>
                    4 => branch!(), // <end:0>
                ], btreemap![
                    1 => term!(=2), 2 => term!(=1), 3 => term!(=1), 4 => term!(=0)
                ],
                vec![
                    ("if iff", vec![(0, 0, "if"), (0, 2, " "), (0, 0, "if"), (0, 1, "f")]),
                ],
                None,
            ),
            (   // test 9
                r##"lexicon I;
                    F: 'f';
                    If: 'if';
                    Iff: 'iff';
                    Space: ' '+;
                "##,
                vec![],
                vec![("F", Some("f")), ("If", Some("if")), ("Iff", Some("iff")), ("Space", None)],
                btreemap![
                    0 => branch!(' ' => 2, 'f' => 3, 'i' => 1),
                    1 => branch!('f' => 4),
                    2 => branch!(' ' => 2), // <end:3>
                    3 => branch!(), // <end:0>
                    4 => branch!('f' => 5), // <end:1>
                    5 => branch!(), // <end:2>
                ], btreemap![
                    2 => term!(=3), 3 => term!(=0), 4 => term!(=1), 5 => term!(=2)
                ],
                vec![
                    ("if iff", vec![(0, 1, "if"), (0, 3, " "), (0, 2, "iff")]),
                ],
                None,
            ),
            (   // test 10
                r##"lexicon J;
                    F: 'f';
                    Iff: 'iff';
                    If: 'if';
                    Space: ' '+;
                "##,
                vec![],
                vec![("F", Some("f")), ("Iff", Some("iff")), ("If", Some("if")), ("Space", None)],
                btreemap![
                    0 => branch!(' ' => 2, 'f' => 3, 'i' => 1),
                    1 => branch!('f' => 4),
                    2 => branch!(' ' => 2), // <end:3>
                    3 => branch!(), // <end:0>
                    4 => branch!('f' => 5), // <end:2>
                    5 => branch!(), // <end:1>
                ], btreemap![
                    2 => term!(=3), 3 => term!(=0), 4 => term!(=2), 5 => term!(=1)
                ],
                vec![
                    ("if iff", vec![(0, 2, "if"), (0, 3, " "), (0, 1, "iff")]),
                ],
                None,
            ),
            (   // test 11
                r##"lexicon K;
                    Zero                  : '0'             -> type(One);
                    Minus                 : '-';            // == Minus2's literal => T(Minus, Some("-"))
                    LSbracket             : '['             -> push(SET_CHAR_MODE);

                    mode SET_CHAR_MODE;
                    RSbracket             : ']'             -> pop;
                    Minus2                : '-'             -> type(Minus);
                    SetChar               : [a-z];
                    One                   : '1';            // != Zero's literal => T(One, None)
                "##,
                vec![],
                vec![("Minus", Some("-")), ("LSbracket", Some("[")), ("RSbracket", Some("]")), ("SetChar", None), ("One", None)],
                btreemap![
                    0 => branch!('-' => 2, '0' => 3, '[' => 4),
                    1 => branch!('-' => 5, '1' => 6, ']' => 7, 'a'-'z' => 8),
                    2 => branch!(), // <end:0>
                    3 => branch!(), // <end:4>
                    4 => branch!(), // <end:1,push(1,state 1)>
                    5 => branch!(), // <end:0>
                    6 => branch!(), // <end:4>
                    7 => branch!(), // <end:2,pop>
                    8 => branch!(), // <end:3>
                ], btreemap![
                    2 => term!(=0), 3 => term!(=4), 4 => term!(=1) + term!(push 1) + term!(pushst 1), 5 => term!(=0), 6 => term!(=4),
                    7 => term!(=2) + term!(pop), 8 => term!(=3)
                ],
                vec![],
                None,
            ),
            (   // test 12
                r##"lexicon L;
                    Zero                  : '0'             -> type(One);
                    Minus                 : '-';            // != Minus2's literal => T("Minus", None)
                    LSbracket             : '['             -> push(SET_CHAR_MODE);

                    mode SET_CHAR_MODE;
                    RSbracket             : ']'             -> pop;
                    Minus2                : '..'            -> type(Minus);
                    SetChar               : [a-z];
                    One                   : '1';            // != Zero's literal => T(One, None)
                "##,
                vec![],
                vec![("Minus", None), ("LSbracket", Some("[")), ("RSbracket", Some("]")), ("SetChar", None), ("One", None)],
                btreemap![
                    0 => branch!('-' => 3, '0' => 4, '[' => 5),
                    1 => branch!('.' => 2, '1' => 6, ']' => 7, 'a'-'z' => 8),
                    2 => branch!('.' => 9),
                    3 => branch!(), // <end:0>
                    4 => branch!(), // <end:4>
                    5 => branch!(), // <end:1,push(1,state 1)>
                    6 => branch!(), // <end:4>
                    7 => branch!(), // <end:2,pop>
                    8 => branch!(), // <end:3>
                    9 => branch!(), // <end:0>
                ], btreemap![
                    3 => term!(=0), 4 => term!(=4), 5 => term!(=1) + term!(push 1) + term!(pushst 1), 6 => term!(=4),
                    7 => term!(=2) + term!(pop), 8 => term!(=3), 9 => term!(=0)
                ],
                vec![],
                None,
            ),
            (   // test 13
                r##"lexicon M;
                    A0 : 'a' -> type(A);
                    A  : 'A' -> skip;
                "##,
                vec![],
                vec![("A", Some("a"))],
                btreemap![
                    0 => branch!('A' => 1, 'a' => 2),
                    1 => branch!(), // <skip>
                    2 => branch!(), // <end:0>
                ], btreemap![
                    1 => term!(skip), 2 => term!(=0)
                ],
                vec![],
                None,
            ),

        ];
        const VERBOSE: bool = false;

        for (test_id, (input, expected_hooks, expected_terminals, expected_graph, expected_end_states, test_strs, expected_pos_grammar))
        in tests.into_iter().enumerate()
        {
            if VERBOSE { println!("// {:=<80}\n// Test {test_id}", ""); }
            let mut lexi = Lexi::new(input);
            lexi.make();
            let listener = lexi.get_listener();
            let result_pos_grammar = listener.get_pos_grammar();
            let result_is_ok = listener.get_log().has_no_errors();
            if VERBOSE {
                if !listener.get_log().is_empty() {
                    println!("Messages:\n{}", listener.get_log());
                }
            }
            let text = format!("test {test_id} failed");
            assert!(result_is_ok, "{text}\n{}", listener.get_log());
            if VERBOSE {
                println!("Rules:");
                let mut rules = listener.rules.iter().to_vec();
                rules.sort_by(|a, b| (&a.1, &a.0).cmp(&(&b.1, &b.0)));
                // rules.sort_by_key(|(s, rt)| (rt, s));
                for (i, (s, rt)) in rules.into_iter().enumerate() {
                    let (t, lit) = match rt {
                        RuleType::Fragment(id) => (listener.fragments.get(*id as usize).unwrap(), listener.fragment_literals.get(*id as usize).unwrap()),
                        RuleType::Terminal(id) => (listener.terminals.get(*id as usize).unwrap(), listener.terminal_literals.get(*id as usize).unwrap()),
                    };
                    println!("- [{i:3}] {rt:?} {s}: {}    {lit:?}", tree_to_string(t, None, true));
                }
            }
            let sym_table = listener.make_symbol_table();
            let terminal_hooks = listener.terminal_hooks.iter().cloned().collect::<HashSet<_>>();
            let result_terminals = sym_table.get_terminals().cloned().to_vec();
            if VERBOSE {
                println!(
                    "Hooks: {}",
                    (0..=terminal_hooks.iter().cloned().max().unwrap_or(0))
                        .filter_map(|i| if terminal_hooks.contains(&i) { Some(i.to_string()) } else { None })
                        .join(", "));
                println!(
                    "Symbol table (terminals):\n{}",
                    result_terminals.iter().index()
                        .map(|(t, x)| format!("{: <20}{x:?}{},", "", if terminal_hooks.contains(&t) { "  // hook" } else { "" }))
                        .join("\n"));
            }
            assert_eq!(listener.terminal_hooks, expected_hooks, "{text}: mismatch hooks");
            let expected_terminals = expected_terminals.into_iter().map(|(s, t)| (s.to_string(), t.map(|st| st.to_string()))).to_vec();
            assert_eq!(result_terminals, expected_terminals, "{text}: mismatch terminals");
            let SymbolicDfa { dfa, .. } = lexi.build_into();
            if VERBOSE {
                println!("Dfa log:\n{}", dfa.get_log());
                println!("Final optimized Dfa:");
                dfa.print(20);
            }
            assert!(dfa.get_log().has_no_errors(), "{text}: Dfa build has failed");
            assert_eq!(dfa.get_state_graph(), &expected_graph, "{text}: mismatch DFA state graph");
            assert_eq!(dfa.get_end_states(), &expected_end_states, "{text}: mismatch DFA end states");
            assert_eq!(result_pos_grammar, expected_pos_grammar, "{text}: mismatch grammar start position");

            let lexer_tables = LexerTables::build_from(LexerGen::build_from(dfa));
            let mut lexer = lexer_tables.make_lexer();
            for (input_id, (input, expected_tokens)) in test_strs.into_iter().enumerate() {
                if VERBOSE {
                    println!("Testing input {input:?}")
                }
                let stream = CharReader::new(input.as_bytes());
                lexer.attach_stream(stream);
                let tokens = lexer.tokens().to_vec();
                if VERBOSE {
                    for (tokenid, channelid, string, pos_span) in &tokens {
                        println!("- {tokenid}, {channelid}, {string}, {pos_span}")
                    }
                    if lexer.has_error() {
                        println!("{:?}", lexer.get_error());
                    } else if lexer.is_eos() {
                        println!("=> OK");
                    } else {
                        println!("stream not entirely consumed");
                    }
                }
                lexer.detach_stream();
                let result_tokens = tokens.into_iter()
                    .map(|(tokenid, channelid, string, _)| (channelid, tokenid, string))
                    .to_vec();
                if VERBOSE {
                    println!("{}", result_tokens.iter().map(|(c, t, s)| format!("({c}, {t}, {s:?})")).join(", "));
                }
                let expected_tokens = expected_tokens.into_iter().map(|(c, i, s)| (c, i, s.to_string())).to_vec();
                assert_eq!(result_tokens, expected_tokens, "{text}, input {input_id}");
            }
        }
    }

    #[test]
    fn lexiparser_errors() {
        let tests = vec![
            (
                TXT1,
                vec![
                    // input    lexer OK: None, lexer error: Some(column)
                    ("a",       Some(2), Some("':'")),
                    ("a:",      Some(3), Some("'0'-'9', 'A'-'Z', '_', 'a'-'z'")),
                    ("a:aA_0",  None, None),
                ]
            ),
            (
                TXT2,
                vec![
                    ("<a>", None, None),
                    ("<a><b,", None, None),
                    ("<a,A,_, 0><b><><?c,C,_, 1!?><?d?><??>[0,\t1, 2][3][]", Some(26), Some("'\\t', ' ', ',', '0'-'9', '?', 'A'-'Z', '_', 'a'-'z'")),
                ]
            )
        ];
        const VERBOSE: bool = false;
        const JUST_SHOW_ANSWERS: bool = false;

        for (test_id, (lexicon, inputs)) in tests.into_iter().enumerate() {
            if VERBOSE || JUST_SHOW_ANSWERS { println!("// {:=<80}\n// Test {test_id}", ""); }
            let text = format!("test {test_id} failed");
            let mut lexi = Lexi::new(lexicon);
            lexi.make();
            let listener = lexi.get_listener();
            let result_is_ok = listener.get_log().has_no_errors();
            if VERBOSE {
                let msg = listener.get_log().get_messages().map(|s| format!("- {s:?}")).join("\n");
                if !msg.is_empty() {
                    println!("Messages:\n{msg}");
                }
            }
            assert!(result_is_ok, "{text}: couldn't parse the lexicon");

            // - builds the dfa from the reg tree
            let SymbolicDfa { dfa, .. } = lexi.build_into();

            // - builds the lexer
            let lexer_tables = LexerTables::build_from(LexerGen::build_from(dfa));
            let mut lexer = lexer_tables.make_lexer();

            // 2) tests the lexer on inputs
            for (input_id, (input, expected_error, expected_valid)) in inputs.into_iter().enumerate() {
                let text = format!("{text} in input {input_id}");
                if VERBOSE {
                    println!("Testing input '{input}'")
                }
                let stream = CharReader::new(input.as_bytes());
                lexer.attach_stream(stream);
                let tokens = lexer.tokens().to_vec();
                let (result_error, valid_segments) = match lexer.get_error() {
                    LexerError::None => (None, None),
                    LexerError::InvalidChar { info }
                    | LexerError::UnrecognizedChar { info }
                    | LexerError::EndOfStream { info } => (Some(info.col), Some(get_valid_segments(&lexer, info.state))),
                    LexerError::EmptyStateStack { info } => (Some(info.col), None),
                    LexerError::NoStreamAttached | LexerError::InfiniteLoop { .. } => panic!("{text}: unexpected error {:?}", lexer.get_error()),
                };
                let result_valid = valid_segments.and_then(|s| Some(s.to_string()));
                if VERBOSE {
                    for (tokenid, channelid, string, pos_span) in &tokens {
                        println!("- {tokenid}, {channelid}, {string}, {pos_span}")
                    }
                    println!(" => {:?}", lexer.get_error());
                    if let Some(s) = &result_valid {
                        println!("    valid characters: {s}")
                    }
                }
                lexer.detach_stream();
                let result_tokens = tokens.into_iter()
                    .map(|(tokenid, channelid, string, _)| (channelid, tokenid, string))
                    .to_vec();
                if VERBOSE {
                    println!("{}", result_tokens.iter().map(|(c, t, s)| format!("({c}, {t}, \"{s}\")")).join(", "));
                }
                if JUST_SHOW_ANSWERS {
                    println!("                    (\"{input}\", {result_error:?}, {result_valid:?}),");
                } else {
                    assert_eq!(result_error, expected_error, "{text}: wrong parsing outcome");
                    assert_eq!(result_valid, expected_valid.map(|s| s.to_string()), "{text}: wrong set of valid characters at this state");
                }
            }
        }
    }

    #[test]
    fn listener_data() {
        let tests = vec![
            (TXT1,
             vec!["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M"],
             btreemap![0 => 0, 1 => 1, 2 => 2, 3 => 3, 4 => 4, 5 => 5, 6 => 6, 7 => 7, 8 => 8, 9 => 9, 10 => 10, 11 => 11, 12 => 12],
            ),
            (TXT2,
             vec!["TAG", "REF_TAG", "SPEC", "ARRAY"],
             btreemap![5 => 0, 6 => 1, 9 => 2, 10 => 3],
            ),
            (TXT3,
             vec!["A", "B", "ID"],
             btreemap![0 => 0, 1 => 1, 2 => 2, 3 => 2],
            ),
            (TXT4,
             vec!["B", "D", "E", "F"],
             btreemap![0 => 1, 1 => 0, 3 => 1, 4 => 2, 5 => 3],
            ),
            (TXT5,
             vec!["R", "P", "O", "M", "K", "I", "H", "G", "E", "C", "B", "A"],
             btreemap![0 => 1, 2 => 0, 4 => 2, 5 => 2, 6 => 3, 7 => 4, 9 => 8, 10 => 6, 11 => 7, 12 => 5, 13 => 6, 14 => 9, 15 => 11, 16 => 10],
            ),
        ];
        const VERBOSE: bool = false;
        const JUST_SHOW_ANSWERS: bool = false;

        for (test_id, (input, expected_sym, expected_end)) in tests.into_iter().enumerate() {
            if VERBOSE { println!("// {:=<80}\n// Test {test_id}", ""); }
            let mut lexi = Lexi::new(input);
            lexi.make();
            let listener = lexi.get_listener();
            let result_is_ok = listener.get_log().has_no_errors();
            if VERBOSE {
                let msg = listener.get_log().get_messages().map(|s| format!("- {s:?}")).join("\n");
                if !msg.is_empty() {
                    println!("Messages:\n{msg}");
                }
            }
            let text = format!("test {test_id} failed");
            assert!(result_is_ok, "{text}");
            if VERBOSE {
                println!("Rules lexicon {}:\n{}", listener.get_name(), listener.rules_to_string(0));
            }
            let expected_sym = expected_sym.into_iter().map(|s| s.to_string()).to_vec();
            let result_end = listener.terminals.iter().enumerate().filter_map(|(id, t)| {
                t.iter_post_depth_simple().find_map(|n|
                    // unfortunately, we can't destructure entirely because term is a Box
                    if let ReType::End(term) = n.get_type() {
                        if let ActionOption::Token(end) = term.action {
                            Some((id, end))
                        } else {
                            None
                        }
                    } else {
                        None
                    }
                )
            }).collect::<BTreeMap<_,_>>();
            let SymbolicDfa { dfa: _dfa, symbol_table, .. } = lexi.build_into();
            let result_sym = symbol_table.get_terminals().map(|(s, _)| s.to_string()).to_vec();
            if JUST_SHOW_ANSWERS {
                println!("             vec![{}],", result_sym.iter().map(|s| format!("\"{s}\"")).join(", "));
                println!("             btreemap![{}],", result_end.iter().map(|(k, v)| format!("{k} => {v}")).join(", "));
            } else {
                assert_eq!(result_sym, expected_sym, "{text}: symbol table");
                assert_eq!(result_end, expected_end, "{text}: end values");
            }
        }
    }
}

mod lexicon {
    use lexigram_lib::CollectJoin;
    use lexigram_lib::log::{LogReader, LogStatus};
    use crate::Lexi;

    #[test]
    fn errors() {
        let tests: Vec<(&str, Vec<&str>, Vec<&str>)> = vec![
            (
                r#"lexicon test0; channels { CH1, CH1 }"#,
                vec!["channel 'CH1' is defined several times"],
                vec![],
            ),
            (
                r"lexicon test1; fragment A: 'a1'; fragment A: 'a2';",
                vec!["symbol 'A' is already defined"],
                vec![],
            ),
            (
                r"lexicon test2; fragment A: 'a1'; A: 'a2';",
                vec!["symbol 'A' is already defined"],
                vec![],
            ),
            (
                r"lexicon test3; A: 'a1'; A: 'a2';",
                vec!["symbol 'A' is already defined"],
                vec![],
            ),
            (
                r#"lexicon test4;
                    R: 'fake';
                    S: 'dummy';
                    A: 'a' -> type(R), type(S);
                    B: 'b' -> skip, type(R);
                    C: 'c' -> skip, more, push(A_MODE);
                    D: 'd' -> type(R), more, pop;
                    E: 'e' -> mode(X_MODE), mode(Y_MODE), skip;
                    F: 'f' -> mode(X_MODE);"#,
                vec![
                    "can't add actions 'type(R)' and 'type(S)'",
                    "can't add actions 'skip' and 'type(R)'",
                    "can't add actions 'skip' and 'more'",
                    "can't add actions 'type(R)' and 'more'",
                    "can't add actions 'mode(X_MODE)' and 'mode(Y_MODE)",
                    "mode 'A_MODE' referenced but not defined",
                    "mode 'X_MODE' referenced but not defined",
                    "mode 'Y_MODE' referenced but not defined",
                ],
                vec![],
            ),
            (
                r"lexicon test5; fragment A: 'a'; B: 'b' -> type(A);",
                vec!["action 'type' must have a terminal; 'A' is a fragment"],
                vec![],
            ),
            (
                r"lexicon test6; channels { CH1 } A: 'a' -> channel(CH2);",
                vec!["undefined channel 'CH2'"],
                vec![],
            ),
            (
                r"lexicon test7; A: ~'abc'; B: ~('ab'|'cd');",
                vec![
                    "~ can only be applied to a char set, not to 'abc'",
                    "~ can only be applied to a char set, not to |", // not a great message...
                ],
                vec![],
            ),
            (
                r"lexicon test8; fragment A: B;",
                vec!["unknown fragment 'B'"],
                vec![],
            ),
            (
                r#"lexicon test9; A: ':\u{d800}'; B: '\u{110000}'; C: 'a'..'\u{d800}';"#,
                vec![
                    "cannot decode string literal ':\\u{d800}': 'd800' isn't a valid unicode",
                    "cannot decode character literal '\\u{110000}': '110000' isn't a valid unicode",
                    "cannot decode character literal ''\\u{d800}'': 'd800' isn't a valid unicode",
                ],
                vec![],
            ),
            (
                r#"lexicon test10; A: 'bad:\a'; B: 'b';"#,
                vec!["lexical error: invalid character 'a'"],
                vec![],
            ),
            (
                r#"lexicon test11; A: 'bad:\u100'; B: 'b';"#,
                vec!["lexical error: invalid character '1'"],
                vec![],
            ),
            (
                r#"lexicon test12; (A): [a-z]; B: 'b';"#,
                vec!["syntax error: found input '[' instead of 'StrLit'"],
                vec![],
            ),
            (
                r#"lexicon test13; (A) -> skip;"#,
                vec!["syntax error: found input 'skip' instead of 'hook'"],
                vec![],
            ),
            (
                r#"lexicon test14;
                A: 'a' -> mode(A_MODE);
                B: 'b';
                mode B_MODE;
                BB: '\n';
                mode A_MODE;
                AA: '\n';
                "#,
                vec![],
                vec!["mode 'B_MODE' is defined but not used"],
            ),
            (
                r#"lexicon test15;
                A: 'a';
                B C;
                "#,
                vec!["syntax error: found input 'C' instead of ':'"],
                vec![],
            ),
            (
                r#"lexicon test16;
                A: [\u{d800}];
                B: [\u{d801}-\u{d802}];
                "#,
                vec!["cannot decode character \\u{d800}", "cannot decode character \\u{d801}", "cannot decode character \\u{d802}"],
                vec![],
            ),
            /* template:
            (
                r#""#,
                vec![],
                vec![],
            ),
            */
        ];
        const VERBOSE: bool = false;

        for (test_id, (lexicon, mut expected_errors, mut expected_warnings)) in tests.into_iter().enumerate() {
            if VERBOSE { println!("\n// {:=<80}\n// Test {test_id}\n{lexicon}\n", ""); }
            let mut lexi = Lexi::new(lexicon);
            let _result = lexi.make();
            let listener = lexi.wrapper.give_listener();
            let text = format!("test {test_id} failed");
            let msg = listener.get_log().get_messages().map(|s| format!("\n- {s}")).join("");
            if VERBOSE {
                if !msg.is_empty() {
                    println!("Messages:{msg}");
                }
            }
            let msg_errors = listener.get_log().get_errors().map(|s| format!("\n- {s}")).join("");
            if expected_errors.is_empty() {
                assert!(listener.get_log().has_no_errors(), "{text} wasn't expecting any error but got those:{msg_errors}")
            } else {
                for m in listener.get_log().get_errors() {
                    let mut i = 0;
                    while i < expected_errors.len() {
                        if m.get_inner_str().contains(expected_errors[i]) {
                            expected_errors.remove(i);
                            break;
                        } else {
                            i += 1;
                        }
                    }
                }
                assert!(
                    expected_errors.is_empty(),
                    "{text} was expecting to find those errors while parsing the lexicon:{}\nbut got only those:{msg_errors}",
                    expected_errors.iter().map(|s| format!("\n- {s}")).join(""));
            }
            let msg_warnings = listener.get_log().get_warnings().map(|s| format!("\n- {s}")).join("");
            if expected_warnings.is_empty() {
                assert!(listener.get_log().has_no_warnings(), "{text} wasn't expecting any warning but got those:{msg_warnings}")
            } else {
                for m in listener.get_log().get_warnings() {
                    let mut i = 0;
                    while i < expected_warnings.len() {
                        if m.get_inner_str().contains(expected_warnings[i]) {
                            expected_warnings.remove(i);
                            break;
                        } else {
                            i += 1;
                        }
                    }
                }
                assert!(
                    expected_warnings.is_empty(),
                    "{text} was expecting to find those warnings while parsing the lexicon:{}\nbut got only those:{msg_warnings}",
                    expected_warnings.iter().map(|s| format!("\n- {s}")).join(""));
            }
        }
    }
}