treetagger = {
    SOI ~ ((end_tag | start_tag | token_line) ~ NEWLINE?)+ ~ EOI
}

TAB          = _{ "\t" }
WHITESPACE   = _{ " " | TAB | NEWLINE }
DOUBLE_QUOTE = _{ "\"" }
/*
Follow the extensive HTML standard for attribute names and allow a wide range of element/attribute names.
https://html.spec.whatwg.org/multipage/syntax.html#attributes-2
*/
NAME_CHAR  = _{
    !(
        CONTROL | " " | "\"" | "'" | ">" | "<" | "/" | "=" | NONCHARACTER_CODE_POINT
    ) ~ ANY}
column_value = { (!(TAB | NEWLINE) ~ ANY)+ }

token_line = ${
    column_value ~ (TAB ~ column_value)* ~ &(NEWLINE | EOI)
}

tag_name     = @{ !(">" | "<") ~ NAME_CHAR+ }
attr_id      = @{ NAME_CHAR+ }
string       = _{ DOUBLE_QUOTE ~ string_value ~ DOUBLE_QUOTE }
string_value = @{ (!("\"" | "\\") ~ ANY)* ~ (escape ~ string_value)? }
escape       = @{ "\\" ~ ("\"" | "\\" | "/" | "b" | "f" | "n" | "r" | "t" | unicode) }
unicode      = @{ "u" ~ ASCII_HEX_DIGIT{4} }

start_tag = {
    "<" ~ tag_name ~ (attr_id ~ "=" ~ string)* ~ ">"
}

end_tag = {
    "</" ~ tag_name ~ ">"
}
