Regen Language

    // You can include code in the grammar file, similar to a #include directive in C/C++
    // This will substitute the contents of "src/sdk/generated.rs.pp" into the generated SDK
    // which can be used for importing custom hooks
    "src/sdk/generated.rs.pp";

    /*===================================
      Tokenizer
    ===================================*/

    // Token types are defined with the token keyword
    token Keyword;
    token Identifier;
    token RegExp;
    token Literal;
    token Symbol;

    // The extract keyword will exclude the token type when parsing the AST
    extract token Comment;

    /*
     The token rules are defined with
     <token type> <literal or regex>;

     If the token type is the ignore keyword, it will ignore the matched pattern
    */

    // Ignore whitespaces
    ignore /\s+/;

    // Comment
    Comment /\/\/[^\n]*\n?/;
    Comment /\/\*([^\*]|(\*[^\/]))*\*\//;

    // Literal and RegExp, which are surrounded by either "" or //
    Literal /"((\\.)|[^\\"])*"/;
    RegExp /\/((\\.)|[^\\\/])*\//;

    // Keywords - there are only 6 and we have already seen 3!
    Keyword "ignore";
    Keyword "extract";
    Keyword "token";
    Keyword "semantic";
    Keyword "rule";
    Keyword "optional";

    // Special Symbols
    Symbol /[{};|()=,:\.\[\]]/;

    // Identifier is alphanumeric and underscore, but does not start with digit
    Identifier /[_a-zA-Z]\w*/;

    /*===================================
      Semantics
    ===================================*/
    // Semantic types are used to augment the token types.
    // For example, the token type "Identifier" can both be a rule name or a variable name
    // When defining the rules, you can override the semantic type of a token
    // or an entire subtree.

    // Semantics
    semantic Variable;
    semantic Token;
    semantic Semantic;
    semantic Rule;
    semantic HookName;
    semantic HookType;

    /*===================================
      Rules
    ===================================*/
    // There are 2 types of rules:
    // Unions: rule A = B | C;
    // Functional: rule A(...) <body>;

    // The first rule is the target of the parser.
    // The parser can be configured to generate one root node,
    // or keep generating root nodes until the end of the file.

    // TopLevelStatement is a union rule which says:
    // A top level statement is either a define rule statement or a top level define statement
    rule TopLevelStatement = DefineRuleStatement | TopLevelDefineStatement;

    // This is a funcitonal rule
    rule ("parse_rule":"Rule")              // <-- this is the parse tree hook
    DefineRuleStatement(                    // <-- name of the rule
        _: token Keyword"rule",             // <-- parameters
        hookAttr: optional HookAttribute,
        (Rule) ruleName: token Identifier,
        body: RuleDefineBody
    ) {                                     // <-- body
        hookAttr, ruleName, body
    };

    rule ("parse_hook":"Hook")                // The parse tree hook is a custom function that
    HookAttribute(                            // takes in the parse tree node, and returns an object
        _: token Symbol"(",                   // to be embedded into the parse tree.
        (HookName) hookName: token Literal,   // Here, the HookAttribute node will be turned into a Hook object
        _: token Symbol":",
        (HookType) hookType: token Literal,
        _: token Symbol")"
    ) {                                       // this body is a dict expression that groups the parameters
        hookName, hookType
    };

    rule ("parse_rule_value":"RuleValue")     // hooks can be applied to union rules as well
    RuleDefineBody =
      UnionRuleBody |
      FunctionalRuleBody;

    rule UnionRuleBody(
        _: token Symbol"=",                   // the parameters define how the tree is derived
        rules: optional UnionRuleList,        // this on the left says to derive a UnionRuleBody,
        _: token Symbol";"                    // the parser needs to see a "=", then an optional
    ) rules;                                  // UnionRuleList, then finally a ";"

    rule UnionRuleList((Rule) first: token Identifier, rest: optional UnionRuleListTail) first | rest;
    rule UnionRuleListTail(
        _: token Symbol"|",                   // "token Symbol"|"" is the type of the parameter
        (Rule) first: token Identifier,       // it means to match a token with type Symbol and is literally "|"
        rest: optional UnionRuleListTail
    ) first | rest;

    rule FunctionalRuleBody(
        _: token Symbol"(",
        params: optional ParamList,           // the type can also be a subrule,
        _: token Symbol")",                   // and both tokens and subrules can be optional
        body: optional Expression,
        _: token Symbol";"
    ) {
        params, body
    };

    rule ("parse_param_list":"Vec<Param>") ParamList(first: Parameter, rest: optional ParamListTail) first | rest;
    rule ParamListTail(_: token Symbol",", first: Parameter, rest: optional ParamListTail) first | rest;

    rule ("parse_param":"Param") Parameter(
        semAttr: optional ParamSemantic,
        (Variable) variable: token Identifier,  // here, "(Variable)" is a semantic annotation
        _: token Symbol":",                     // it will mark the token with the Variable semantic
        type: optional RuleType
    ) {
        semAttr, variable, type
    };

    rule ParamSemantic(
        _: token Symbol"(",
        (Semantic) semanticName: optional token Identifier,
        _: token Symbol")"
    ) semanticName;                      // the body of the rule can be one of 3 things
                                         // Dict: {a, b, ...} where a and b are parameters
                                         // Val: a where a is a parameter
                                         // Concat: a | ... | b where a and b are parameters, and b must be a vector type

    rule RuleType(
        kwOptional: optional token Keyword"optional",
        kwToken: optional token Keyword"token",
        id: token Identifier,
        tokenContent: optional token Literal
    ) {
        kwOptional, kwToken, id, tokenContent
    };

    rule TopLevelDefineStatement(body: TopLevelDefine, _: token Symbol";") body;

    rule TopLevelDefine =
      TokenLiteral
      | DefineTokenTypeStatement
      | DefineIgnoreTokenRuleStatement
      | DefineTokenRuleStatement
      | DefineSemanticStatement;

    rule ("parse_token_def":"TokenDef") DefineTokenTypeStatement(
        kwExtract: optional token Keyword"extract",
        _: token Keyword"token",
        (Token) tokenType: token Identifier
    ) {
        kwExtract, tokenType
    };

    rule ("parse_token_ignore_rule":"TokenRule")
    DefineIgnoreTokenRuleStatement(_: token Keyword"ignore", value: LiteralOrRegExp) value;

    rule ("parse_token_rule":"TokenRule") DefineTokenRuleStatement((Token) tokenType: token Identifier, value: LiteralOrRegExp) {
        tokenType, value
    };

    rule LiteralOrRegExp = TokenLiteral | TokenRegExp;
    rule TokenLiteral(t: token Literal) t;
    rule TokenRegExp(t: token RegExp) t;

    rule ("parse_semantic":"String") DefineSemanticStatement(_: token Keyword"semantic", (Semantic) id: token Identifier) id;

    rule ("parse_expr":"Expr") Expression = ConcatExpression | DictExpression;

    rule ConcatExpression(                       // here is an example of a vector type
        (Variable) first: token Identifier,      // the return types are all inferred and recursively resolved
        rest: optional ConcatExpressionTail      // here, the return type is a vector of "token Identifier"
    ) first | rest;                              // it is resolved as:
    rule ConcatExpressionTail(                   //  - ConcatExpression must return a vector of "token Identifier"
        _: token Symbol"|",                      //      because it is a concat expression
        rest: ConcatExpression                   //  - ConcatExpressionTail must return a vector of "token Identifier"
    ) rest;                                      //      because it is the last variable in the concatenation
                                                 //  - ConcatExpressionTail returns "rest", which is ConcatExpression
    rule DictExpression(                         //  - it is resolved because return types are consistent
        _: token Symbol"{",
        values: optional VariableList,
        _: token Symbol"}"
    ) values;

    rule VariableList((Variable) first: token Identifier, rest: optional VariableListTail) first | rest;
    rule VariableListTail(_: token Symbol",", rest: optional VariableList) rest;