Regen Language

// You can include code in the grammar file, similar to a #include directive in C/C++
// This will substitute the contents of "src/sdk/generated.rs.pp" into the generated SDK
// which can be used for importing custom hooks
"src/sdk/generated.rs.pp";

/*===================================
  Tokenizer
===================================*/

// Token types are defined with the token keyword
token Keyword;
token Identifier;
token RegExp;
token Literal;
token Symbol;

// The extract keyword will exclude the token type when parsing the AST
extract token Comment;

/*
 The token rules are defined with
 <token type> <literal or regex>;

 If the token type is the ignore keyword, it will ignore the matched pattern
*/

// Ignore whitespaces
ignore /\s+/;

// Comment
Comment /\/\/[^\n]*\n?/;
Comment /\/\*([^\*]|(\*[^\/]))*\*\//;

// Literal and RegExp, which are surrounded by either "" or //
Literal /"((\\.)|[^\\"])*"/;
RegExp /\/((\\.)|[^\\\/])*\//;

// Keywords - there are only 6 and we have already seen 3!
Keyword "ignore";
Keyword "extract";
Keyword "token";
Keyword "semantic";
Keyword "rule";
Keyword "optional";

// Special Symbols
Symbol /[{};|()=,:\.\[\]]/;

// Identifier is alphanumeric and underscore, but does not start with digit
Identifier /[_a-zA-Z]\w*/;

/*===================================
  Semantics
===================================*/
// Semantic types are used to augment the token types.
// For example, the token type "Identifier" can both be a rule name or a variable name
// When defining the rules, you can override the semantic type of a token
// or an entire subtree.

// Semantics
semantic Variable;
semantic Token;
semantic Semantic;
semantic Rule;
semantic HookName;
semantic HookType;

/*===================================
  Rules
===================================*/
// There are 2 types of rules:
// Unions: rule A = B | C;
// Functional: rule A(...) <body>;

// The first rule is the target of the parser.
// The parser can be configured to generate one root node,
// or keep generating root nodes until the end of the file.

// TopLevelStatement is a union rule which says:
// A top level statement is either a define rule statement or a top level define statement
rule TopLevelStatement = DefineRuleStatement | TopLevelDefineStatement;

// This is a funcitonal rule
rule ("parse_rule":"Rule")              // <-- this is the parse tree hook
DefineRuleStatement(                    // <-- name of the rule
    _: token Keyword"rule",             // <-- parameters
    hookAttr: optional HookAttribute,
    (Rule) ruleName: token Identifier,
    body: RuleDefineBody
) {                                     // <-- body
    hookAttr, ruleName, body
};


rule ("parse_hook":"Hook")                // The parse tree hook is a custom function that
HookAttribute(                            // takes in the parse tree node, and returns an object
    _: token Symbol"(",                   // to be embedded into the parse tree.
    (HookName) hookName: token Literal,   // Here, the HookAttribute node will be turned into a Hook object
    _: token Symbol":",
    (HookType) hookType: token Literal,
    _: token Symbol")"
) {                                       // this body is a dict expression that groups the parameters
    hookName, hookType
};

rule ("parse_rule_value":"RuleValue")     // hooks can be applied to union rules as well
RuleDefineBody = 
  UnionRuleBody | 
  FunctionalRuleBody;

rule UnionRuleBody(
    _: token Symbol"=",                   // the parameters define how the tree is derived
    rules: optional UnionRuleList,        // this on the left says to derive a UnionRuleBody,
    _: token Symbol";"                    // the parser needs to see a "=", then an optional 
) rules;                                  // UnionRuleList, then finally a ";"

rule UnionRuleList((Rule) first: token Identifier, rest: optional UnionRuleListTail) first | rest;
rule UnionRuleListTail(
    _: token Symbol"|",                   // "token Symbol"|"" is the type of the parameter
    (Rule) first: token Identifier,       // it means to match a token with type Symbol and is literally "|"
    rest: optional UnionRuleListTail
) first | rest;


rule FunctionalRuleBody(
    _: token Symbol"(",
    params: optional ParamList,           // the type can also be a subrule, 
    _: token Symbol")",                   // and both tokens and subrules can be optional
    body: optional Expression,
    _: token Symbol";"
) {
    params, body
};


rule ("parse_param_list":"Vec<Param>") ParamList(first: Parameter, rest: optional ParamListTail) first | rest;
rule ParamListTail(_: token Symbol",", first: Parameter, rest: optional ParamListTail) first | rest;

rule ("parse_param":"Param") Parameter(
    semAttr: optional ParamSemantic,
    (Variable) variable: token Identifier,  // here, "(Variable)" is a semantic annotation
    _: token Symbol":",                     // it will mark the token with the Variable semantic
    type: optional RuleType
) {
    semAttr, variable, type
};

rule ParamSemantic(
    _: token Symbol"(",
    (Semantic) semanticName: optional token Identifier,
    _: token Symbol")"
) semanticName;                      // the body of the rule can be one of 3 things
                                     // Dict: {a, b, ...} where a and b are parameters
                                     // Val: a where a is a parameter
                                     // Concat: a | ... | b where a and b are parameters, and b must be a vector type         

rule RuleType(
    kwOptional: optional token Keyword"optional",
    kwToken: optional token Keyword"token",
    id: token Identifier,
    tokenContent: optional token Literal
) {
    kwOptional, kwToken, id, tokenContent
};

rule TopLevelDefineStatement(body: TopLevelDefine, _: token Symbol";") body;

rule TopLevelDefine = 
  TokenLiteral
  | DefineTokenTypeStatement
  | DefineIgnoreTokenRuleStatement
  | DefineTokenRuleStatement
  | DefineSemanticStatement;

rule ("parse_token_def":"TokenDef") DefineTokenTypeStatement(
    kwExtract: optional token Keyword"extract",
    _: token Keyword"token",
    (Token) tokenType: token Identifier
) {
    kwExtract, tokenType
};

rule ("parse_token_ignore_rule":"TokenRule")
DefineIgnoreTokenRuleStatement(_: token Keyword"ignore", value: LiteralOrRegExp) value;

rule ("parse_token_rule":"TokenRule") DefineTokenRuleStatement((Token) tokenType: token Identifier, value: LiteralOrRegExp) {
    tokenType, value
};

rule LiteralOrRegExp = TokenLiteral | TokenRegExp;
rule TokenLiteral(t: token Literal) t;
rule TokenRegExp(t: token RegExp) t;

rule ("parse_semantic":"String") DefineSemanticStatement(_: token Keyword"semantic", (Semantic) id: token Identifier) id;

rule ("parse_expr":"Expr") Expression = ConcatExpression | DictExpression;

rule ConcatExpression(                       // here is an example of a vector type
    (Variable) first: token Identifier,      // the return types are all inferred and recursively resolved
    rest: optional ConcatExpressionTail      // here, the return type is a vector of "token Identifier"
) first | rest;                              // it is resolved as:
rule ConcatExpressionTail(                   //  - ConcatExpression must return a vector of "token Identifier"
    _: token Symbol"|",                      //      because it is a concat expression
    rest: ConcatExpression                   //  - ConcatExpressionTail must return a vector of "token Identifier"
) rest;                                      //      because it is the last variable in the concatenation
                                             //  - ConcatExpressionTail returns "rest", which is ConcatExpression
rule DictExpression(                         //  - it is resolved because return types are consistent
    _: token Symbol"{",
    values: optional VariableList,
    _: token Symbol"}"
) values;

rule VariableList((Variable) first: token Identifier, rest: optional VariableListTail) first | rest;
rule VariableListTail(_: token Symbol",", rest: optional VariableList) rest;