// Hermes Schema Definition Language (SDL) Grammar

// Main entry point - a file contains one or more index definitions
file = { SOI ~ index_def+ ~ EOI }

// Index definition
index_def = { "index" ~ identifier ~ "{" ~ (field_def | default_fields_def | query_router_def | reorder_on_merge_def)* ~ "}" }

// Index-level option: BP-reorder `reorder`-attributed BMP fields inside merges.
// Absent = disabled (merges block-copy).
//   reorder_on_merge: true
reorder_on_merge_def = { "reorder_on_merge" ~ ":" ~ bool_lit }
bool_lit = { "true" | "false" }

// Default fields definition
default_fields_def = { "default_fields" ~ ":" ~ "[" ~ identifier ~ ("," ~ identifier)* ~ "]" }

// Query router definition for routing queries to specific fields based on regex
// Example:
//   query_router {
//       pattern: r"10\.\d{4,}/[^\s]+"
//       substitution: "doi://{0}"
//       target_field: uris
//       mode: exclusive
//   }
query_router_def = { "query_router" ~ "{" ~ query_router_prop+ ~ "}" }

// Query router properties
query_router_prop = {
    query_router_pattern |
    query_router_substitution |
    query_router_target |
    query_router_mode
}

query_router_pattern = { "pattern" ~ ":" ~ regex_string }
query_router_substitution = { "substitution" ~ ":" ~ quoted_string }
query_router_target = { "target_field" ~ ":" ~ identifier }
query_router_mode = { "mode" ~ ":" ~ routing_mode }

// Routing mode: exclusive (only target field) or additional (target + default fields)
routing_mode = { "exclusive" | "additional" }

// Regex string: r"..." or just "..."
regex_string = { raw_string | quoted_string }
raw_string = @{ "r\"" ~ raw_string_inner ~ "\"" }
raw_string_inner = @{ (!("\"") ~ ANY)* }
quoted_string = @{ "\"" ~ string_inner ~ "\"" }
string_inner = @{ (!("\"" | "\\") ~ ANY | "\\" ~ ANY)* }

// Field definition with optional tokenizer
// Examples:
//   field title: text [indexed, stored]
//   field body: text<en_stem> [indexed]
//   field name: text<simple> [indexed, stored]
//   field embedding: dense_vector<768> [indexed<rabitq>]
//   field embedding: dense_vector<768> [indexed<ivf_rabitq, num_clusters: 256, nprobe: 32>]
//   field embedding: dense_vector<768> [indexed<flat>]  # brute-force, no ANN index
// Note: dense_vector_config matches before binary_dense_vector_config (both match `<N>`).
// The Rust parser remaps based on field_type when needed.
field_def = { "field" ~ identifier ~ ":" ~ field_type ~ (sparse_vector_config | dense_vector_config | binary_dense_vector_config | tokenizer_spec)? ~ attributes? }

// Field types
// Note: "binary_dense_vector"/"binary_vector" must precede "binary" (bytes alias)
// to avoid PEG prefix-matching ambiguity
field_type = {
    "text" | "string" | "str" |
    "u64" | "uint" | "unsigned" |
    "i64" | "int" | "integer" |
    "f64" | "float" | "double" |
    "binary_dense_vector" | "binary_vector" |
    "bytes" | "binary" | "blob" |
    "json" |
    "sparse_vector" |
    "dense_vector" | "vector"
}

// Sparse vector configuration - only index_size (positional)
// Quantization and weight_threshold go in indexed<...> attribute
// Examples:
//   field embedding: sparse_vector [indexed, stored]
//   field embedding: sparse_vector<u16> [indexed]
//   field embedding: sparse_vector<u32> [indexed<quantization: uint8, weight_threshold: 0.1>]
sparse_vector_config = { "<" ~ index_size_spec ~ ">" }
index_size_spec = { "u16" | "u32" }
quantization_spec = { "float32" | "float16" | "uint8" | "uint4" | "f32" | "f16" | "u8" | "u4" }
weight_threshold_spec = @{ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? }

// Dense vector configuration - dimension and optional quantization type
// All index-related params (index type, num_clusters, nprobe, build_threshold) are in indexed<...>
// Examples:
//   field embedding: dense_vector<768> [indexed]                           # default f32
//   field embedding: dense_vector<768, f16> [indexed]                      # half precision, 2× savings
//   field embedding: dense_vector<768, uint8> [indexed]                    # scalar quantized, 4× savings
//   field embedding: dense_vector<768> [indexed<rabitq>]                   # explicit RaBitQ
//   field embedding: dense_vector<768, f16> [indexed<ivf_rabitq>]          # f16 + IVF-RaBitQ
//   field embedding: dense_vector<768> [indexed<flat>]                     # brute-force only
//   field embedding: dense_vector<768> [stored]                            # not indexed, just stored
dense_vector_config = { "<" ~ dense_vector_params ~ ">" }
dense_vector_params = { dense_vector_keyword_params | dense_vector_positional_params }

// Keyword-based params: dims and optional quantization
dense_vector_keyword_params = { dims_kwarg ~ ("," ~ quant_type_spec)? }
dims_kwarg = { "dims" ~ ":" ~ dimension_spec }

// Positional params: dimension and optional quantization
dense_vector_positional_params = { dimension_spec ~ ("," ~ quant_type_spec)? }

dimension_spec = @{ ASCII_DIGIT+ }
quant_type_spec = { "f32" | "f16" | "uint8" | "u8" }

// Binary dense vector configuration - just dimension (number of bits)
// Examples:
//   field hash: binary_dense_vector<768> [indexed]
//   field hash: binary_vector<1024> [indexed, stored]
binary_dense_vector_config = { "<" ~ dimension_spec ~ ">" }

// Tokenizer specification: <tokenizer_name>
tokenizer_spec = { "<" ~ identifier ~ ">" }

// Attributes like [indexed, stored]
// indexed can have optional index config: indexed<positions> or indexed<rabitq, centroids: "path">
// stored can have optional multi config: stored<multi> for array fields
attributes = { "[" ~ attribute ~ ("," ~ attribute)* ~ "]" }
attribute = { indexed_with_config | "indexed" | stored_with_config | "stored" | "fast" | "primary" | "reorder" }

// Stored attribute with optional multi flag
// Examples:
//   stored
//   stored<multi>  # for array fields
stored_with_config = { "stored" ~ "<" ~ "multi" ~ ">" }

// Indexed attribute with optional vector index configuration
// Examples:
//   indexed
//   indexed<positions>  # for phrase queries
//   indexed<flat>       # brute-force search (no ANN index)
//   indexed<rabitq>     # RaBitQ binary quantization
//   indexed<ivf_rabitq> # IVF-RaBitQ (centroids trained from data)
//   indexed<ivf_rabitq, num_clusters: 256, nprobe: 32>
//   indexed<rabitq, build_threshold: 5000>  # build ANN after 5000 vectors
//   indexed<quantization: uint8, weight_threshold: 0.1>  # for sparse vectors
//   indexed<positions, quantization: uint8>  # sparse with positions
//   indexed<quantization: uint8, query<tokenizer: "model/name", weighting: idf>>  # with query config
indexed_with_config = { "indexed" ~ "<" ~ index_config_params ~ ">" }
index_config_params = { index_config_param ~ ("," ~ index_config_param)* }
index_config_param = { index_type_kwarg | num_clusters_kwarg | nprobe_kwarg | build_threshold_kwarg | quantization_kwarg | weight_threshold_kwarg | bmp_block_size_kwarg | block_size_kwarg | pruning_kwarg | min_terms_kwarg | sparse_format_kwarg | sparse_dims_kwarg | sparse_max_weight_kwarg | doc_mass_kwarg | soar_kwarg | bits_kwarg | query_config_block | positions_kwarg | index_type_spec }
// Position tracking modes:
//   positions - full tracking (element ordinal + token position)
//   ordinal - only element ordinal for multi-valued fields
//   token_position - only token position within text for phrase queries
positions_kwarg = { "positions" | "ordinal" | "token_position" }
index_type_kwarg = { "index" ~ ":" ~ index_type_spec }
index_type_spec = { "flat" | "scann" | "ivf_rabitq" | "ivf" | "rabitq" }
num_clusters_kwarg = { "num_clusters" ~ ":" ~ num_clusters_spec }
num_clusters_spec = @{ ASCII_DIGIT+ }
build_threshold_kwarg = { "build_threshold" ~ ":" ~ build_threshold_spec }
build_threshold_spec = @{ ASCII_DIGIT+ }
nprobe_kwarg = { "nprobe" ~ ":" ~ nprobe_spec }
quantization_kwarg = { "quantization" ~ ":" ~ quantization_spec }
weight_threshold_kwarg = { "weight_threshold" ~ ":" ~ weight_threshold_spec }
block_size_kwarg = { "block_size" ~ ":" ~ block_size_spec }
block_size_spec = @{ ASCII_DIGIT+ }
// BMP block size (docs per block, power of two, max 256; default 64).
// Uniform across all segments of the field. Grid memory scales as
// 1/bmp_block_size — large corpora should set 256 (docs/bmp-grid-compression.md).
bmp_block_size_kwarg = { "bmp_block_size" ~ ":" ~ block_size_spec }
pruning_kwarg = { "pruning" ~ ":" ~ pruning_spec }
pruning_spec = @{ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? }
min_terms_kwarg = { "min_terms" ~ ":" ~ min_terms_spec }
min_terms_spec = @{ ASCII_DIGIT+ }
sparse_format_kwarg = { "format" ~ ":" ~ sparse_format_spec }
sparse_format_spec = { "bmp" | "maxscore" }
sparse_dims_kwarg = { "dims" ~ ":" ~ sparse_dims_spec }
sparse_dims_spec = @{ ASCII_DIGIT+ }
sparse_max_weight_kwarg = { "max_weight" ~ ":" ~ sparse_max_weight_spec }
sparse_max_weight_spec = @{ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? }
// RaBitQ bits per dimension for IVF-RaBitQ dense indexes:
//   bits: 1  # classic binary RaBitQ (default)
//   bits: 4  # extended multi-bit codes (tighter estimates, smaller rerank pool)
bits_kwarg = { "bits" ~ ":" ~ bits_spec }
bits_spec = @{ ASCII_DIGIT }
// Document-side mass cropping for sparse vectors:
//   doc_mass: 0.9  # keep top-weight entries covering 90% of each vector's mass
doc_mass_kwarg = { "doc_mass" ~ ":" ~ pruning_spec }
// SOAR spilled secondary cluster assignments for IVF-based dense indexes:
//   soar: selective  # spill only vectors near cluster boundaries (recommended)
//   soar: full       # spill every vector to 1 secondary cluster
//   soar: aggressive # spill every vector to 2 secondary clusters
//   soar: off        # no spilling (default)
soar_kwarg = { "soar" ~ ":" ~ soar_spec }
soar_spec = { "selective" | "full" | "aggressive" | "off" }
nprobe_spec = @{ ASCII_DIGIT+ }
path_chars = @{ (!("\"" ) ~ ANY)* }

// Query-time configuration for sparse vectors
// Example: query<tokenizer: "model/name", weighting: idf>
query_config_block = { "query" ~ "<" ~ query_config_params ~ ">" }
query_config_params = { query_config_param ~ ("," ~ query_config_param)* }
query_config_param = { query_tokenizer_kwarg | query_weighting_kwarg | query_weight_threshold_kwarg | query_max_dims_kwarg | query_pruning_kwarg | query_min_query_dims_kwarg }
query_tokenizer_kwarg = { "tokenizer" ~ ":" ~ tokenizer_path }
query_weighting_kwarg = { "weighting" ~ ":" ~ weighting_spec }
tokenizer_path = { "\"" ~ path_chars ~ "\"" }
weighting_spec = { "one" | "idf_file" | "idf" }
query_weight_threshold_kwarg = { "weight_threshold" ~ ":" ~ weight_threshold_spec }
query_max_dims_kwarg = { "max_dims" ~ ":" ~ nprobe_spec }
query_pruning_kwarg = { "pruning" ~ ":" ~ pruning_spec }
query_min_query_dims_kwarg = { "min_query_dims" ~ ":" ~ min_terms_spec }

// Identifier (field names, index names, tokenizer names)
identifier = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_")* }

// Whitespace and comments
WHITESPACE = _{ " " | "\t" | "\r" | "\n" }
COMMENT = _{ "#" ~ (!"\n" ~ ANY)* }
