pub trait Tokenizer {
  type T: Clone;
  fn tokenize(&mut self) -> Vec<TokenImpl<Self::T>>;
}

/// Token data generated by tokenizer to be used in semantic analysis and AST generation
#[derive(Debug, Clone)]
pub struct TokenImpl<T> {
  /// Token type
  pub token_type: T,
  /// Token value
  pub value: String,
  /// Token absolute position in the input text
  /// (start, end) - start is inclusive, end is exclusive
  pub pos: (usize, usize),
}

/// Action the tokenizer can take at each step
pub enum TokenizerAction<T> {
  /// Panic when a token cannot be matched.
  ///
  /// The tokenizer will skip the next character, mark it as unrecognized and try to match again.
  Panic,
  /// Keep the token
  /// If should extract, the token will be saved in a second list that doesn't get passed into AST generation
  Keep(bool /* should extract */, TokenImpl<T>),
  /// Discoard the token (for example, whitespaces)
  Ignore,
}

/// A stream of tokens with support for backtracking for AST generation
pub struct TokenStream<'t, T> {
  /// Token data
  tokens: &'t [TokenImpl<T>],
  /// Current token index
  index: usize,
  /// Position stack
  stack: Vec<usize>,
  /// Max stack size
  max_stack_size: usize,
  /// Best guess at which token is causing a syntax error
  best_error_guess: usize,
}

impl<'t, T> TokenStream<'t, T> {
  /// Create a new TokenStream with the given tokens
  pub fn new(tokens: &'t [TokenImpl<T>], max_stack_size: usize) -> Self {
    Self {
      tokens,
      index: 0,
      stack: Vec::new(),
      max_stack_size,
      best_error_guess: 0,
    }
  }

  /// Returns if there is no token left after the current position
  pub fn is_exhausted(&self) -> bool {
    self.index >= self.tokens.len()
  }

  /// Get the best guess at which token is causing a syntax error
  pub fn get_guess_err_token(&self) -> Option<&'t TokenImpl<T>> {
    self
      .tokens
      .get(self.best_error_guess)
      .or(self.tokens.get(self.index))
  }

  /// Set the error guess at the current index
  pub fn set_error(&mut self, force: bool) {
    if force || self.index > self.best_error_guess {
      self.best_error_guess = self.index;
    }
  }

  /// Returns the next token if available, and advance the position
  /// A reference is returned to avoid copying the token
  pub fn consume(&mut self) -> Option<&'t TokenImpl<T>> {
    match self.tokens.get(self.index) {
      Some(token) => {
        self.index += 1;
        Some(&token)
      }
      None => None,
    }
  }

  /// Push the current position to stack so it can be restored
  pub fn push(&mut self) -> bool {
    if self.stack.len() >= self.max_stack_size {
      return false;
    }
    self.stack.push(self.index);
    true
  }

  /// Pop position stack without restoring the position
  pub fn pop(&mut self) {
    self.stack.pop();
  }

  /// Restore the position to be the index on the stack top.
  /// This does not pop the stack.
  pub fn restore(&mut self) {
    self.index = *self.stack.last().unwrap();
  }
}
