
tokenize-file := λpath . (match path (
   ()
   (() ())
   ((_ _) ())
   (fp (tail(
      (set currently-tokenizing-file path)
      (tokenize (load-file fp))
   )))
));

currently-tokenizing-file := ();
tokenize := λtext . (tail(
   (local program)
   (local buffer)
   (local in_comment)
   (local char)
   (local parens-counter)

   (while (head-string text) (tail(
      (set char (head-string text))
      (set text (tail-string text))
   (match char (      
      ()

#     These characters are special characters
#     They are removed during tokenization
      (\o (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
         (set in_comment True)
      ))
      (\n (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
         (set in_comment ())
      ))
      (\s (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
      ))
      (\t (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
      ))

#     These characters are special characters
#     They are isolated during tokenization
      (\[ (if in_comment () (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())

         (set parens-counter (inc parens-counter))
         (set program (program \[))
      )))
      (\] (if in_comment () (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())

         (set parens-counter (dec parens-counter))
         (set program (program \]))
      )))
      (\' (if in_comment () (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
         (set program (program \'))
      )))
      (\: (if in_comment () (
         (if buffer (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
         (set program (program \:))
      )))

      (c (if in_comment () (
         (set buffer (
            buffer (clone-rope c)
         ))
         (if (eq(\l (clone-rope buffer))) (
            (set program (program (clone-rope buffer)))
            (set buffer ())
         ) ())
      )))
   )))))
   (if buffer (
      (set program (program (clone-rope buffer)))
   ) ())

   (if parens-counter (
      (fail (Hanging Parens In File currently-tokenizing-file))
   ) ())
   (debug-memory-usage 'tokenize)
   (reverse-list program)
));

