################################################################################
# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from           #
# runtime/src/iree/tokenizer/BUILD.bazel                                       #
#                                                                              #
# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary   #
# CMake-only content.                                                          #
#                                                                              #
# To disable autogeneration for this file entirely, delete this header.        #
################################################################################

iree_add_all_subdirs()

iree_cc_library(
  NAME
    byte_level_tables
  HDRS
    "byte_level_tables.h"
  DEPS

  PUBLIC
)

iree_cc_library(
  NAME
    model
  HDRS
    "model.h"
  SRCS
    "model.c"
  DEPS
    ::segmenter
    ::types
    iree::base
  PUBLIC
)

iree_cc_library(
  NAME
    decoder
  HDRS
    "decoder.h"
  SRCS
    "decoder.c"
  DEPS
    iree::base
  PUBLIC
)

iree_cc_library(
  NAME
    normalizer
  HDRS
    "normalizer.h"
  SRCS
    "normalizer.c"
  DEPS
    iree::base
  PUBLIC
)

iree_cc_library(
  NAME
    postprocessor
  HDRS
    "postprocessor.h"
  SRCS
    "postprocessor.c"
  DEPS
    ::types
    iree::base
    iree::tokenizer::vocab
  PUBLIC
)

iree_cc_test(
  NAME
    postprocessor_test
  SRCS
    "postprocessor_test.cc"
  DEPS
    ::postprocessor
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
)

iree_cc_library(
  NAME
    segmenter
  HDRS
    "segmenter.h"
  SRCS
    "segmenter.c"
  DEPS
    iree::base
  PUBLIC
)

iree_cc_library(
  NAME
    special_tokens
  HDRS
    "special_tokens.h"
  SRCS
    "special_tokens.c"
  DEPS
    ::types
    iree::base
  PUBLIC
)

iree_cc_test(
  NAME
    special_tokens_test
  SRCS
    "special_tokens_test.cc"
  DEPS
    ::special_tokens
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
)

iree_cc_library(
  NAME
    tokenizer
  HDRS
    "tokenizer.h"
  SRCS
    "tokenizer.c"
  DEPS
    ::decoder
    ::model
    ::normalizer
    ::postprocessor
    ::segmenter
    ::special_tokens
    ::types
    iree::base
    iree::base::internal
    iree::base::internal::unicode
    iree::tokenizer::decoder::byte_fallback
    iree::tokenizer::vocab
  PUBLIC
)

iree_cc_library(
  NAME
    tokenizer_test_util
  HDRS
    "tokenizer_test_util.h"
  DEPS
    ::tokenizer
    iree::base
    iree::testing::gtest
    iree::tokenizer::model::bpe
    iree::tokenizer::model::wordpiece
    iree::tokenizer::normalizer::bert
    iree::tokenizer::segmenter::bert
    iree::tokenizer::segmenter::punctuation
    iree::tokenizer::segmenter::whitespace
    iree::tokenizer::testing::test_util
    iree::tokenizer::vocab
    iree::tokenizer::vocab::vocab_builder
  TESTONLY
  PUBLIC
)

iree_cc_test(
  NAME
    tokenizer_test
  SRCS
    "tokenizer_test.cc"
  DEPS
    ::tokenizer
    ::tokenizer_test_util
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
)

iree_cc_test(
  NAME
    tokenizer_encode_test
  SRCS
    "tokenizer_encode_test.cc"
  DEPS
    ::special_tokens
    ::tokenizer
    ::tokenizer_test_util
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
    iree::tokenizer::model::unigram
    iree::tokenizer::model::wordpiece
    iree::tokenizer::normalizer::bert
    iree::tokenizer::normalizer::lowercase
    iree::tokenizer::normalizer::nfc
    iree::tokenizer::normalizer::nfd
    iree::tokenizer::normalizer::prepend
    iree::tokenizer::normalizer::regex_replace
    iree::tokenizer::normalizer::replace
    iree::tokenizer::normalizer::sequence
    iree::tokenizer::normalizer::strip
    iree::tokenizer::normalizer::strip_accents
    iree::tokenizer::regex::compile
    iree::tokenizer::segmenter::bert
    iree::tokenizer::segmenter::metaspace
    iree::tokenizer::segmenter::punctuation
    iree::tokenizer::segmenter::sequence
    iree::tokenizer::segmenter::split
    iree::tokenizer::segmenter::whitespace
)

iree_cc_test(
  NAME
    tokenizer_decode_test
  SRCS
    "tokenizer_decode_test.cc"
  DEPS
    ::tokenizer
    ::tokenizer_test_util
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
    iree::tokenizer::decoder::byte_fallback
    iree::tokenizer::decoder::byte_level
    iree::tokenizer::decoder::metaspace
    iree::tokenizer::decoder::replace
    iree::tokenizer::decoder::sequence
    iree::tokenizer::decoder::wordpiece
)

iree_cc_test(
  NAME
    tokenizer_huggingface_test
  SRCS
    "tokenizer_huggingface_test.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
    iree::tokenizer::format::huggingface::tokenizer_json
    iree::tokenizer::testdata::streaming_testdata
)

iree_cc_test(
  NAME
    tokenizer_consistency_test
  SRCS
    "tokenizer_consistency_test.cc"
  DEPS
    ::tokenizer
    ::tokenizer_test_util
    iree::base
    iree::testing::gtest
    iree::testing::gtest_main
    iree::tokenizer::regex::compile
    iree::tokenizer::segmenter::split
)

iree_cc_binary_benchmark(
  NAME
    tokenizer_benchmark
  SRCS
    "tokenizer_benchmark.cc"
  DEPS
    ::tokenizer
    benchmark
    iree::base
    iree::testing::benchmark_main
    iree::tokenizer::decoder::byte_level
    iree::tokenizer::model::bpe
    iree::tokenizer::segmenter::whitespace
    iree::tokenizer::vocab
    iree::tokenizer::vocab::vocab_builder
  TESTONLY
)

iree_cc_library(
  NAME
    types
  HDRS
    "types.h"
  DEPS
    iree::base
  PUBLIC
)

iree_cc_fuzz(
  NAME
    tokenizer_encode_fuzz
  SRCS
    "tokenizer_encode_fuzz.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::tokenizer::testing::fuzzing_util
)

iree_cc_fuzz(
  NAME
    tokenizer_decode_fuzz
  SRCS
    "tokenizer_decode_fuzz.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::tokenizer::testing::fuzzing_util
)

iree_cc_fuzz(
  NAME
    tokenizer_ringbuffer_fuzz
  SRCS
    "tokenizer_ringbuffer_fuzz.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::tokenizer::testing::fuzzing_util
)

iree_cc_fuzz(
  NAME
    tokenizer_roundtrip_fuzz
  SRCS
    "tokenizer_roundtrip_fuzz.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::tokenizer::testing::fuzzing_util
)

iree_cc_fuzz(
  NAME
    tokenizer_batch_encode_fuzz
  SRCS
    "tokenizer_batch_encode_fuzz.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::tokenizer::testing::fuzzing_util
)

iree_cc_fuzz(
  NAME
    tokenizer_batch_decode_fuzz
  SRCS
    "tokenizer_batch_decode_fuzz.cc"
  DEPS
    ::tokenizer
    iree::base
    iree::tokenizer::testing::fuzzing_util
)

iree_cc_fuzz(
  NAME
    postprocessor_fuzz
  SRCS
    "postprocessor_fuzz.cc"
  DEPS
    ::postprocessor
    ::types
    iree::base
)

iree_cc_fuzz(
  NAME
    special_tokens_fuzz
  SRCS
    "special_tokens_fuzz.cc"
  DEPS
    ::special_tokens
    iree::base
)

### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
