package nx-text

  1. Overview
  2. Docs

Module Nx_text.TokenizerSource

Sourcetype 'a t

Tokenizer with tag type for configuration

Sourceval words : [ `Words ] t

Whitespace and punctuation tokenizer

Sourceval chars : [ `Chars ] t

Unicode character tokenizer

Sourceval regex : string -> [ `Regex ] t

Regex-based tokenizer

Sourceval bpe : vocab:string -> merges:string -> [ `BPE ] t

Byte-Pair Encoding tokenizer

Sourceval wordpiece : vocab:string -> unk_token:string -> [ `WordPiece ] t

WordPiece tokenizer

Sourceval run : _ t -> string -> string list

Apply tokenizer to text

Sourceval run_with_offsets : _ t -> string -> (string * int * int) list

Tokenize with character offsets

Sourceval with_normalizer : (string -> string) -> 'a t -> 'a t

Add text normalizer to tokenizer

Sourceval with_pre_tokenizer : (string -> string list) -> 'a t -> 'a t

Add pre-tokenization step