package nx-text

You can search for identifiers within the package.

in-package search v0.2.0

Legend:
Page
Library
Module
Module type
Parameter
Class
Class type
Source

Module `Nx_text.Vocab`Source

Sourcetype t = vocab

Sourceval create : unit -> t

Create empty vocabulary

Sourceval add : t -> string -> unit

Add token to vocabulary

Sourceval add_batch : t -> string list -> unit

Add multiple tokens

Sourceval get_index : t -> string -> int option

Get token index

Sourceval get_token : t -> int -> string option

Get token by index

Sourceval from_tokens : ?max_size:int -> ?min_freq:int -> string list -> t

Build from token list

Sourceval size : t -> int

Vocabulary size

Special token indices

Sourceval pad_idx : t -> int

Sourceval unk_idx : t -> int

Sourceval bos_idx : t -> int

Sourceval eos_idx : t -> int