package saga

  1. Overview
  2. Docs

Module Saga_tokenizers.Added_tokenSource

Sourcetype t

Added token with customization, matching HF Added_token.

Sourceval create : ?content:string -> ?single_word:bool -> ?lstrip:bool -> ?rstrip:bool -> ?normalized:bool -> ?special:bool -> unit -> t

Create added token with options and defaults.

Sourceval content : t -> string
Sourceval lstrip : t -> bool
Sourceval normalized : t -> bool
Sourceval rstrip : t -> bool
Sourceval single_word : t -> bool