tokenizers

Tokenizers are used to prepare textual inputs for a model.

Example: Create an AutoTokenizer and use it to tokenize a sentence. This will automatically detect the tokenizer type based on the tokenizer class defined in tokenizer.json.

import { AutoTokenizer } from '@huggingface/transformers';

const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');
const { input_ids } = await tokenizer('I love transformers!');
// Tensor {
//   data: BigInt64Array(6) [101n, 1045n, 2293n, 19081n, 999n, 102n],
//   dims: [1, 6],
//   type: 'int64',
//   size: 6,
// }

• tokenizers
  • static
    • .TokenizerModel ⇐ Callable
      • new TokenizerModel(config)
      • instance
        • .vocab : Array.<string>
        • .tokens_to_ids : Map.<string, number>
        • .fuse_unk : boolean
        • ._call(tokens) ⇒ Array.<string>
        • .encode(tokens) ⇒ Array.<string>
        • .convert_tokens_to_ids(tokens) ⇒ Array.<number>
        • .convert_ids_to_tokens(ids) ⇒ Array.<string>
      • static
        • .fromConfig(config, ...args) ⇒ TokenizerModel
    • .PreTrainedTokenizer
      • new PreTrainedTokenizer(tokenizerJSON, tokenizerConfig)
      • instance
        • .added_tokens : Array.<AddedToken>
        • .remove_space : boolean
        • ._call(text, options) ⇒ BatchEncoding
        • ._encode_text(text) ⇒ Array<string> | null
        • ._tokenize_helper(text, options) ⇒ *
        • .tokenize(text, options) ⇒ Array.<string>
        • .encode(text, options) ⇒ Array.<number>
        • .batch_decode(batch, decode_args) ⇒ Array.<string>
        • .decode(token_ids, [decode_args]) ⇒ string
        • .decode_single(token_ids, decode_args) ⇒ string
        • .get_chat_template(options) ⇒ string
        • .apply_chat_template(conversation, options) ⇒ string | Tensor | Array<number> | Array<Array<number>> | BatchEncoding
      • static
        • .from_pretrained(pretrained_model_name_or_path, options) ⇒ Promise.<PreTrainedTokenizer>
    • .BertTokenizer ⇐ PreTrainedTokenizer
    • .AlbertTokenizer ⇐ PreTrainedTokenizer
    • .NllbTokenizer
      • ._build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) ⇒ Object
    • .M2M100Tokenizer
      • ._build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) ⇒ Object
    • .WhisperTokenizer ⇐ PreTrainedTokenizer
      • ._decode_asr(sequences, options) ⇒ *
      • .decode() : *
    • .MarianTokenizer
      • new MarianTokenizer(tokenizerJSON, tokenizerConfig)
      • ._encode_text(text) ⇒ Array
    • .AutoTokenizer
      • new AutoTokenizer()
      • .from_pretrained(pretrained_model_name_or_path, options) ⇒ Promise.<PreTrainedTokenizer>
    • .is_chinese_char(cp) ⇒ boolean
  • inner
    • ~AddedToken
      • new AddedToken(config)
    • ~WordPieceTokenizer ⇐ TokenizerModel
      • new WordPieceTokenizer(config)
      • .tokens_to_ids : Map.<string, number>
      • .unk_token_id : number
      • .unk_token : string
      • .max_input_chars_per_word : number
      • .vocab : Array.<string>
      • .encode(tokens) ⇒ Array.<string>
    • ~Unigram ⇐ TokenizerModel
      • new Unigram(config, moreConfig)
      • .populateNodes(lattice)
      • .tokenize(normalized) ⇒ Array.<string>
      • .encode(tokens) ⇒ Array.<string>
    • ~BPE ⇐ TokenizerModel
      • new BPE(config)
      • .tokens_to_ids : Map.<string, number>
      • .merges : *
        • .config.merges : *
      • .cache : Map.<string, Array<string>>
      • .bpe(token) ⇒ Array.<string>
      • .encode(tokens) ⇒ Array.<string>
    • ~LegacyTokenizerModel
      • new LegacyTokenizerModel(config, moreConfig)
      • .tokens_to_ids : Map.<string, number>
    • ~Normalizer
      • new Normalizer(config)
      • instance
        • .normalize(text) ⇒ string
        • ._call(text) ⇒ string
      • static
        • .fromConfig(config) ⇒ Normalizer
    • ~Replace ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~NFC ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~NFKC ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~NFKD ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~StripNormalizer
      • .normalize(text) ⇒ string
    • ~StripAccents ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~Lowercase ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~Prepend ⇐ Normalizer
      • .normalize(text) ⇒ string
    • ~NormalizerSequence ⇐ Normalizer
      • new NormalizerSequence(config)
      • .normalize(text) ⇒ string
    • ~BertNormalizer ⇐ Normalizer
      • ._tokenize_chinese_chars(text) ⇒ string
      • .stripAccents(text) ⇒ string
      • .normalize(text) ⇒ string
    • ~PreTokenizer ⇐ Callable
      • instance
        • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
        • .pre_tokenize(text, [options]) ⇒ Array.<string>
        • ._call(text, [options]) ⇒ Array.<string>
      • static
        • .fromConfig(config) ⇒ PreTokenizer
    • ~BertPreTokenizer ⇐ PreTokenizer
      • new BertPreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~ByteLevelPreTokenizer ⇐ PreTokenizer
      • new ByteLevelPreTokenizer(config)
      • .add_prefix_space : boolean
      • .trim_offsets : boolean
      • .use_regex : boolean
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~SplitPreTokenizer ⇐ PreTokenizer
      • new SplitPreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~PunctuationPreTokenizer ⇐ PreTokenizer
      • new PunctuationPreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~DigitsPreTokenizer ⇐ PreTokenizer
      • new DigitsPreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~PostProcessor ⇐ Callable
      • new PostProcessor(config)
      • instance
        • .post_process(tokens, ...args) ⇒ PostProcessedOutput
        • ._call(tokens, ...args) ⇒ PostProcessedOutput
      • static
        • .fromConfig(config) ⇒ PostProcessor
    • ~BertProcessing
      • new BertProcessing(config)
      • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput
    • ~TemplateProcessing ⇐ PostProcessor
      • new TemplateProcessing(config)
      • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput
    • ~ByteLevelPostProcessor ⇐ PostProcessor
      • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput
    • ~PostProcessorSequence
      • new PostProcessorSequence(config)
      • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput
    • ~Decoder ⇐ Callable
      • new Decoder(config)
      • instance
        • .added_tokens : Array.<AddedToken>
        • ._call(tokens) ⇒ string
        • .decode(tokens) ⇒ string
        • .decode_chain(tokens) ⇒ Array.<string>
      • static
        • .fromConfig(config) ⇒ Decoder
    • ~FuseDecoder
      • .decode_chain() : *
    • ~WordPieceDecoder ⇐ Decoder
      • new WordPieceDecoder(config)
      • .decode_chain() : *
    • ~ByteLevelDecoder ⇐ Decoder
      • new ByteLevelDecoder(config)
      • .convert_tokens_to_string(tokens) ⇒ string
      • .decode_chain() : *
    • ~CTCDecoder
      • .convert_tokens_to_string(tokens) ⇒ string
      • .decode_chain() : *
    • ~DecoderSequence ⇐ Decoder
      • new DecoderSequence(config)
      • .decode_chain() : *
    • ~MetaspacePreTokenizer ⇐ PreTokenizer
      • new MetaspacePreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~MetaspaceDecoder ⇐ Decoder
      • new MetaspaceDecoder(config)
      • .decode_chain() : *
    • ~Precompiled ⇐ Normalizer
      • new Precompiled(config)
      • .normalize(text) ⇒ string
    • ~PreTokenizerSequence ⇐ PreTokenizer
      • new PreTokenizerSequence(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~WhitespacePreTokenizer
      • new WhitespacePreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~WhitespaceSplit ⇐ PreTokenizer
      • new WhitespaceSplit(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~ReplacePreTokenizer
      • new ReplacePreTokenizer(config)
      • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • ~BYTES_TO_UNICODE ⇒ Object
    • ~loadTokenizer(pretrained_model_name_or_path, options) ⇒ Promise.<Array<any>>
    • ~regexSplit(text, regex) ⇒ Array.<string>
    • ~createPattern(pattern, invert) ⇒ RegExp | null
    • ~objectToMap(obj) ⇒ Map.<string, any>
    • ~prepareTensorForDecode(tensor) ⇒ Array.<number>
    • ~clean_up_tokenization(text) ⇒ string
    • ~remove_accents(text) ⇒ string
    • ~lowercase_and_remove_accent(text) ⇒ string
    • ~whitespace_split(text) ⇒ Array.<string>
    • ~PretrainedTokenizerOptions : Object
    • ~BPENode : Object
    • ~SplitDelimiterBehavior : ’removed’ | ’isolated’ | ’mergedWithPrevious’ | ’mergedWithNext’ | ’contiguous’
    • ~PostProcessedOutput : Object
    • ~EncodingSingle : Object
    • ~Message : Object
    • ~BatchEncoding : Array<number> | Array<Array<number>> | Tensor

tokenizers.TokenizerModel ⇐ <code> Callable </code>

Abstract base class for tokenizer models.

Kind: static class of tokenizers
Extends: Callable

• .TokenizerModel ⇐ Callable
  • new TokenizerModel(config)
  • instance
    • .vocab : Array.<string>
    • .tokens_to_ids : Map.<string, number>
    • .fuse_unk : boolean
    • ._call(tokens) ⇒ Array.<string>
    • .encode(tokens) ⇒ Array.<string>
    • .convert_tokens_to_ids(tokens) ⇒ Array.<number>
    • .convert_ids_to_tokens(ids) ⇒ Array.<string>
  • static
    • .fromConfig(config, ...args) ⇒ TokenizerModel

new TokenizerModel(config)

Creates a new instance of TokenizerModel.

tokenizerModel.vocab : <code> Array. < string > </code>

Kind: instance property of TokenizerModel

tokenizerModel.tokens_to_ids : <code> Map. < string, number > </code>

A mapping of tokens to ids.

Kind: instance property of TokenizerModel

tokenizerModel.fuse_unk : <code> boolean </code>

Whether to fuse unknown tokens when encoding. Defaults to false.

Kind: instance property of TokenizerModel

tokenizerModel._call(tokens) ⇒ <code> Array. < string > </code>

Internal function to call the TokenizerModel instance.

Kind: instance method of TokenizerModel
Overrides: _call
Returns: Array.<string> - The encoded tokens.

tokenizerModel.encode(tokens) ⇒ <code> Array. < string > </code>

Encodes a list of tokens into a list of token IDs.

Kind: instance method of TokenizerModel
Returns: Array.<string> - The encoded tokens.
Throws:

• Will throw an error if not implemented in a subclass.

tokenizerModel.convert_tokens_to_ids(tokens) ⇒ <code> Array. < number > </code>

Converts a list of tokens into a list of token IDs.

Kind: instance method of TokenizerModel
Returns: Array.<number> - The converted token IDs.

tokenizerModel.convert_ids_to_tokens(ids) ⇒ <code> Array. < string > </code>

Converts a list of token IDs into a list of tokens.

Kind: instance method of TokenizerModel
Returns: Array.<string> - The converted tokens.

TokenizerModel.fromConfig(config, ...args) ⇒ <code> TokenizerModel </code>

Instantiates a new TokenizerModel instance based on the configuration object provided.

Kind: static method of TokenizerModel
Returns: TokenizerModel - A new instance of a TokenizerModel.
Throws:

• Will throw an error if the TokenizerModel type in the config is not recognized.

tokenizers.PreTrainedTokenizer

Kind: static class of tokenizers

• .PreTrainedTokenizer
  • new PreTrainedTokenizer(tokenizerJSON, tokenizerConfig)
  • instance
    • .added_tokens : Array.<AddedToken>
    • .remove_space : boolean
    • ._call(text, options) ⇒ BatchEncoding
    • ._encode_text(text) ⇒ Array<string> | null
    • ._tokenize_helper(text, options) ⇒ *
    • .tokenize(text, options) ⇒ Array.<string>
    • .encode(text, options) ⇒ Array.<number>
    • .batch_decode(batch, decode_args) ⇒ Array.<string>
    • .decode(token_ids, [decode_args]) ⇒ string
    • .decode_single(token_ids, decode_args) ⇒ string
    • .get_chat_template(options) ⇒ string
    • .apply_chat_template(conversation, options) ⇒ string | Tensor | Array<number> | Array<Array<number>> | BatchEncoding
  • static
    • .from_pretrained(pretrained_model_name_or_path, options) ⇒ Promise.<PreTrainedTokenizer>

new PreTrainedTokenizer(tokenizerJSON, tokenizerConfig)

Create a new PreTrainedTokenizer instance.

preTrainedTokenizer.added_tokens : <code> Array. < AddedToken > </code>

Kind: instance property of PreTrainedTokenizer

preTrainedTokenizer.remove_space : <code> boolean </code>

Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).

Kind: instance property of PreTrainedTokenizer

preTrainedTokenizer._call(text, options) ⇒ <code> BatchEncoding </code>

Encode/tokenize the given text(s).

Kind: instance method of PreTrainedTokenizer
Returns: BatchEncoding - Object to be passed to the model.

preTrainedTokenizer._encode_text(text) ⇒ <code> Array < string > </code> | <code> null </code>

Encodes a single text using the preprocessor pipeline of the tokenizer.

Kind: instance method of PreTrainedTokenizer
Returns: Array<string> | null - The encoded tokens.

preTrainedTokenizer._tokenize_helper(text, options) ⇒ <code> * </code>

Internal helper function to tokenize a text, and optionally a pair of texts.

Kind: instance method of PreTrainedTokenizer
Returns: * - An object containing the tokens and optionally the token type IDs.

preTrainedTokenizer.tokenize(text, options) ⇒ <code> Array. < string > </code>

Converts a string into a sequence of tokens.

Kind: instance method of PreTrainedTokenizer
Returns: Array.<string> - The list of tokens.

preTrainedTokenizer.encode(text, options) ⇒ <code> Array. < number > </code>

Encodes a single text or a pair of texts using the model’s tokenizer.

Kind: instance method of PreTrainedTokenizer
Returns: Array.<number> - An array of token IDs representing the encoded text(s).

preTrainedTokenizer.batch_decode(batch, decode_args) ⇒ <code> Array. < string > </code>

Decode a batch of tokenized sequences.

Kind: instance method of PreTrainedTokenizer
Returns: Array.<string> - List of decoded sequences.

preTrainedTokenizer.decode(token_ids, [decode_args]) ⇒ <code> string </code>

Decodes a sequence of token IDs back to a string.

Kind: instance method of PreTrainedTokenizer
Returns: string - The decoded string.
Throws:

• Error If `token_ids` is not a non-empty array of integers.

preTrainedTokenizer.decode_single(token_ids, decode_args) ⇒ <code> string </code>

Decode a single list of token ids to a string.

Kind: instance method of PreTrainedTokenizer
Returns: string - The decoded string

preTrainedTokenizer.get_chat_template(options) ⇒ <code> string </code>

Retrieve the chat template string used for tokenizing chat messages. This template is used internally by the apply_chat_template method and can also be used externally to retrieve the model’s chat template for better generation tracking.

Kind: instance method of PreTrainedTokenizer
Returns: string - The chat template string.

preTrainedTokenizer.apply_chat_template(conversation, options) ⇒ <code> string </code> | <code> Tensor </code> | <code> Array < number > </code> | <code> Array < Array < number > > </code> | <code> BatchEncoding </code>

Converts a list of message objects with "role" and "content" keys to a list of token ids. This method is intended for use with chat models, and will read the tokenizer’s chat_template attribute to determine the format and control tokens to use when converting.

See here for more information.

Example: Applying a chat template to a conversation.

import { AutoTokenizer } from "@huggingface/transformers";

const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1");

const chat = [
  { "role": "user", "content": "Hello, how are you?" },
  { "role": "assistant", "content": "I'm doing great. How can I help you today?" },
  { "role": "user", "content": "I'd like to show off how chat templating works!" },
]

const text = tokenizer.apply_chat_template(chat, { tokenize: false });
// "<s>[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?</s> [INST] I'd like to show off how chat templating works! [/INST]"

const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false });
// [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793]

Kind: instance method of PreTrainedTokenizer
Returns: string | Tensor | Array<number> | Array<Array<number>> | BatchEncoding - The tokenized output.

PreTrainedTokenizer.from_pretrained(pretrained_model_name_or_path, options) ⇒ <code> Promise. < PreTrainedTokenizer > </code>

Loads a pre-trained tokenizer from the given pretrained_model_name_or_path.

Kind: static method of PreTrainedTokenizer
Returns: Promise.<PreTrainedTokenizer> - A new instance of the PreTrainedTokenizer class.
Throws:

• Error Throws an error if the tokenizer.json or tokenizer_config.json files are not found in the `pretrained_model_name_or_path`.

tokenizers.BertTokenizer ⇐ <code> PreTrainedTokenizer </code>

BertTokenizer is a class used to tokenize text for BERT models.

Kind: static class of tokenizers
Extends: PreTrainedTokenizer

tokenizers.AlbertTokenizer ⇐ <code> PreTrainedTokenizer </code>

Albert tokenizer

Kind: static class of tokenizers
Extends: PreTrainedTokenizer

tokenizers.NllbTokenizer

The NllbTokenizer class is used to tokenize text for NLLB (“No Language Left Behind”) models.

No Language Left Behind (NLLB) is a first-of-its-kind, AI breakthrough project that open-sources models capable of delivering high-quality translations directly between any pair of 200+ languages — including low-resource languages like Asturian, Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere, regardless of their language preferences. For more information, check out their paper.

For a list of supported languages (along with their language codes),

Kind: static class of tokenizers
See: https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200

nllbTokenizer._build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) ⇒ <code> Object </code>

Helper function to build translation inputs for an NllbTokenizer.

Kind: instance method of NllbTokenizer
Returns: Object - Object to be passed to the model.

tokenizers.M2M100Tokenizer

The M2M100Tokenizer class is used to tokenize text for M2M100 (“Many-to-Many”) models.

M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many multilingual translation. It was introduced in this paper and first released in this repository.

For a list of supported languages (along with their language codes),

Kind: static class of tokenizers
See: https://huggingface.co./facebook/m2m100_418M#languages-covered

m2M100Tokenizer._build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) ⇒ <code> Object </code>

Helper function to build translation inputs for an M2M100Tokenizer.

Kind: instance method of M2M100Tokenizer
Returns: Object - Object to be passed to the model.

tokenizers.WhisperTokenizer ⇐ <code> PreTrainedTokenizer </code>

WhisperTokenizer tokenizer

Kind: static class of tokenizers
Extends: PreTrainedTokenizer

• .WhisperTokenizer ⇐ PreTrainedTokenizer
  • ._decode_asr(sequences, options) ⇒ *
  • .decode() : *

whisperTokenizer._decode_asr(sequences, options) ⇒ <code> * </code>

Decodes automatic speech recognition (ASR) sequences.

Kind: instance method of WhisperTokenizer
Returns: * - The decoded sequences.

whisperTokenizer.decode() : <code> * </code>

Kind: instance method of WhisperTokenizer

tokenizers.MarianTokenizer

Kind: static class of tokenizers
Todo

• This model is not yet supported by Hugging Face’s “fast” tokenizers library (https://github.com/huggingface/tokenizers). Therefore, this implementation (which is based on fast tokenizers) may produce slightly inaccurate results.

• .MarianTokenizer
  • new MarianTokenizer(tokenizerJSON, tokenizerConfig)
  • ._encode_text(text) ⇒ Array

new MarianTokenizer(tokenizerJSON, tokenizerConfig)

Create a new MarianTokenizer instance.

marianTokenizer._encode_text(text) ⇒ <code> Array </code>

Encodes a single text. Overriding this method is necessary since the language codes must be removed before encoding with sentencepiece model.

Kind: instance method of MarianTokenizer
Returns: Array - The encoded tokens.
See: https://github.com/huggingface/transformers/blob/12d51db243a00726a548a43cc333390ebae731e3/src/transformers/models/marian/tokenization_marian.py#L204-L213

tokenizers.AutoTokenizer

Helper class which is used to instantiate pretrained tokenizers with the from_pretrained function. The chosen tokenizer class is determined by the type specified in the tokenizer config.

Kind: static class of tokenizers

• .AutoTokenizer
  • new AutoTokenizer()
  • .from_pretrained(pretrained_model_name_or_path, options) ⇒ Promise.<PreTrainedTokenizer>

new AutoTokenizer()

Example

const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');

AutoTokenizer.from_pretrained(pretrained_model_name_or_path, options) ⇒ <code> Promise. < PreTrainedTokenizer > </code>

Instantiate one of the tokenizer classes of the library from a pretrained model.

The tokenizer class to instantiate is selected based on the tokenizer_class property of the config object (either passed as an argument or loaded from pretrained_model_name_or_path if possible)

Kind: static method of AutoTokenizer
Returns: Promise.<PreTrainedTokenizer> - A new instance of the PreTrainedTokenizer class.

tokenizers.is_chinese_char(cp) ⇒ <code> boolean </code>

Checks whether the given Unicode codepoint represents a CJK (Chinese, Japanese, or Korean) character.

A “chinese character” is defined as anything in the CJK Unicode block: https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)

Note that the CJK Unicode block is NOT all Japanese and Korean characters, despite its name. The modern Korean Hangul alphabet is a different block, as is Japanese Hiragana and Katakana. Those alphabets are used to write space-separated words, so they are not treated specially and are handled like all other languages.

Kind: static method of tokenizers
Returns: boolean - True if the codepoint represents a CJK character, false otherwise.

tokenizers~AddedToken

Represent a token added by the user on top of the existing Model vocabulary. AddedToken can be configured to specify the behavior they should have in various situations like:

• Whether they should only match single words
• Whether to include any whitespace on its left or right

Kind: inner class of tokenizers

new AddedToken(config)

Creates a new instance of AddedToken.

tokenizers~WordPieceTokenizer ⇐ <code> TokenizerModel </code>

A subclass of TokenizerModel that uses WordPiece encoding to encode tokens.

Kind: inner class of tokenizers
Extends: TokenizerModel

• ~WordPieceTokenizer ⇐ TokenizerModel
  • new WordPieceTokenizer(config)
  • .tokens_to_ids : Map.<string, number>
  • .unk_token_id : number
  • .unk_token : string
  • .max_input_chars_per_word : number
  • .vocab : Array.<string>
  • .encode(tokens) ⇒ Array.<string>

new WordPieceTokenizer(config)

wordPieceTokenizer.tokens_to_ids : <code> Map. < string, number > </code>

A mapping of tokens to ids.

Kind: instance property of WordPieceTokenizer

wordPieceTokenizer.unk_token_id : <code> number </code>

The id of the unknown token.

Kind: instance property of WordPieceTokenizer

wordPieceTokenizer.unk_token : <code> string </code>

The unknown token string.

Kind: instance property of WordPieceTokenizer

wordPieceTokenizer.max_input_chars_per_word : <code> number </code>

The maximum number of characters allowed per word.

Kind: instance property of WordPieceTokenizer

wordPieceTokenizer.vocab : <code> Array. < string > </code>

An array of tokens.

Kind: instance property of WordPieceTokenizer

wordPieceTokenizer.encode(tokens) ⇒ <code> Array. < string > </code>

Encodes an array of tokens using WordPiece encoding.

Kind: instance method of WordPieceTokenizer
Returns: Array.<string> - An array of encoded tokens.

tokenizers~Unigram ⇐ <code> TokenizerModel </code>

Class representing a Unigram tokenizer model.

Kind: inner class of tokenizers
Extends: TokenizerModel

• ~Unigram ⇐ TokenizerModel
  • new Unigram(config, moreConfig)
  • .populateNodes(lattice)
  • .tokenize(normalized) ⇒ Array.<string>
  • .encode(tokens) ⇒ Array.<string>

new Unigram(config, moreConfig)

Create a new Unigram tokenizer model.

unigram.populateNodes(lattice)

Populates lattice nodes.

Kind: instance method of Unigram

unigram.tokenize(normalized) ⇒ <code> Array. < string > </code>

Encodes an array of tokens into an array of subtokens using the unigram model.

Kind: instance method of Unigram
Returns: Array.<string> - An array of subtokens obtained by encoding the input tokens using the unigram model.

unigram.encode(tokens) ⇒ <code> Array. < string > </code>

Encodes an array of tokens using Unigram encoding.

Kind: instance method of Unigram
Returns: Array.<string> - An array of encoded tokens.

tokenizers~BPE ⇐ <code> TokenizerModel </code>

BPE class for encoding text into Byte-Pair-Encoding (BPE) tokens.

Kind: inner class of tokenizers
Extends: TokenizerModel

• ~BPE ⇐ TokenizerModel
  • new BPE(config)
  • .tokens_to_ids : Map.<string, number>
  • .merges : *
    • .config.merges : *
  • .cache : Map.<string, Array<string>>
  • .bpe(token) ⇒ Array.<string>
  • .encode(tokens) ⇒ Array.<string>

new BPE(config)

Create a BPE instance.

bpE.tokens_to_ids : <code> Map. < string, number > </code>

Kind: instance property of BPE

bpE.merges : <code> * </code>

Kind: instance property of BPE

merges.config.merges : <code> * </code>

Kind: static property of merges

bpE.cache : <code> Map. < string, Array < string > > </code>

Kind: instance property of BPE

bpE.bpe(token) ⇒ <code> Array. < string > </code>

Apply Byte-Pair-Encoding (BPE) to a given token. Efficient heap-based priority queue implementation adapted from https://github.com/belladoreai/llama-tokenizer-js.

Kind: instance method of BPE
Returns: Array.<string> - The BPE encoded tokens.

bpE.encode(tokens) ⇒ <code> Array. < string > </code>

Encodes the input sequence of tokens using the BPE algorithm and returns the resulting subword tokens.

Kind: instance method of BPE
Returns: Array.<string> - The resulting subword tokens after applying the BPE algorithm to the input sequence of tokens.

tokenizers~LegacyTokenizerModel

Legacy tokenizer class for tokenizers with only a vocabulary.

Kind: inner class of tokenizers

• ~LegacyTokenizerModel
  • new LegacyTokenizerModel(config, moreConfig)
  • .tokens_to_ids : Map.<string, number>

new LegacyTokenizerModel(config, moreConfig)

Create a LegacyTokenizerModel instance.

legacyTokenizerModel.tokens_to_ids : <code> Map. < string, number > </code>

Kind: instance property of LegacyTokenizerModel

tokenizers~Normalizer

A base class for text normalization.

Kind: inner abstract class of tokenizers

• ~Normalizer
  • new Normalizer(config)
  • instance
    • .normalize(text) ⇒ string
    • ._call(text) ⇒ string
  • static
    • .fromConfig(config) ⇒ Normalizer

new Normalizer(config)

normalizer.normalize(text) ⇒ <code> string </code>

Normalize the input text.

Kind: instance abstract method of Normalizer
Returns: string - The normalized text.
Throws:

• Error If this method is not implemented in a subclass.

normalizer._call(text) ⇒ <code> string </code>

Alias for Normalizer#normalize.

Kind: instance method of Normalizer
Returns: string - The normalized text.

Normalizer.fromConfig(config) ⇒ <code> Normalizer </code>

Factory method for creating normalizers from config objects.

Kind: static method of Normalizer
Returns: Normalizer - A Normalizer object.
Throws:

• Error If an unknown Normalizer type is specified in the config.

tokenizers~Replace ⇐ <code> Normalizer </code>

Replace normalizer that replaces occurrences of a pattern with a given string or regular expression.

Kind: inner class of tokenizers
Extends: Normalizer

replace.normalize(text) ⇒ <code> string </code>

Normalize the input text by replacing the pattern with the content.

Kind: instance method of Replace
Returns: string - The normalized text after replacing the pattern with the content.

tokenizers~NFC ⇐ <code> Normalizer </code>

A normalizer that applies Unicode normalization form C (NFC) to the input text.

Kind: inner class of tokenizers
Extends: Normalizer

nfC.normalize(text) ⇒ <code> string </code>

Normalize the input text by applying Unicode normalization form C (NFC).

Kind: instance method of NFC
Returns: string - The normalized text.

tokenizers~NFKC ⇐ <code> Normalizer </code>

NFKC Normalizer.

Kind: inner class of tokenizers
Extends: Normalizer

nfkC.normalize(text) ⇒ <code> string </code>

Normalize text using NFKC normalization.

Kind: instance method of NFKC
Returns: string - The normalized text.

tokenizers~NFKD ⇐ <code> Normalizer </code>

NFKD Normalizer.

Kind: inner class of tokenizers
Extends: Normalizer

nfkD.normalize(text) ⇒ <code> string </code>

Normalize text using NFKD normalization.

Kind: instance method of NFKD
Returns: string - The normalized text.

tokenizers~StripNormalizer

A normalizer that strips leading and/or trailing whitespace from the input text.

Kind: inner class of tokenizers

stripNormalizer.normalize(text) ⇒ <code> string </code>

Strip leading and/or trailing whitespace from the input text.

Kind: instance method of StripNormalizer
Returns: string - The normalized text.

tokenizers~StripAccents ⇐ <code> Normalizer </code>

StripAccents normalizer removes all accents from the text.

Kind: inner class of tokenizers
Extends: Normalizer

stripAccents.normalize(text) ⇒ <code> string </code>

Remove all accents from the text.

Kind: instance method of StripAccents
Returns: string - The normalized text without accents.

tokenizers~Lowercase ⇐ <code> Normalizer </code>

A Normalizer that lowercases the input string.

Kind: inner class of tokenizers
Extends: Normalizer

lowercase.normalize(text) ⇒ <code> string </code>

Lowercases the input string.

Kind: instance method of Lowercase
Returns: string - The normalized text.

tokenizers~Prepend ⇐ <code> Normalizer </code>

A Normalizer that prepends a string to the input string.

Kind: inner class of tokenizers
Extends: Normalizer

prepend.normalize(text) ⇒ <code> string </code>

Prepends the input string.

Kind: instance method of Prepend
Returns: string - The normalized text.

tokenizers~NormalizerSequence ⇐ <code> Normalizer </code>

A Normalizer that applies a sequence of Normalizers.

Kind: inner class of tokenizers
Extends: Normalizer

• ~NormalizerSequence ⇐ Normalizer
  • new NormalizerSequence(config)
  • .normalize(text) ⇒ string

new NormalizerSequence(config)

Create a new instance of NormalizerSequence.

normalizerSequence.normalize(text) ⇒ <code> string </code>

Apply a sequence of Normalizers to the input text.

Kind: instance method of NormalizerSequence
Returns: string - The normalized text.

tokenizers~BertNormalizer ⇐ <code> Normalizer </code>

A class representing a normalizer used in BERT tokenization.

Kind: inner class of tokenizers
Extends: Normalizer

• ~BertNormalizer ⇐ Normalizer
  • ._tokenize_chinese_chars(text) ⇒ string
  • .stripAccents(text) ⇒ string
  • .normalize(text) ⇒ string

bertNormalizer._tokenize_chinese_chars(text) ⇒ <code> string </code>

Adds whitespace around any CJK (Chinese, Japanese, or Korean) character in the input text.

Kind: instance method of BertNormalizer
Returns: string - The tokenized text with whitespace added around CJK characters.

bertNormalizer.stripAccents(text) ⇒ <code> string </code>

Strips accents from the given text.

Kind: instance method of BertNormalizer
Returns: string - The text with accents removed.

bertNormalizer.normalize(text) ⇒ <code> string </code>

Normalizes the given text based on the configuration.

Kind: instance method of BertNormalizer
Returns: string - The normalized text.

tokenizers~PreTokenizer ⇐ <code> Callable </code>

A callable class representing a pre-tokenizer used in tokenization. Subclasses should implement the pre_tokenize_text method to define the specific pre-tokenization logic.

Kind: inner class of tokenizers
Extends: Callable

• ~PreTokenizer ⇐ Callable
  • instance
    • .pre_tokenize_text(text, [options]) ⇒ Array.<string>
    • .pre_tokenize(text, [options]) ⇒ Array.<string>
    • ._call(text, [options]) ⇒ Array.<string>
  • static
    • .fromConfig(config) ⇒ PreTokenizer

preTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Method that should be implemented by subclasses to define the specific pre-tokenization logic.

Kind: instance abstract method of PreTokenizer
Returns: Array.<string> - The pre-tokenized text.
Throws:

• Error If the method is not implemented in the subclass.

preTokenizer.pre_tokenize(text, [options]) ⇒ <code> Array. < string > </code>

Tokenizes the given text into pre-tokens.

Kind: instance method of PreTokenizer
Returns: Array.<string> - An array of pre-tokens.

preTokenizer._call(text, [options]) ⇒ <code> Array. < string > </code>

Alias for PreTokenizer#pre_tokenize.

Kind: instance method of PreTokenizer
Overrides: _call
Returns: Array.<string> - An array of pre-tokens.

PreTokenizer.fromConfig(config) ⇒ <code> PreTokenizer </code>

Factory method that returns an instance of a subclass of PreTokenizer based on the provided configuration.

Kind: static method of PreTokenizer
Returns: PreTokenizer - An instance of a subclass of PreTokenizer.
Throws:

• Error If the provided configuration object does not correspond to any known pre-tokenizer.

tokenizers~BertPreTokenizer ⇐ <code> PreTokenizer </code>

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~BertPreTokenizer ⇐ PreTokenizer
  • new BertPreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new BertPreTokenizer(config)

A PreTokenizer that splits text into wordpieces using a basic tokenization scheme similar to that used in the original implementation of BERT.

bertPreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Tokenizes a single text using the BERT pre-tokenization scheme.

Kind: instance method of BertPreTokenizer
Returns: Array.<string> - An array of tokens.

tokenizers~ByteLevelPreTokenizer ⇐ <code> PreTokenizer </code>

A pre-tokenizer that splits text into Byte-Pair-Encoding (BPE) subwords.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~ByteLevelPreTokenizer ⇐ PreTokenizer
  • new ByteLevelPreTokenizer(config)
  • .add_prefix_space : boolean
  • .trim_offsets : boolean
  • .use_regex : boolean
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new ByteLevelPreTokenizer(config)

Creates a new instance of the ByteLevelPreTokenizer class.

byteLevelPreTokenizer.add_prefix_space : <code> boolean </code>

Whether to add a leading space to the first word.This allows to treat the leading word just as any other word.

Kind: instance property of ByteLevelPreTokenizer

byteLevelPreTokenizer.trim_offsets : <code> boolean </code>

Whether the post processing step should trim offsetsto avoid including whitespaces.

Kind: instance property of ByteLevelPreTokenizer
Todo

• Use this in the pretokenization step.

byteLevelPreTokenizer.use_regex : <code> boolean </code>

Whether to use the standard GPT2 regex for whitespace splitting.Set it to False if you want to use your own splitting. Defaults to true.

Kind: instance property of ByteLevelPreTokenizer

byteLevelPreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Tokenizes a single piece of text using byte-level tokenization.

Kind: instance method of ByteLevelPreTokenizer
Returns: Array.<string> - An array of tokens.

tokenizers~SplitPreTokenizer ⇐ <code> PreTokenizer </code>

Splits text using a given pattern.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~SplitPreTokenizer ⇐ PreTokenizer
  • new SplitPreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new SplitPreTokenizer(config)

splitPreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Tokenizes text by splitting it using the given pattern.

Kind: instance method of SplitPreTokenizer
Returns: Array.<string> - An array of tokens.

tokenizers~PunctuationPreTokenizer ⇐ <code> PreTokenizer </code>

Splits text based on punctuation.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~PunctuationPreTokenizer ⇐ PreTokenizer
  • new PunctuationPreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new PunctuationPreTokenizer(config)

punctuationPreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Tokenizes text by splitting it using the given pattern.

Kind: instance method of PunctuationPreTokenizer
Returns: Array.<string> - An array of tokens.

tokenizers~DigitsPreTokenizer ⇐ <code> PreTokenizer </code>

Splits text based on digits.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~DigitsPreTokenizer ⇐ PreTokenizer
  • new DigitsPreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new DigitsPreTokenizer(config)

digitsPreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Tokenizes text by splitting it using the given pattern.

Kind: instance method of DigitsPreTokenizer
Returns: Array.<string> - An array of tokens.

tokenizers~PostProcessor ⇐ <code> Callable </code>

Kind: inner class of tokenizers
Extends: Callable

• ~PostProcessor ⇐ Callable
  • new PostProcessor(config)
  • instance
    • .post_process(tokens, ...args) ⇒ PostProcessedOutput
    • ._call(tokens, ...args) ⇒ PostProcessedOutput
  • static
    • .fromConfig(config) ⇒ PostProcessor

new PostProcessor(config)

postProcessor.post_process(tokens, ...args) ⇒ <code> PostProcessedOutput </code>

Method to be implemented in subclass to apply post-processing on the given tokens.

Kind: instance method of PostProcessor
Returns: PostProcessedOutput - The post-processed tokens.
Throws:

• Error If the method is not implemented in subclass.

postProcessor._call(tokens, ...args) ⇒ <code> PostProcessedOutput </code>

Alias for PostProcessor#post_process.

Kind: instance method of PostProcessor
Overrides: _call
Returns: PostProcessedOutput - The post-processed tokens.

PostProcessor.fromConfig(config) ⇒ <code> PostProcessor </code>

Factory method to create a PostProcessor object from a configuration object.

Kind: static method of PostProcessor
Returns: PostProcessor - A PostProcessor object created from the given configuration.
Throws:

• Error If an unknown PostProcessor type is encountered.

tokenizers~BertProcessing

A post-processor that adds special tokens to the beginning and end of the input.

Kind: inner class of tokenizers

• ~BertProcessing
  • new BertProcessing(config)
  • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput

new BertProcessing(config)

bertProcessing.post_process(tokens, [tokens_pair]) ⇒ <code> PostProcessedOutput </code>

Adds the special tokens to the beginning and end of the input.

Kind: instance method of BertProcessing
Returns: PostProcessedOutput - The post-processed tokens with the special tokens added to the beginning and end.

tokenizers~TemplateProcessing ⇐ <code> PostProcessor </code>

Post processor that replaces special tokens in a template with actual tokens.

Kind: inner class of tokenizers
Extends: PostProcessor

• ~TemplateProcessing ⇐ PostProcessor
  • new TemplateProcessing(config)
  • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput

new TemplateProcessing(config)

Creates a new instance of TemplateProcessing.

templateProcessing.post_process(tokens, [tokens_pair]) ⇒ <code> PostProcessedOutput </code>

Replaces special tokens in the template with actual tokens.

Kind: instance method of TemplateProcessing
Returns: PostProcessedOutput - An object containing the list of tokens with the special tokens replaced with actual tokens.

tokenizers~ByteLevelPostProcessor ⇐ <code> PostProcessor </code>

A PostProcessor that returns the given tokens as is.

Kind: inner class of tokenizers
Extends: PostProcessor

byteLevelPostProcessor.post_process(tokens, [tokens_pair]) ⇒ <code> PostProcessedOutput </code>

Post process the given tokens.

Kind: instance method of ByteLevelPostProcessor
Returns: PostProcessedOutput - An object containing the post-processed tokens.

tokenizers~PostProcessorSequence

A post-processor that applies multiple post-processors in sequence.

Kind: inner class of tokenizers

• ~PostProcessorSequence
  • new PostProcessorSequence(config)
  • .post_process(tokens, [tokens_pair]) ⇒ PostProcessedOutput

new PostProcessorSequence(config)

Creates a new instance of PostProcessorSequence.

postProcessorSequence.post_process(tokens, [tokens_pair]) ⇒ <code> PostProcessedOutput </code>

Post process the given tokens.

Kind: instance method of PostProcessorSequence
Returns: PostProcessedOutput - An object containing the post-processed tokens.

tokenizers~Decoder ⇐ <code> Callable </code>

The base class for token decoders.

Kind: inner class of tokenizers
Extends: Callable

• ~Decoder ⇐ Callable
  • new Decoder(config)
  • instance
    • .added_tokens : Array.<AddedToken>
    • ._call(tokens) ⇒ string
    • .decode(tokens) ⇒ string
    • .decode_chain(tokens) ⇒ Array.<string>
  • static
    • .fromConfig(config) ⇒ Decoder

new Decoder(config)

Creates an instance of Decoder.

decoder.added_tokens : <code> Array. < AddedToken > </code>

Kind: instance property of Decoder

decoder._call(tokens) ⇒ <code> string </code>

Calls the decode method.

Kind: instance method of Decoder
Overrides: _call
Returns: string - The decoded string.

decoder.decode(tokens) ⇒ <code> string </code>

Decodes a list of tokens.

Kind: instance method of Decoder
Returns: string - The decoded string.

decoder.decode_chain(tokens) ⇒ <code> Array. < string > </code>

Apply the decoder to a list of tokens.

Kind: instance method of Decoder
Returns: Array.<string> - The decoded list of tokens.
Throws:

• Error If the `decode_chain` method is not implemented in the subclass.

Decoder.fromConfig(config) ⇒ <code> Decoder </code>

Creates a decoder instance based on the provided configuration.

Kind: static method of Decoder
Returns: Decoder - A decoder instance.
Throws:

• Error If an unknown decoder type is provided.

tokenizers~FuseDecoder

Fuse simply fuses all tokens into one big string. It’s usually the last decoding step anyway, but this decoder exists incase some decoders need to happen after that step

Kind: inner class of tokenizers

fuseDecoder.decode_chain() : <code> * </code>

Kind: instance method of FuseDecoder

tokenizers~WordPieceDecoder ⇐ <code> Decoder </code>

A decoder that decodes a list of WordPiece tokens into a single string.

Kind: inner class of tokenizers
Extends: Decoder

• ~WordPieceDecoder ⇐ Decoder
  • new WordPieceDecoder(config)
  • .decode_chain() : *

new WordPieceDecoder(config)

Creates a new instance of WordPieceDecoder.

wordPieceDecoder.decode_chain() : <code> * </code>

Kind: instance method of WordPieceDecoder

tokenizers~ByteLevelDecoder ⇐ <code> Decoder </code>

Byte-level decoder for tokenization output. Inherits from the Decoder class.

Kind: inner class of tokenizers
Extends: Decoder

• ~ByteLevelDecoder ⇐ Decoder
  • new ByteLevelDecoder(config)
  • .convert_tokens_to_string(tokens) ⇒ string
  • .decode_chain() : *

new ByteLevelDecoder(config)

Create a ByteLevelDecoder object.

byteLevelDecoder.convert_tokens_to_string(tokens) ⇒ <code> string </code>

Convert an array of tokens to string by decoding each byte.

Kind: instance method of ByteLevelDecoder
Returns: string - The decoded string.

byteLevelDecoder.decode_chain() : <code> * </code>

Kind: instance method of ByteLevelDecoder

tokenizers~CTCDecoder

The CTC (Connectionist Temporal Classification) decoder. See https://github.com/huggingface/tokenizers/blob/bb38f390a61883fc2f29d659af696f428d1cda6b/tokenizers/src/decoders/ctc.rs

Kind: inner class of tokenizers

• ~CTCDecoder
  • .convert_tokens_to_string(tokens) ⇒ string
  • .decode_chain() : *

ctcDecoder.convert_tokens_to_string(tokens) ⇒ <code> string </code>

Converts a connectionist-temporal-classification (CTC) output tokens into a single string.

Kind: instance method of CTCDecoder
Returns: string - The decoded string.

ctcDecoder.decode_chain() : <code> * </code>

Kind: instance method of CTCDecoder

tokenizers~DecoderSequence ⇐ <code> Decoder </code>

Apply a sequence of decoders.

Kind: inner class of tokenizers
Extends: Decoder

• ~DecoderSequence ⇐ Decoder
  • new DecoderSequence(config)
  • .decode_chain() : *

new DecoderSequence(config)

Creates a new instance of DecoderSequence.

decoderSequence.decode_chain() : <code> * </code>

Kind: instance method of DecoderSequence

tokenizers~MetaspacePreTokenizer ⇐ <code> PreTokenizer </code>

This PreTokenizer replaces spaces with the given replacement character, adds a prefix space if requested, and returns a list of tokens.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~MetaspacePreTokenizer ⇐ PreTokenizer
  • new MetaspacePreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new MetaspacePreTokenizer(config)

metaspacePreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

This method takes a string, replaces spaces with the replacement character, adds a prefix space if requested, and returns a new list of tokens.

Kind: instance method of MetaspacePreTokenizer
Returns: Array.<string> - A new list of pre-tokenized tokens.

tokenizers~MetaspaceDecoder ⇐ <code> Decoder </code>

MetaspaceDecoder class extends the Decoder class and decodes Metaspace tokenization.

Kind: inner class of tokenizers
Extends: Decoder

• ~MetaspaceDecoder ⇐ Decoder
  • new MetaspaceDecoder(config)
  • .decode_chain() : *

new MetaspaceDecoder(config)

Constructs a new MetaspaceDecoder object.

metaspaceDecoder.decode_chain() : <code> * </code>

Kind: instance method of MetaspaceDecoder

tokenizers~Precompiled ⇐ <code> Normalizer </code>

A normalizer that applies a precompiled charsmap. This is useful for applying complex normalizations in C++ and exposing them to JavaScript.

Kind: inner class of tokenizers
Extends: Normalizer

• ~Precompiled ⇐ Normalizer
  • new Precompiled(config)
  • .normalize(text) ⇒ string

new Precompiled(config)

Create a new instance of Precompiled normalizer.

precompiled.normalize(text) ⇒ <code> string </code>

Normalizes the given text by applying the precompiled charsmap.

Kind: instance method of Precompiled
Returns: string - The normalized text.

tokenizers~PreTokenizerSequence ⇐ <code> PreTokenizer </code>

A pre-tokenizer that applies a sequence of pre-tokenizers to the input text.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~PreTokenizerSequence ⇐ PreTokenizer
  • new PreTokenizerSequence(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new PreTokenizerSequence(config)

Creates an instance of PreTokenizerSequence.

preTokenizerSequence.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Applies each pre-tokenizer in the sequence to the input text in turn.

Kind: instance method of PreTokenizerSequence
Returns: Array.<string> - The pre-tokenized text.

tokenizers~WhitespacePreTokenizer

Splits on word boundaries (using the following regular expression: \w+|[^\w\s]+).

Kind: inner class of tokenizers

• ~WhitespacePreTokenizer
  • new WhitespacePreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new WhitespacePreTokenizer(config)

Creates an instance of WhitespacePreTokenizer.

whitespacePreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Pre-tokenizes the input text by splitting it on word boundaries.

Kind: instance method of WhitespacePreTokenizer
Returns: Array.<string> - An array of tokens produced by splitting the input text on whitespace.

tokenizers~WhitespaceSplit ⇐ <code> PreTokenizer </code>

Splits a string of text by whitespace characters into individual tokens.

Kind: inner class of tokenizers
Extends: PreTokenizer

• ~WhitespaceSplit ⇐ PreTokenizer
  • new WhitespaceSplit(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new WhitespaceSplit(config)

Creates an instance of WhitespaceSplit.

whitespaceSplit.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Pre-tokenizes the input text by splitting it on whitespace characters.

Kind: instance method of WhitespaceSplit
Returns: Array.<string> - An array of tokens produced by splitting the input text on whitespace.

tokenizers~ReplacePreTokenizer

Kind: inner class of tokenizers

• ~ReplacePreTokenizer
  • new ReplacePreTokenizer(config)
  • .pre_tokenize_text(text, [options]) ⇒ Array.<string>

new ReplacePreTokenizer(config)

replacePreTokenizer.pre_tokenize_text(text, [options]) ⇒ <code> Array. < string > </code>

Pre-tokenizes the input text by replacing certain characters.

Kind: instance method of ReplacePreTokenizer
Returns: Array.<string> - An array of tokens produced by replacing certain characters.

tokenizers~BYTES_TO_UNICODE ⇒ <code> Object </code>

Returns list of utf-8 byte and a mapping to unicode strings. Specifically avoids mapping to whitespace/control characters the BPE code barfs on.

Kind: inner constant of tokenizers
Returns: Object - Object with utf-8 byte keys and unicode string values.

tokenizers~loadTokenizer(pretrained_model_name_or_path, options) ⇒ <code> Promise. < Array < any > > </code>

Loads a tokenizer from the specified path.

Kind: inner method of tokenizers
Returns: Promise.<Array<any>> - A promise that resolves with information about the loaded tokenizer.

tokenizers~regexSplit(text, regex) ⇒ <code> Array. < string > </code>

Helper function to split a string on a regex, but keep the delimiters. This is required, because the JavaScript .split() method does not keep the delimiters, and wrapping in a capturing group causes issues with existing capturing groups (due to nesting).

Kind: inner method of tokenizers
Returns: Array.<string> - The split string.

tokenizers~createPattern(pattern, invert) ⇒ <code> RegExp </code> | <code> null </code>

Helper method to construct a pattern from a config object.

Kind: inner method of tokenizers
Returns: RegExp | null - The compiled pattern.

tokenizers~objectToMap(obj) ⇒ <code> Map. < string, any > </code>

Helper function to convert an Object to a Map

Kind: inner method of tokenizers
Returns: Map.<string, any> - The map.

tokenizers~prepareTensorForDecode(tensor) ⇒ <code> Array. < number > </code>

Helper function to convert a tensor to a list before decoding.

Kind: inner method of tokenizers
Returns: Array.<number> - The tensor as a list.

tokenizers~clean_up_tokenization(text) ⇒ <code> string </code>

Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms

Kind: inner method of tokenizers
Returns: string - The cleaned up text.

tokenizers~remove_accents(text) ⇒ <code> string </code>

Helper function to remove accents from a string.

Kind: inner method of tokenizers
Returns: string - The text with accents removed.

tokenizers~lowercase_and_remove_accent(text) ⇒ <code> string </code>

Helper function to lowercase a string and remove accents.

Kind: inner method of tokenizers
Returns: string - The lowercased text with accents removed.

tokenizers~whitespace_split(text) ⇒ <code> Array. < string > </code>

Split a string on whitespace.

Kind: inner method of tokenizers
Returns: Array.<string> - The split string.

tokenizers~PretrainedTokenizerOptions : <code> Object </code>

Additional tokenizer-specific properties.

Kind: inner typedef of tokenizers
Properties

tokenizers~BPENode : <code> Object </code>

Kind: inner typedef of tokenizers
Properties

tokenizers~SplitDelimiterBehavior : <code> ’ removed ’ </code> | <code> ’ isolated ’ </code> | <code> ’ mergedWithPrevious ’ </code> | <code> ’ mergedWithNext ’ </code> | <code> ’ contiguous ’ </code>

Kind: inner typedef of tokenizers

tokenizers~PostProcessedOutput : <code> Object </code>

Kind: inner typedef of tokenizers
Properties

tokenizers~EncodingSingle : <code> Object </code>

Kind: inner typedef of tokenizers
Properties

tokenizers~Message : <code> Object </code>

Kind: inner typedef of tokenizers
Properties

tokenizers~BatchEncoding : <code> Array < number > </code> | <code> Array < Array < number > > </code> | <code> Tensor </code>

Holds the output of the tokenizer’s call function.

Kind: inner typedef of tokenizers
Properties