tokenizer

(s3prl.dataio.encoder.tokenizer)

Load tokenizer to encode & decode

Modified from tensorflow_datasets.features.text.* Reference: https://www.tensorflow.org/datasets/api_docs/python/tfds/features/text_lib

Authors:

• Heng-Jui Chang 2022

CharacterTokenizer

class s3prl.dataio.encoder.tokenizer.CharacterTokenizer(vocab_list: Optional[List[str]] = None)

Bases: Tokenizer

Character tokenizer.

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(vocab_file: Optional[str] = None, vocab_list: Optional[List[str]] = None)

property vocab_size: int

property token_type: str

vocab_to_idx(vocab)

idx_to_vocab(idx)

property eos_idx: int

property pad_idx: int

property unk_idx: int

CharacterSlotTokenizer

class s3prl.dataio.encoder.tokenizer.CharacterSlotTokenizer(vocab_list: List[str], slots: List[str])

Bases: Tokenizer

Character tokenizer with slots.

encode(sent: str, iobs: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(vocab_file: str, slots_file: str)

property vocab_size: int

property token_type: str

vocab_to_idx(vocab)

idx_to_vocab(idx)

property eos_idx: int

property pad_idx: int

property unk_idx: int

SubwordTokenizer

class s3prl.dataio.encoder.tokenizer.SubwordTokenizer(spm)

Bases: Tokenizer

Subword tokenizer using sentencepiece.

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(filepath: str)

property vocab_size: int

property token_type: str

property eos_idx: int

property pad_idx: int

property unk_idx: int

SubwordSlotTokenizer

class s3prl.dataio.encoder.tokenizer.SubwordSlotTokenizer(spm, slots)

Bases: Tokenizer

Subword tokenizer with slots.

encode(sent: str, iobs: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(filepath: str, slots_file: str)

property vocab_size: int

property token_type: str

property eos_idx: int

property pad_idx: int

property unk_idx: int

WordTokenizer

class s3prl.dataio.encoder.tokenizer.WordTokenizer(vocab_list: Optional[List[str]] = None)

Bases: CharacterTokenizer

Word tokenizer.

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

property token_type: str

property eos_idx: int

idx_to_vocab(idx)

classmethod load_from_file(vocab_file: Optional[str] = None, vocab_list: Optional[List[str]] = None)

property pad_idx: int

property unk_idx: int

property vocab_size: int

vocab_to_idx(vocab)

PhonemeTokenizer

class s3prl.dataio.encoder.tokenizer.PhonemeTokenizer(vocab_list: Optional[List[str]] = None)

Bases: WordTokenizer

Phoneme tokenizer.

property token_type: str

decode(idxs: List[int], ignore_repeat: bool = False) → str

encode(s: str) → List[int]

property eos_idx: int

idx_to_vocab(idx)

classmethod load_from_file(vocab_file: Optional[str] = None, vocab_list: Optional[List[str]] = None)

property pad_idx: int

property unk_idx: int

property vocab_size: int

vocab_to_idx(vocab)

load_tokenizer

s3prl.dataio.encoder.tokenizer.load_tokenizer(mode: str, vocab_file: Optional[str] = None, vocab_list: Optional[List[str]] = None, slots_file: Optional[str] = None) → Tokenizer

Load a text tokenizer.

Parameters:

• mode (str) – Mode (“character”, “character-slot”, “subword”, “subword-slot”, “word”, “bert-…”)

• vocab_file (str, optional) – Path to vocabularies. Defaults to None.

• vocab_list (List[str], optional) – List of vocabularies. Defaults to None.

• slots_file (str, optional) – Path to slots. Defaults to None.

Raises:

NotImplementedError – If mode is not implemented.

Returns:

Text tokenizer.

Return type:

Tokenizer

default_phoneme_tokenizer

s3prl.dataio.encoder.tokenizer.default_phoneme_tokenizer() → PhonemeTokenizer

Returns a default LibriSpeech phoneme tokenizer.

Returns:

Vocabs include 71 phonemes

Return type:

PhonemeTokenizer