encoder

(s3prl.dataio.encoder)

Encode the raw data into numeric format, and then decode it

s3prl.dataio.encoder.category	Simple categorical encoder
s3prl.dataio.encoder.g2p	Basic G2P
s3prl.dataio.encoder.tokenizer	Load tokenizer to encode & decode
s3prl.dataio.encoder.vocabulary	Create vocabulary (train tokenizer)

CategoryEncoder

class s3prl.dataio.encoder.CategoryEncoder(category: List[str])

Bases: object

encode(label: str) → int

decode(index: int) → str

CategoryEncoders

class s3prl.dataio.encoder.CategoryEncoders(categories: List[List[str]])

Bases: object

encode(labels: List[str]) → List[int]

decode(indices: List[int]) → List[str]

G2P

class s3prl.dataio.encoder.G2P(file_list: Optional[List[str]] = None, allow_unk: bool = False)

Bases: object

Grapheme-to-phoneme

Parameters:

• file_list (List[str], optional) – List of lexicon files. Defaults to None.

• allow_unk (bool) – If false, raise Error when a word can not be recognized by this basic G2P

encode(text: str) → str

Converts grapheme-based sentences to phonemes

Parameters:

text (str) – Sentence

Returns:

Phonemized sentence

Return type:

str

Tokenizer

class s3prl.dataio.encoder.Tokenizer

Bases: object

abstract encode(text: str, iob: Optional[str] = None) → List[int]

abstract decode(idxs: List[int], ignore_repeat: bool = False) → str

abstract property vocab_size: int

abstract property token_type: str

abstract classmethod load_from_file(vocab_file: str)

property pad_idx: int

property eos_idx: int

property unk_idx: int

BertTokenizer

class s3prl.dataio.encoder.BertTokenizer(tokenizer)

Bases: Tokenizer

Bert Tokenizer.

https://github.com/huggingface/pytorch-transformers/blob/master/pytorch_transformers/tokenization_bert.py

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(vocab_file: str)

property vocab_size: int

property token_type: str

property eos_idx: int

property pad_idx: int

property unk_idx: int

WordTokenizer

class s3prl.dataio.encoder.WordTokenizer(vocab_list: Optional[List[str]] = None)

Bases: CharacterTokenizer

Word tokenizer.

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

property token_type: str

property eos_idx: int

idx_to_vocab(idx)

classmethod load_from_file(vocab_file: Optional[str] = None, vocab_list: Optional[List[str]] = None)

property pad_idx: int

property unk_idx: int

property vocab_size: int

vocab_to_idx(vocab)

CharacterTokenizer

class s3prl.dataio.encoder.CharacterTokenizer(vocab_list: Optional[List[str]] = None)

Bases: Tokenizer

Character tokenizer.

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(vocab_file: Optional[str] = None, vocab_list: Optional[List[str]] = None)

property vocab_size: int

property token_type: str

vocab_to_idx(vocab)

idx_to_vocab(idx)

property eos_idx: int

property pad_idx: int

property unk_idx: int

CharacterSlotTokenizer

class s3prl.dataio.encoder.CharacterSlotTokenizer(vocab_list: List[str], slots: List[str])

Bases: Tokenizer

Character tokenizer with slots.

encode(sent: str, iobs: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(vocab_file: str, slots_file: str)

property vocab_size: int

property token_type: str

vocab_to_idx(vocab)

idx_to_vocab(idx)

property eos_idx: int

property pad_idx: int

property unk_idx: int

SubwordTokenizer

class s3prl.dataio.encoder.SubwordTokenizer(spm)

Bases: Tokenizer

Subword tokenizer using sentencepiece.

encode(s: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(filepath: str)

property vocab_size: int

property token_type: str

property eos_idx: int

property pad_idx: int

property unk_idx: int

SubwordSlotTokenizer

class s3prl.dataio.encoder.SubwordSlotTokenizer(spm, slots)

Bases: Tokenizer

Subword tokenizer with slots.

encode(sent: str, iobs: str) → List[int]

decode(idxs: List[int], ignore_repeat: bool = False) → str

classmethod load_from_file(filepath: str, slots_file: str)

property vocab_size: int

property token_type: str

property eos_idx: int

property pad_idx: int

property unk_idx: int

generate_basic_vocab

s3prl.dataio.encoder.generate_basic_vocab(mode: str, text_list: List[str], vocab_size: int = -1, coverage: float = 1.0, sort_vocab: bool = True) → List[str]

Generates basic vocabularies, including character and word-based vocabularies.

Parameters:

• mode (str) – Vocabulary type (character or word).

• text_list (List[str]) – List of text data.

• vocab_size (int, optional) – Vocabulary size, if not specified, vocab_size would be coverage * actual vocab size. Defaults to -1.

• coverage (float, optional) – Vocabulary coverage. Defaults to 1.0.

• sort_vocab (bool, optional) – Sort vocabularies alphabetically. Defaults to True.

Returns:

A list of vocabularies.

Return type:

List[str]

generate_subword_vocab

s3prl.dataio.encoder.generate_subword_vocab(text_list: Optional[List[str]] = None, text_file: Optional[str] = None, output_file: Optional[str] = None, vocab_size: int = 1000, character_coverage: float = 1.0) → str

Generates subword vocabularies based on sentencepiece.

Parameters:

• text_list (List[str], optional) – List of text data. Defaults to None.

• text_file (str, optional) – Path to text data. Defaults to None.

• output_file (str, optional) – Path to save trained subword vocabularies. Defaults to “”.

• vocab_size (int, optional) – Vocabulary size. Defaults to 8000.

• character_coverage (float, optional) – Coverage of characters in text data. Defaults to 1.0.

Raises:

ImportError – If sentencepiece is not installed.

Returns:

Path to ${output_file}.model.

Return type:

str

generate_vocab

s3prl.dataio.encoder.generate_vocab(mode: str, text_list: Optional[List[str]] = None, text_file: Optional[str] = None, read_lines: int = 10000000, **vocab_args) → Union[List[str], str]

Generates vocabularies given text data.

Parameters:

• mode (str) – Vocabulary type

• text_list (List[str], optional) – List of text data. Defaults to None.

• text_file (str, optional) – Path to text data. Defaults to None.

• read_lines (int, optional) – Maximum lines to read from text_file. Defaults to 10000000.

• vocab_args – if mode != subword, arguments for generate_basic_vocab if mode == subword, arguments for generate_subword_vocab

Returns:

A list of vocabularies or a path to .vocab file.

Return type:

Union[List[str], str]