ferritin_plms::esm::tokenization::sequence_tokenizer

Trait EsmTokenizerBase

Source 
pub trait EsmTokenizerBase {

Show 14 methods    // Required methods
    fn encode(&self) -> Result<()>;
    fn decode(&self) -> Result<()>;
    fn mask_token(&self) -> &str;
    fn mask_token_id(&self) -> u32;
    fn bos_token(&self) -> &str;
    fn bos_token_id(&self) -> u32;
    fn eos_token(&self) -> &str;
    fn eos_token_id(&self) -> u32;
    fn pad_token(&self) -> &str;
    fn pad_token_id(&self) -> u32;
    fn chain_break_token(&self) -> &str;
    fn chain_break_token_id(&self) -> u32;
    fn all_token_ids(&self) -> Vec<u32>;
    fn special_token_ids(&self) -> Vec<u32>;

}

Required Methods§

Source

fn encode(&self) -> Result<()>

Source

fn decode(&self) -> Result<()>

Source

fn mask_token(&self) -> &str

Source

fn mask_token_id(&self) -> u32

Source

fn bos_token(&self) -> &str

Source

fn bos_token_id(&self) -> u32

Source

fn eos_token(&self) -> &str

Source

fn eos_token_id(&self) -> u32

Source

fn pad_token(&self) -> &str

Source

fn pad_token_id(&self) -> u32

Source

fn chain_break_token(&self) -> &str

Source

fn chain_break_token_id(&self) -> u32

Source

fn all_token_ids(&self) -> Vec<u32>

Source

fn special_token_ids(&self) -> Vec<u32>

Implementors§