Struct EsmSequenceTokenizer 

pub struct EsmSequenceTokenizer { /* private fields */ }

Implementations

impl EsmSequenceTokenizer

pub fn new( unk_token: &str, cls_token: &str, pad_token: &str, mask_token: &str, eos_token: &str, chain_break_token: &str, ) -> Result<Self>

impl EsmSequenceTokenizer

pub fn tokenize_sequence( &self, sequence: &str, add_special_tokens: bool, ) -> Vec<u32>

Tokenize an amino-acid sequence string into token IDs.

Looks up each character in SEQUENCE_VOCAB. Unknown characters map to the <unk> token (index 3). When add_special_tokens is true, prepends BOS (<cls> = 0) and appends EOS (<eos> = 2).

pub fn decode_sequence(&self, token_ids: &[u32]) -> String

Decode token IDs back to an amino-acid sequence string.

Skips the standard special tokens (BOS=0, PAD=1, EOS=2, MASK=32) and concatenates the remaining vocabulary entries.

Trait Implementations

impl Default for EsmSequenceTokenizer

fn default() -> Self

Returns the “default value” for a type.

impl EsmTokenizerBase for EsmSequenceTokenizer

fn encode(&self) -> Result<()>

fn decode(&self) -> Result<()>

fn mask_token(&self) -> &str

fn mask_token_id(&self) -> u32

fn bos_token(&self) -> &str

fn bos_token_id(&self) -> u32

fn eos_token(&self) -> &str

fn eos_token_id(&self) -> u32

fn pad_token(&self) -> &str

fn pad_token_id(&self) -> u32

fn chain_break_token(&self) -> &str

fn chain_break_token_id(&self) -> u32

fn all_token_ids(&self) -> Vec<u32>

fn special_token_ids(&self) -> Vec<u32>