ferritin_plms/esm3/tokenization/mod.rs
1//! ESM3 multi-track tokenization.
2//!
3//! | Track | Vocab | Layout |
4//! |-----------|--------|---------------------------------------------|
5//! | sequence | 33 | `<cls>=0 <pad>=1 <eos>=2 ... <mask>=32` |
6//! | structure | 4101 | VQ-VAE 0..4095 + MASK/EOS/BOS/PAD/CB |
7//! | ss8 | 11 | PAD=0 MASK=1 UNK=2 GHITEBSC=3..10 |
8//! | sasa | 19 | PAD=0 MASK=1 UNK=2 bins=3..18 |
9//!
10//! Function/InterPro tokenization requires external data files (future work).
11
12pub mod sasa;
13pub mod sequence;
14pub mod ss8;
15pub mod structure;
16
17pub use sasa::tokenize_sasa;
18pub use sequence::{decode_sequence, tokenize_sequence};
19pub use ss8::tokenize_ss8;
20pub use structure::tokenize_structure;