|
|
from transformers import PretrainedConfig
|
|
|
from dataclasses import dataclass, asdict
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
class QuantNoiseConfig:
|
|
|
pq: float = 0.0
|
|
|
pq_block_size: int = 8
|
|
|
|
|
|
def to_dict(self):
|
|
|
return asdict(self)
|
|
|
|
|
|
@classmethod
|
|
|
def from_dict(cls, data):
|
|
|
return cls(**data)
|
|
|
|
|
|
|
|
|
repo_name = "damilojohn/AfroLid"
|
|
|
|
|
|
|
|
|
class AfroLidConfig(PretrainedConfig):
|
|
|
model_type = "afrolid"
|
|
|
|
|
|
def __init__(self,
|
|
|
encoder_vocab_size=64001,
|
|
|
decoder_vocab_size=528,
|
|
|
embed_dim=768,
|
|
|
ffn_dim=3072,
|
|
|
num_heads=12,
|
|
|
num_layers=12,
|
|
|
max_seq_len=512,
|
|
|
dropout=0.1,
|
|
|
attention_dropout=0.1,
|
|
|
activation_dropout=0.0,
|
|
|
layerdrop=0.0,
|
|
|
normalize_before=False,
|
|
|
learned_pos=False,
|
|
|
max_source_positions=1024,
|
|
|
max_target_positions=1024,
|
|
|
no_token_positional_embeddings=False,
|
|
|
share_decoder_input_output_embed=True,
|
|
|
share_all_embeddings=False,
|
|
|
layernorm_embedding=False,
|
|
|
checkpoint_activations=False,
|
|
|
offload_activations=False,
|
|
|
bias=False,
|
|
|
**kwargs):
|
|
|
"""
|
|
|
AfroLid configuration class for an encoder-decoder transformer model,
|
|
|
with support for QuantNoiseConfig.
|
|
|
"""
|
|
|
self.encoder_vocab_size = encoder_vocab_size
|
|
|
self.decoder_vocab_size = decoder_vocab_size
|
|
|
self.embed_dim = embed_dim
|
|
|
self.ffn_dim = ffn_dim
|
|
|
self.num_heads = num_heads
|
|
|
self.num_layers = num_layers
|
|
|
self.max_seq_len = max_seq_len
|
|
|
self.dropout = dropout
|
|
|
self.attention_dropout = attention_dropout
|
|
|
self.activation_dropout = activation_dropout
|
|
|
self.layerdrop = layerdrop
|
|
|
self.normalize_before = normalize_before
|
|
|
self.learned_pos = learned_pos
|
|
|
self.max_source_positions = max_source_positions
|
|
|
self.max_target_positions = max_target_positions
|
|
|
self.no_token_positional_embeddings = no_token_positional_embeddings
|
|
|
self.share_decoder_input_output_embed = share_decoder_input_output_embed
|
|
|
self.share_all_embeddings = share_all_embeddings
|
|
|
self.layernorm_embedding = layernorm_embedding
|
|
|
self.checkpoint_activations = checkpoint_activations
|
|
|
self.offload_activations = offload_activations
|
|
|
self.bias = bias
|
|
|
|
|
|
super().__init__(**kwargs)
|
|
|
|
