utils.py

import argparse

class OsSoluConfig:
    d_model: int                                # Hidden size of the model.
    vocab_size: int                             # Vocabulary size of the input sequence. Unsure about this.
    learning_rate: float                        # Learning rate for the optimiser.
    num_embeddings: int                         # Number of embeddings. Unsure about this.
    num_blocks: int                             # Number of transformer blocks.
    dropout: float                              # Probability of dropout.
    ln_eps: float                               # Layer norm epsilon.
    num_heads: int                              # Number of attention heads in each attention layer.
    self_attention_type: str                    # What type of attention to use: rotary or unidirectional. 
    max_positional_embeddings: int              # Maximum number of positional embeddings.
    
    def __init__(self, args: argparse.Namespace) -> None:
        """Initialise this config class with values provided by a command-line argument parser.
           Values are never None here, as we provide suitable defaults in the parser call."""
        self.d_model = args.d_model
        self.vocab_size = args.vocab_size
        self.learning_rate = args.learning_rate
        self.num_embeddings = args.num_embeddings
        self.num_blocks = args.num_blocks
        self.dropout = args.dropout
        self.ln_eps = args.ln_eps
        self.num_heads = args.num_heads
        self.self_attention_type = args.self_attention_type
        self.max_positional_embeddings = args.max_positional_embeddings