<commit_before>//! A machine learning library for supervised regression trainings
//!
//! This library wants to enable its user to write training algorithms
//! independent of the model trained or the cost function tried to
//! minimize.
//! Consequently its two main traits are currently `Model` and `Cost`.
//! The two submodules `model` and `cost` provide ready to use
//! implementations of said traits.

#![warn(missing_docs)]

extern crate num;

use std::iter::Iterator;
use num::{Zero, One, Float};

/// A Model is defines how to predict a target from an input
///
/// A model usually depends on several coefficents whose values
/// are derived using a training algorithm 
pub trait Model : Clone{
    /// Input features
    type Input;
    /// Target type
    type Target : Float;

    /// Predicts a target for the inputs based on the internal coefficents
    fn predict(&self, &Self::Input) -> Self::Target;

    /// The number of internal coefficents this model depends on
    fn num_coefficents(&self) -> usize;

    /// Value predict derived by the n-th `coefficent` at `input`
    fn gradient(&self, coefficent : usize, input : &Self::Input) -> Self::Target;

    /// Mutable reference to the n-th `coefficent`
    fn coefficent(& mut self, coefficent : usize) -> & mut Self::Target;
}

/// Cost functions those value is supposed be minimized by the training algorithm
pub trait Cost{

    /// Error type used by the cost function
    ///
    /// Usually `f64` or `f32`
    type Error : Float;

    /// Value of the cost function derived by the n-th coefficent at x expressed in Error(x) and dY(x)/dx
    ///
    /// This method is called by SGD based training algorithm in order to
    /// determine the delta of the coefficents
    fn gradient(&self, error : Self::Error, gradient_error_by_coefficent : Self::Error) -> Self::Error;
}

/// Changes all coefficents of model based on their derivation of the cost function at features
///
/// Can be used to implement stochastic or batch gradient descent
pub fn inert_gradient_descent_step<C, M>(
    cost : &C,
    model : &mut M,
    features : &M::Input,
    truth : M::Target,
    learning_rate : M::Target,
    inertia : M::Target,
    velocity : & mut Vec<M::Target>
)
    where C : Cost, M : Model<Target=C::Error>
{
    let inv_inertia = M::Target::one() - inertia;
    let prediction = model.predict(&features);
    let error = prediction - truth;

    for ci in 0..model.num_coefficents(){

        velocity[ci] = inertia * velocity[ci] - inv_inertia * learning_rate * cost.gradient(error, model.gradient(ci, features));
        *model.coefficent(ci) = *model.coefficent(ci) + velocity[ci];
    }
}

/// An SGD training step with a velocity term
///
/// Will not get stuck on saddle points as easily as a plain SGD and will converge quicker in general.
/// A good default for `inertia` is 0.9
pub fn gradient_descent_step<C, M>(cost : &C, model : &mut M, features : &M::Input, truth : M::Target, learning_rate : M::Target)
    where C : Cost, M : Model<Target=C::Error>
{
    let prediction = model.predict(&features);<commit_msg>fix: documentation of sgd and inert_sgd switched<commit_after>//! A machine learning library for supervised regression trainings
//!
//! This library wants to enable its user to write training algorithms
//! independent of the model trained or the cost function tried to
//! minimize.
//! Consequently its two main traits are currently `Model` and `Cost`.
//! The two submodules `model` and `cost` provide ready to use
//! implementations of said traits.

#![warn(missing_docs)]

extern crate num;

use std::iter::Iterator;
use num::{Zero, One, Float};

/// A Model is defines how to predict a target from an input
///
/// A model usually depends on several coefficents whose values
/// are derived using a training algorithm 
pub trait Model : Clone{
    /// Input features
    type Input;
    /// Target type
    type Target : Float;

    /// Predicts a target for the inputs based on the internal coefficents
    fn predict(&self, &Self::Input) -> Self::Target;

    /// The number of internal coefficents this model depends on
    fn num_coefficents(&self) -> usize;

    /// Value predict derived by the n-th `coefficent` at `input`
    fn gradient(&self, coefficent : usize, input : &Self::Input) -> Self::Target;

    /// Mutable reference to the n-th `coefficent`
    fn coefficent(& mut self, coefficent : usize) -> & mut Self::Target;
}

/// Cost functions those value is supposed be minimized by the training algorithm
pub trait Cost{

    /// Error type used by the cost function
    ///
    /// Usually `f64` or `f32`
    type Error : Float;

    /// Value of the cost function derived by the n-th coefficent at x expressed in Error(x) and dY(x)/dx
    ///
    /// This method is called by SGD based training algorithm in order to
    /// determine the delta of the coefficents
    fn gradient(&self, error : Self::Error, gradient_error_by_coefficent : Self::Error) -> Self::Error;
}

/// Changes all coefficents of model based on their derivation of the cost function at features
///
/// Will not get stuck on saddle points as easily as a plain SGD and will converge quicker in general.
/// A good default for `inertia` is 0.9
pub fn inert_gradient_descent_step<C, M>(
    cost : &C,
    model : &mut M,
    features : &M::Input,
    truth : M::Target,
    learning_rate : M::Target,
    inertia : M::Target,
    velocity : & mut Vec<M::Target>
)
    where C : Cost, M : Model<Target=C::Error>
{
    let inv_inertia = M::Target::one() - inertia;
    let prediction = model.predict(&features);
    let error = prediction - truth;

    for ci in 0..model.num_coefficents(){

        velocity[ci] = inertia * velocity[ci] - inv_inertia * learning_rate * cost.gradient(error, model.gradient(ci, features));
        *model.coefficent(ci) = *model.coefficent(ci) + velocity[ci];
    }
}

/// An SGD training step with a velocity term
///
/// Can be used to implement stochastic or batch gradient descent
pub fn gradient_descent_step<C, M>(cost : &C, model : &mut M, features : &M::Input, truth : M::Target, learning_rate : M::Target)
    where C : Cost, M : Model<Target=C::Error>
{
    let prediction = model.predict(&features);
{'fraction_non_alphanumeric': 0.11620451995512102, 'fraction_numerical': 0.002243949350857509, 'mean_word_length': 3.190732034922767, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1136834', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1771, 'n_words': 737}