<commit_before>extern crate num_traits;

use std::collections::HashSet;
use std::fmt;
use std::fmt::Debug;
use std::hash::Hash;
use std::vec::Vec;
use self::num_traits::One;
use std::str::FromStr;

use std::marker::PhantomData;

use automata;
use cfg;
use push_down::{PushDown, PushDownAutomaton, PushDownInstruction};

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)]
pub enum PushState<X, Y> {
    Designated,
    Initial,
    Nt(X),
    T(Y),
}

impl<X: fmt::Display, Y: fmt::Display> fmt::Display for PushState<X, Y> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &PushState::Designated
                => write!(f, "@"),
            &PushState::Initial
                => write!(f, "I"),
            &PushState::Nt(ref x)
                => write!(f, "({})", x),
            &PushState::T(ref x)
                => write!(f, "({})", x),
        }
    }
}

impl<N: Clone + Debug + Ord + PartialEq + Hash,
     T: Clone + Debug + Ord + PartialEq + Hash,
     W: Clone + Debug + Ord + PartialEq + One + FromStr
     > From<cfg::CFG<N, T, W>> for PushDownAutomaton<PushState<N,T>, T, W>
    where <W as FromStr>::Err: Debug{
     fn from(g: cfg::CFG<N, T, W>) -> Self {
        let mut transitions = Vec::new();

        let mut t_buffer= HashSet::new();

        ///creates a Transition for every rule that replaces the nonterminal of the left side with the rightside of the rule
        for r in g.rules.clone(){
            let mut st = Vec::new();
            for v in r.composition.composition{

                match v{
                    cfg::LetterT::Value(x) => {
                        t_buffer.insert(x.clone());
                        st.push(PushState::T(x.clone()));
                    },
                    cfg::LetterT::Label(x) => {
                        st.push(PushState::Nt(x.clone()));
                    },
                }
            }

            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: r.weight.clone(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Nt(r.head.clone()),
                        new_val: st.clone(),
                    }
                }
            );

        }
        ///creates a transition for every terminal that reads the word
        for t in t_buffer{
            let mut tvec = Vec::new();
            tvec.push(t.clone());
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: tvec.clone(),
                    weight: W::one(),
                    instruction: PushDownInstruction::Pop {
                        current_val: PushState::T(t.clone()),
                    }
                }
            );


        }
        ///creates a transition for the `Initial` symbol to all Nonterminals in `initial` with weight `1/initial.len()`
        let ini_val=1.0/g.initial.len() as f64;
        let ini_weight=ini_val.to_string();

        for ini in g.initial{
            let mut tvec = Vec::new();
            tvec.push(PushState::Nt(ini));
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: ini_weight.parse().unwrap(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Initial,
                        new_val: tvec,
                    }
                }
            );
        }

        PushDownAutomaton::new(
            transitions,<commit_msg>Small fix for initial weights<commit_after>extern crate num_traits;

use std::collections::HashSet;
use std::fmt;
use std::fmt::Debug;
use std::hash::Hash;
use std::vec::Vec;
use self::num_traits::One;
use std::str::FromStr;

use std::marker::PhantomData;

use automata;
use cfg;
use push_down::{PushDown, PushDownAutomaton, PushDownInstruction};

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)]
pub enum PushState<X, Y> {
    Designated,
    Initial,
    Nt(X),
    T(Y),
}

impl<X: fmt::Display, Y: fmt::Display> fmt::Display for PushState<X, Y> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &PushState::Designated
                => write!(f, "@"),
            &PushState::Initial
                => write!(f, "I"),
            &PushState::Nt(ref x)
                => write!(f, "({})", x),
            &PushState::T(ref x)
                => write!(f, "({})", x),
        }
    }
}

impl<N: Clone + Debug + Ord + PartialEq + Hash,
     T: Clone + Debug + Ord + PartialEq + Hash,
     W: Clone + Debug + Ord + PartialEq + One + FromStr
     > From<cfg::CFG<N, T, W>> for PushDownAutomaton<PushState<N,T>, T, W>
    where <W as FromStr>::Err: Debug{
     fn from(g: cfg::CFG<N, T, W>) -> Self {
        let mut transitions = Vec::new();

        let mut t_buffer= HashSet::new();

        ///creates a Transition for every rule that replaces the nonterminal of the left side with the rightside of the rule
        for r in g.rules.clone(){
            let mut st = Vec::new();
            for v in r.composition.composition{

                match v{
                    cfg::LetterT::Value(x) => {
                        t_buffer.insert(x.clone());
                        st.push(PushState::T(x.clone()));
                    },
                    cfg::LetterT::Label(x) => {
                        st.push(PushState::Nt(x.clone()));
                    },
                }
            }

            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: r.weight.clone(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Nt(r.head.clone()),
                        new_val: st.clone(),
                    }
                }
            );

        }
        ///creates a transition for every terminal that reads the word
        for t in t_buffer{
            let mut tvec = Vec::new();
            tvec.push(t.clone());
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: tvec.clone(),
                    weight: W::one(),
                    instruction: PushDownInstruction::Pop {
                        current_val: PushState::T(t.clone()),
                    }
                }
            );


        }
        ///creates a transition for the `Initial` symbol to all Nonterminals in `initial` with weight `one`

        for ini in g.initial{
            let mut tvec = Vec::new();
            tvec.push(PushState::Nt(ini));
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: W::one(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Initial,
                        new_val: tvec,
                    }
                }
            );
        }

        PushDownAutomaton::new(
            transitions,
{'fraction_non_alphanumeric': 0.12871287128712872, 'fraction_numerical': 0.0006781500067815, 'mean_word_length': 1.4433399602385686, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7118075', 'n_tokens_mistral': 2132, 'n_tokens_neox': 1999, 'n_words': 489}