<commit_before>namespace util{

    void
    validate_probability_vector(const std::vector<float> &p){
        float sum = 0;
        for(auto x: p){
            assert(isfinite(x));
            assert(x >= 0);
            sum+=x;
        }
        assert(std::abs(1 - sum) < 0.01);
    }

    template<typename T> void
    removeFirst(std::vector<T> &v, T element){
        auto it = std::find(v.begin(),v.end(), element);
        if (it != v.end()) {
          v.erase(it);
        }
    }

    // http://stackoverflow.com/a/1267878/982745
    template< class T >
    std::vector<T>
    selectByIndex(const std::vector<T> &v, const std::vector<size_t> &index )  {
        std::vector<T> new_v;
        new_v.reserve(index.size());
        for(size_t i: index){
            new_v.push_back(v[i]);
        }

        return new_v;
    }


    template<class T>
    void
    normalize(std::vector<T> &v){
        for(auto x: v) {
            assert(isfinite(x));
        }
        Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1>> vec(v.data(), v.size());
        vec /= vec.sum();
    }
}<commit_msg>Add defaultdict class<commit_after>namespace util{

    void
    validate_probability_vector(const std::vector<float> &p){
        float sum = 0;
        for(auto x: p){
            assert(isfinite(x));
            assert(x >= 0);
            sum+=x;
        }
        assert(std::abs(1 - sum) < 0.01);
    }

    template<typename T> void
    removeFirst(std::vector<T> &v, T element){
        auto it = std::find(v.begin(),v.end(), element);
        if (it != v.end()) {
          v.erase(it);
        }
    }

    // http://stackoverflow.com/a/1267878/982745
    template< class T >
    std::vector<T>
    selectByIndex(const std::vector<T> &v, const std::vector<size_t> &index )  {
        std::vector<T> new_v;
        new_v.reserve(index.size());
        for(size_t i: index){
            new_v.push_back(v[i]);
        }

        return new_v;
    }


    template<class T>
    void
    normalize(std::vector<T> &v){
        for(auto x: v) {
            assert(isfinite(x));
        }
        Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1>> vec(v.data(), v.size());
        vec /= vec.sum();
    }

    template<class T, class J>
    class defaultdict{
        J default_value;
        std::map<T, J> map;
    public:
        defaultdict(J val){
            default_value = val;
            map = std::map<T, J>();
        }

        J get(T t) {
            if(map.count(t) > 0){
                return map[t];
            }
            else{
                return default_value;
            }
        }

        void
        set(T t, J j){
            map[t] = j;
        }

        void
        incr(T t, J by){
            if(map.count(t) > 0){
                map[t] += by;
            }
            else{
                map[t] = by + default_value;
            }
        }

        void
        decr(T t, J by){
            if(map.count(t) > 0){
                map[t] -= by;
            }
            else{
                map[t] = default_value - by;
            }
        }
    };
}
{'fraction_non_alphanumeric': 0.15429309231762428, 'fraction_numerical': 0.013879922530664945, 'mean_word_length': 1.447867298578199, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4018333', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1037, 'n_words': 204}