<commit_before>//#include <QCoreApplication>
#include <map>
#include <list>
#include <vector>
#include <iostream>
#include <set>
#include <fstream>
#include <string>
#include <sstream>
struct Object {
    std::string word;
    bool operator <(const Object &b) const
    {
        return this->word < b.word;
    }
};

Object next_char(const std::vector<Object>& list) {
    std::map<Object, std::pair<int,int> > map;
    for (int i = 0; i < list.size(); ++i) {
        Object ch = list.at(i);
        std::map<Object, std::pair<int,int> >::iterator it = map.find(ch);
        if (it != map.end()) {
            it->second.first++;
            it->second.second = i;
        } else {
            map.insert(std::make_pair(ch, std::make_pair(1,i)));
        }
    }

    std::map <Object,int> weightMap;
    std::map<Object, std::pair<int,int> >::iterator mit = map.begin();
    std::map<Object, std::pair<int,int> >::iterator mend = map.end();
    for (; mit != mend; ++mit) {
        int value = 2 * (mit->second.first) + mit->second.second;
        weightMap.insert(std::make_pair(mit->first, value));<commit_msg>add stdlib.h<commit_after>//#include <QCoreApplication>
#include <map>
#include <list>
#include <vector>
#include <iostream>
#include <set>
#include <fstream>
#include <string>
#include <sstream>
#include <stdio.h>
#include <stdlib.h>
struct Object {
    std::string word;
    bool operator <(const Object &b) const
    {
        return this->word < b.word;
    }
};

Object next_char(const std::vector<Object>& list) {
    std::map<Object, std::pair<int,int> > map;
    for (int i = 0; i < list.size(); ++i) {
        Object ch = list.at(i);
        std::map<Object, std::pair<int,int> >::iterator it = map.find(ch);
        if (it != map.end()) {
            it->second.first++;
            it->second.second = i;
        } else {
            map.insert(std::make_pair(ch, std::make_pair(1,i)));
        }
    }

    std::map <Object,int> weightMap;
    std::map<Object, std::pair<int,int> >::iterator mit = map.begin();
    std::map<Object, std::pair<int,int> >::iterator mend = map.end();
    for (; mit != mend; ++mit) {
        int value = 2 * (mit->second.first) + mit->second.second;
        weightMap.insert(std::make_pair(mit->first, value));
{'fraction_non_alphanumeric': 0.19020300088261252, 'fraction_numerical': 0.00264783759929391, 'mean_word_length': 2.998236331569665, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2449718', 'n_tokens_mistral': 797, 'n_tokens_neox': 780, 'n_words': 176}