<commit_before>import * as _ from "lodash";

export function load(it: Iterator<any>): any[] {
    let items = [], item;
    while((item = it.next().value) !== undefined) {
        items.push(item);
    }
    return items;
}

export function processTuples(tuples: Iterator<any>): any[] {
    let newTuples = [];
    let tuple;
    while((tuple = tuples.next().value) !== undefined) {
        newTuples.push(tuple);
    }
    //Sorting
    let keys = {}, sortingKeys = [];
    newTuples.forEach(tuple => {
        Object.keys(tuple).filter(key => { return key.split("_")[0] === "group"; }).forEach(key => {
            keys[key] = true;
        });
    });
    sortingKeys = Object.keys(keys);
    sortingKeys.sort((a, b) => {
        let v1 = parseInt(a.split("_")[3], 10);
        let v2 = parseInt(b.split("_")[3], 10);
        if(v1 > v2) {
            return 1;
        } else if(v1 === v2) {
            return 0;
        } else {
            return -1;
        }
    });
    newTuples = [newTuples];
    sortingKeys.reduce((done, key) => {
        let ascending = key.split("_")[1] === "true";
        if(done.length > 0) {
            newTuples = newTuples.map(tuples => {
                let result = [];
                let groups = _.groupBy(tuples, tuple => {
                    return done.map(key => {
                        return "" + tuple[key];
                    });
                });
                Object.keys(groups).forEach(key => {
                    result.push(groups[key]);
                });
                return result;
            });<commit_msg>:wrench:<commit_after>import * as _ from "lodash";

export function load(it: Iterator<any>): any[] {
    let items = [], item;
    while((item = it.next().value) !== undefined) {
        items.push(item);
    }
    return items;
}

export function processTuples(tuples: Iterator<any>): any[] {
    let newTuples = [];
    let tuple;
    while((tuple = tuples.next().value) !== undefined) {
        newTuples.push(tuple);
    }
    //Sorting
    let keys = {}, sortingKeys = [];
    newTuples.forEach(tuple => {
        Object.keys(tuple).filter(key => { return key.split("_")[0] === "group"; }).forEach(key => {
            keys[key] = true;
        });
    });
    sortingKeys = Object.keys(keys);
    sortingKeys.sort((a, b) => {
        let v1 = parseInt(a.split("_")[3], 10);
        let v2 = parseInt(b.split("_")[3], 10);
        if(v1 > v2) {
            return 1;
        } else if(v1 === v2) {
            return 0;
        } else {
            return -1;
        }
    });
    newTuples = [newTuples];
    sortingKeys.reduce((done, key) => {
        let ascending = key.split("_")[1] === "true";
        if(done.length > 0) {
            newTuples = newTuples.map(tuples => {
                let result = [];
                let groups = _.groupBy(tuples, tuple => {
                    return done.map(key => {
                        return "" + tuple[key];
                    });
                });
                Object.keys(groups).forEach(key => {
                    result.push(groups[key]);
                });
                return result;
            })[0];
{'fraction_non_alphanumeric': 0.17182564750473783, 'fraction_numerical': 0.011686670878079595, 'mean_word_length': 1.6065843621399176, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1320951', 'n_tokens_mistral': 1018, 'n_tokens_neox': 965, 'n_words': 187}