<commit_before>/* eslint no-use-before-define: 0 */

/*
    @license
    Copyright (C) 2016 <NAME>
    License: MIT
    See license.txt for full license text.
*/
"use strict";

import util, { extend, types } from 'zana-util';
import check from 'zana-check';

// const DATA = Symbol('data'); // swap this.data to use DATA instead

function buildMapArray(count) {
    let mapArray = new Array(count);
    for (let i = 0; i < count; i++)
        mapArray[i] = i;
    return mapArray;
}

function buildKeyArray(elements, selector, count) {
    let keyArray = new Array(count);
    for (let i = 0; i < count; i++)
        keyArray[i] = selector(elements[i]);
    return keyArray;
}

function compareKeys(comparer, keys, i1, i2) {
    let k1 = keys[i1];
    let k2 = keys[i2];
    let c = comparer(k1, k2);
    if (c === 0)
        return i1 - i2;
    return c;
}

function expand(iter) {
    if (iter && typeof iter === 'function') // could be mishandled. throw an error if iter() doesn't have [Symbol.iterator] defined?
        return iter();
    return iter;
}

function quicksort(keys, map, comparer, left, right) {
    do {
        let i = left;
        let j = right;
        let x = map[i + ((j - i) >> 1)];
        do {
            while (i < map.length && compareKeys(comparer, keys, x, map[i]) > 0)
                i++;
            while (j >= 0 && compareKeys(comparer, keys, x, map[j]) < 0)
                j--;
            if (i > j)
                break; // left index has crossed right index, stop the loop
            if (i < j)
                [map[i], map[j]] = [map[j], map[i]];
            i++;
            j--;
        } while (i <= j);
        if (j - left <= right - i) {
            if (left < j)
                quicksort(keys, map, comparer, left, j);
            left = i;
        }
        else {
            if (i < right)
                quicksort(keys, map, comparer, i, right);
            right = j;
        }
    } while (left < right);
}

function* reverse(iter, a) {
    if (!a.done) {
        yield* reverse(iter, iter.next());
        yield a.value;
    }
}

// lots of expanding here, may need some performance improvements
// consider detecting generatorFunctions with check.isIterable (somehow. es6 spec yet?)
function _flatten(item) {
    return function*() {
        let prev = expand(item);
        if (!check.isIterable(prev))
            yield prev;
        else {
            for (let v of prev) {
                if (!check.isIterable(v) && !check.instance(v, Function))
                    yield v;
                else
                    yield* expand(_flatten(v));
            }
        }
    };
}

export default class Iterable {
    constructor(data) {
        this.data = data || [];
    }

    static empty() {
        return new Iterable([]);
    }

    static iter(...args) {
        if (args.length === 1)
            return new Iterable(args[0]);
        else if (args.length > 1)
            return new MultiIterable(...args);
        else
            return Iterable.empty();
    }

    get [Symbol.toStringTag]() {
        return 'Iterable';
    }

    [Symbol.iterator]() {
        return expand(this.data)[Symbol.iterator](); // covers arrays, sets, generator functions, generators..
    }

    aggregate(
          func = x => x
        , seed = null
    ) {
        let iter = this[Symbol.iterator]();
        let result = null;
        if (seed === null)
            result = iter.next().value; // what about empty iterables?
        else
            result = func(seed, iter.next().value);
        for (let v of iter)
            result = func(result, v);
        return result;
    }

    any(predicate = check.exists) {
        for (let v of this) {
            if (predicate(v))
                return true;
        }
        return false;
    }

    at(index) {
        if (check.instance(this.data, Array))
            return this.data[index];
        for (let v of this) {
            if (index-- === 0)
                return v;
        }
    }

    average(selector = x => x) {
        let len = this.length();
        if (len === 0)
            return len;
        return this.sum(selector) / len;
    }

    concat(...args) {
        let iters = [this.data];
        for (let arg of args) {
            if (arg instanceof Iterable) // a bit hacky... but functional. need to make sure we can use iter1.concat(iter1)<commit_msg>refactor: return NaN from average of empty iterable instead of 0<commit_after>/* eslint no-use-before-define: 0 */

/*
    @license
    Copyright (C) 2016 <NAME>
    License: MIT
    See license.txt for full license text.
*/
"use strict";

import util, { extend, types } from 'zana-util';
import check from 'zana-check';

// const DATA = Symbol('data'); // swap this.data to use DATA instead

function buildMapArray(count) {
    let mapArray = new Array(count);
    for (let i = 0; i < count; i++)
        mapArray[i] = i;
    return mapArray;
}

function buildKeyArray(elements, selector, count) {
    let keyArray = new Array(count);
    for (let i = 0; i < count; i++)
        keyArray[i] = selector(elements[i]);
    return keyArray;
}

function compareKeys(comparer, keys, i1, i2) {
    let k1 = keys[i1];
    let k2 = keys[i2];
    let c = comparer(k1, k2);
    if (c === 0)
        return i1 - i2;
    return c;
}

function expand(iter) {
    if (iter && typeof iter === 'function') // could be mishandled. throw an error if iter() doesn't have [Symbol.iterator] defined?
        return iter();
    return iter;
}

function quicksort(keys, map, comparer, left, right) {
    do {
        let i = left;
        let j = right;
        let x = map[i + ((j - i) >> 1)];
        do {
            while (i < map.length && compareKeys(comparer, keys, x, map[i]) > 0)
                i++;
            while (j >= 0 && compareKeys(comparer, keys, x, map[j]) < 0)
                j--;
            if (i > j)
                break; // left index has crossed right index, stop the loop
            if (i < j)
                [map[i], map[j]] = [map[j], map[i]];
            i++;
            j--;
        } while (i <= j);
        if (j - left <= right - i) {
            if (left < j)
                quicksort(keys, map, comparer, left, j);
            left = i;
        }
        else {
            if (i < right)
                quicksort(keys, map, comparer, i, right);
            right = j;
        }
    } while (left < right);
}

function* reverse(iter, a) {
    if (!a.done) {
        yield* reverse(iter, iter.next());
        yield a.value;
    }
}

// lots of expanding here, may need some performance improvements
// consider detecting generatorFunctions with check.isIterable (somehow. es6 spec yet?)
function _flatten(item) {
    return function*() {
        let prev = expand(item);
        if (!check.isIterable(prev))
            yield prev;
        else {
            for (let v of prev) {
                if (!check.isIterable(v) && !check.instance(v, Function))
                    yield v;
                else
                    yield* expand(_flatten(v));
            }
        }
    };
}

export default class Iterable {
    constructor(data) {
        this.data = data || [];
    }

    static empty() {
        return new Iterable([]);
    }

    static iter(...args) {
        if (args.length === 1)
            return new Iterable(args[0]);
        else if (args.length > 1)
            return new MultiIterable(...args);
        else
            return Iterable.empty();
    }

    get [Symbol.toStringTag]() {
        return 'Iterable';
    }

    [Symbol.iterator]() {
        return expand(this.data)[Symbol.iterator](); // covers arrays, sets, generator functions, generators..
    }

    aggregate(
          func = x => x
        , seed = null
    ) {
        let iter = this[Symbol.iterator]();
        let result = null;
        if (seed === null)
            result = iter.next().value; // what about empty iterables?
        else
            result = func(seed, iter.next().value);
        for (let v of iter)
            result = func(result, v);
        return result;
    }

    any(predicate = check.exists) {
        for (let v of this) {
            if (predicate(v))
                return true;
        }
        return false;
    }

    at(index) {
        if (check.instance(this.data, Array))
            return this.data[index];
        for (let v of this) {
            if (index-- === 0)
                return v;
        }
    }

    average(selector = x => x) {
        let len = this.length();
        if (len === 0)
            return NaN;
        return this.sum(selector) / len;
    }

    concat(...args) {
        let iters = [this.data];
        for (let arg of args) {
            if (arg instanceof Iterable) // a bit hacky... but functional. need to make sure we can use iter1.concat(iter1)
{'fraction_non_alphanumeric': 0.13433510337814936, 'fraction_numerical': 0.00689187662411027, 'mean_word_length': 1.6306092124814264, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4576851', 'n_tokens_mistral': 2737, 'n_tokens_neox': 2652, 'n_words': 835}