<commit_before>'use strict';

const Promise      = require('bluebird');
const _            = require('lodash');
const utils        = require('./utils');

module.exports = class Collection {

    constructor(props, items) {
        this.type      = 'collection';
        this._items    = new Set(items || []);
    }

    items() {
        return Array.from(this._items);
    }

    setItems(items) {
        this._items = new Set(items || []);
        return this;
    }

    get size() {
        return this._items.size;
    }

    toJSON() {
        return {
            type: this.type,
            items: this.items().map(i => i.toJSON())
        };
    }

    find(str) {
        if (this.size === 0) {
            return undefined;
        }
        for (let item of this) {
            if (item.type === 'collection') {
                const search = item.find(str);
                if (search) return search;
            } else if (item.handle === str || `@${item.handle}` === str) {
                return item;
            }
        }
    }

    findCollection(str) {
        const type = str.startsWith('@') ? 'handle' : (str.includes('/') ? 'path' : 'handle');
        if (this.size === 0) {
            return undefined;
        }
        for (let item of this) {
            if (item.type === 'collection') {
                if (item[type] === str || `@${item[type]}` === str)  {
                    return item;
                }
                const search = item.findCollection(str);
                if (search) return search;
            }
        }
    }

    flatten(deep) {
        return this.newSelf(this.flattenItems(this.items(), deep));
    }

    squash(){
        return this.newSelf(this.squashItems(this.items()));
    }

    filter(predicate){
        return this.newSelf(this.filterItems(this.items(), predicate));
    }

    filterItems(items, predicate) {
        let matcher = _.iteratee(predicate);
        let ret = [];
        for (let item of items) {
            if (item.type === 'collection') {
                let collection = item.filter(predicate);
                if (collection.size) {
                    ret.push(collection);
                }
            } else {
                if (matcher(item)) {
                    ret.push(item);
                }
            }
        }
        return ret;
    }

    flattenItems(items, deep) {
        let ret = [];
        for (let item of items) {
            if (item.type === 'collection') {
                ret = _.concat(ret, this.flattenItems(item.items(), deep));
            } else {
                if (deep && _.isFunction(item.flatten)) {
                    ret = _.concat(ret, item.flatten());
                } else {
                    ret.push(item);
                }
            }
        }
        return ret;
    }

    squashItems(items) {
        const squashed = [];
        function squash(items){
            items = _.sortBy(items, function(i){
                return i.type === 'collection' ? 1 : 0;
            });
            for (let item of items) {
                if (item.type === 'collection') {
                    const children = item.items();
                    const entities = children.filter(c => c.type !== 'collection');
                    const collections = children.filter(c => c.type == 'collection');
                    if (entities.length) {
                        squashed.push(item.newSelf(entities));
                    } else {
                        squash(collections);<commit_msg>Fix issue with nested collections not appearing when using squash()<commit_after>'use strict';

const Promise      = require('bluebird');
const _            = require('lodash');
const utils        = require('./utils');

module.exports = class Collection {

    constructor(props, items) {
        this.type      = 'collection';
        this._items    = new Set(items || []);
    }

    items() {
        return Array.from(this._items);
    }

    setItems(items) {
        this._items = new Set(items || []);
        return this;
    }

    get size() {
        return this._items.size;
    }

    toJSON() {
        return {
            type: this.type,
            items: this.items().map(i => i.toJSON())
        };
    }

    find(str) {
        if (this.size === 0) {
            return undefined;
        }
        for (let item of this) {
            if (item.type === 'collection') {
                const search = item.find(str);
                if (search) return search;
            } else if (item.handle === str || `@${item.handle}` === str) {
                return item;
            }
        }
    }

    findCollection(str) {
        const type = str.startsWith('@') ? 'handle' : (str.includes('/') ? 'path' : 'handle');
        if (this.size === 0) {
            return undefined;
        }
        for (let item of this) {
            if (item.type === 'collection') {
                if (item[type] === str || `@${item[type]}` === str)  {
                    return item;
                }
                const search = item.findCollection(str);
                if (search) return search;
            }
        }
    }

    flatten(deep) {
        return this.newSelf(this.flattenItems(this.items(), deep));
    }

    squash(){
        return this.newSelf(this.squashItems(this.items()));
    }

    filter(predicate){
        return this.newSelf(this.filterItems(this.items(), predicate));
    }

    filterItems(items, predicate) {
        let matcher = _.iteratee(predicate);
        let ret = [];
        for (let item of items) {
            if (item.type === 'collection') {
                let collection = item.filter(predicate);
                if (collection.size) {
                    ret.push(collection);
                }
            } else {
                if (matcher(item)) {
                    ret.push(item);
                }
            }
        }
        return ret;
    }

    flattenItems(items, deep) {
        let ret = [];
        for (let item of items) {
            if (item.type === 'collection') {
                ret = _.concat(ret, this.flattenItems(item.items(), deep));
            } else {
                if (deep && _.isFunction(item.flatten)) {
                    ret = _.concat(ret, item.flatten());
                } else {
                    ret.push(item);
                }
            }
        }
        return ret;
    }

    squashItems(items) {
        const squashed = [];
        function squash(items){
            items = _.sortBy(items, function(i){
                return i.type === 'collection' ? 1 : 0;
            });
            for (let item of items) {
                if (item.type === 'collection') {
                    const children = item.items();
                    const entities = children.filter(c => c.type !== 'collection');
                    const collections = children.filter(c => c.type == 'collection');
                    if (entities.length) {
                        squashed.push(item.newSelf(entities));
                    }
                    if (collections.length) {
                        squash(collections);
{'fraction_non_alphanumeric': 0.13735880630316552, 'fraction_numerical': 0.0011156045181982988, 'mean_word_length': 1.309079201545396, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6672870', 'n_tokens_mistral': 1991, 'n_tokens_neox': 1892, 'n_words': 425}