<commit_before>var path = require('path'),
    vow = require('vow'),
    vfs = require('enb/lib/fs/async-fs'),
    dropRequireCache = require('enb/lib/fs/drop-require-cache'),
    requireOrEval = require('enb/lib/fs/require-or-eval');

module.exports = require('enb/lib/build-flow').create()
    .name('references')
    .target('target', '?.references.js')
    .useDirList('tmpl-specs')
    .builder(function (dirs) {
        var references = {},
            bemjsons = [],
            htmls = [];

        dirs.forEach(function (dir) {
            dir.files.forEach(function (file) {
                var filename = file.name,
                    fullname = file.fullname,
                    name = filename.split('.')[0],
                    lang = getLang(fullname),
                    item = {
                        name: name,
                        fullname: fullname,
                        lang: lang
                    };

                if (hasExt(filename, '.bemjson.js')) {
                    bemjsons.push(item);
                } else if (hasExt(filename, '.html')) {
                    htmls.push(item);
                }
            });
        });

        return vow.all([
            vow.all(bemjsons.map(function (file) {
                var name = file.name,
                    lang = file.lang,
                    filename = file.fullname;

                dropRequireCache(require, filename);

                return requireOrEval(filename)
                    .then(function (code) {
                        var reference = references[name] || (references[name] = {});

                        if (lang) {
                            reference[lang] || (reference[lang] = {});
                            reference[lang].bemjson = code;
                        } else {
                            reference.bemjson = code;
                        }
                    });
            })),
            vow.all(htmls.map(function (file) {
                var name = file.name,
                    lang = file.lang;

                return vfs.read(file.fullname, 'utf-8')
                    .then(function (source) {
                        var reference = references[name] || (references[name] = {});

                        if (lang) {
                            reference[lang] || (reference[lang] = {});
                            reference[lang].html = source;
                        } else {
                            reference.html = source;
                        }
                    });
            }))
        ]).then(function () {
            return 'module.exports = ' + JSON.stringify(references) + ';';
        });
    })
    .createTech();

function hasExt(filename, ext) {
    var index = filename.indexOf(ext);

    return index !== -1 && (index === filename.length - ext.length);
}

function getLang(filename) {
    var splited = path.basename(filename).split('.');

    if (splited.length === 3 && splited[2] === 'html') {
        return splited[1];
    }

    if (splited.length === 4 && splited[3] === 'js' && splited[2] === 'bemjson') {
        return splited[1];
    }
}<commit_msg>Fixed bug with reading order of references<commit_after>var path = require('path'),
    vow = require('vow'),
    vfs = require('enb/lib/fs/async-fs'),
    dropRequireCache = require('enb/lib/fs/drop-require-cache'),
    requireOrEval = require('enb/lib/fs/require-or-eval');

module.exports = require('enb/lib/build-flow').create()
    .name('references')
    .target('target', '?.references.js')
    .useDirList('tmpl-specs')
    .builder(function (dirs) {
        var references = {},
            bemjsons = [],
            htmls = [];

        dirs.forEach(function (dir) {
            dir.files.forEach(function (file) {
                var filename = file.name,
                    fullname = file.fullname,
                    name = filename.split('.')[0],
                    lang = getLang(fullname),
                    item = {
                        name: name,
                        fullname: fullname,
                        lang: lang
                    };

                if (hasExt(filename, '.bemjson.js')) {
                    bemjsons.push(item);
                } else if (hasExt(filename, '.html')) {
                    htmls.push(item);
                }
            });
        });

        return vow.all([
            vow.all(bemjsons.map(function (file) {
                var name = file.name,
                    lang = file.lang,
                    filename = file.fullname;

                dropRequireCache(require, filename);

                return requireOrEval(filename)
                    .then(function (code) {
                        return {
                            name: name,
                            lang: lang,
                            code: code
                        };
                    });
            })),
            vow.all(htmls.map(function (file) {
                var name = file.name,
                    lang = file.lang;

                return vfs.read(file.fullname, 'utf-8')
                    .then(function (source) {
                        return {
                            name: name,
                            lang: lang,
                            source: source
                        };
                    });
            }))
        ]).spread(function (bemjsons, htmls) {
            bemjsons.forEach(function (bemjson) {
                var name = bemjson.name,
                    code = bemjson.code,
                    lang = bemjson.lang,
                    reference = references[name] || (references[name] = {});

                if (lang) {
                    reference[lang] || (reference[lang] = {});
                    reference[lang].bemjson = code;
                } else {
                    reference.bemjson = code;
                }
            });

            htmls.forEach(function (html) {
                var name = html.name,
                    source = html.source,
                    lang = html.lang,
                    reference = references[name] || (references[name] = {});

                if (lang) {
                    reference[lang] || (reference[lang] = {});
                    reference[lang].html = source;
                } else {
                    reference.html = source;
                }
            });

            return 'module.exports = ' + JSON.stringify(references) + ';';
        });
    })
    .createTech();

function hasExt(filename, ext) {
    var index = filename.indexOf(ext);

    return index !== -1 && (index === filename.length - ext.length);
}

function getLang(filename) {
    var splited = path.basename(filename).split('.');

    if (splited.length === 3 && splited[2] === 'html') {
        return splited[1];
    }

    if (splited.length === 4 && splited[3] === 'js' && splited[2] === 'bemjson') {
        return splited[1];
    }
}
{'fraction_non_alphanumeric': 0.1330940416367552, 'fraction_numerical': 0.002871500358937545, 'mean_word_length': 1.157992565055762, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4404515', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1770, 'n_words': 328}