<commit_before>var metalSmith = require('metalsmith');

function idToName(id) {
    return id.replace(/-/g, ' ');
}

metalSmith(__dirname)
    .destination('../site-build')
    .source('src')
    .use(require('metalsmith-collections')({
        assertions: {
            pattern: 'assertions/*/*.md'
        },
        apiPages: {
            pattern: 'api/*.md'
        },
        pages: {
            pattern: '*.md'
        }
    }))
    .use(require('./lib/include-static-assets')({ path: require('path').resolve(__dirname, 'static') }))
    // Dynamicly generate metadata for assertion files
    .use(function (files, metalsmith, next) {
        Object.keys(files).filter(function (file) {
            return /\.md$/.test(file);
        }).forEach(function (file) {
            var id = file.match(/([^\/]+)\.md$/)[1];
            var name = idToName(id);

            if (!files[file].title) {
                files[file].title = name;
            }

            if (files[file].collection.indexOf('apiPages') !== -1) {
                files[file].template = 'api.ejs';
            } else if (files[file].collection.indexOf('assertions') !== -1) {
                var type = file.match(/^assertions\/([^\/]+)/)[1];

                files[file].template = 'assertion.ejs';
                files[file].windowTitle = type + ' - ' + name;
                files[file].type = type;
            }

            if (!files[file].windowTitle) {
                files[file].windowTitle = files[file].title;
            }

            files[file].url = '/' + file.replace(/\.md$/, '/');
        });
        next();
    })
    .use(function (files, metalsmith, next) {
        var metadata = metalsmith.metadata();

        // Make sure that the most important types are listed first and in this order:
        var assertionsByType = {
            any: [],
            array: [],
            boolean: [],
            function: [],
            number: [],
            object: [],
            string: []
        };
        metadata.collections.assertions.forEach(function (assertion) {
            assertionsByType[assertion.type] = assertionsByType[assertion.type] || [];
            assertionsByType[assertion.type].push(assertion);
        });
        Object.keys(assertionsByType).forEach(function (type) {
            assertionsByType[type].sort(function (a, b) {
                if (a.name < b.name) return -1;
                if (a.name > b.name) return 1;
                return 0;
            });
        });
        metadata.assertionsByType = assertionsByType;
        next();
    })
    .use(function (files, metalsmith, next) {
        var metadata = metalsmith.metadata();
        var indexData = [];

        metadata.collections.apiPages.forEach(function (assertion) {
            indexData.push({
                label: assertion.title + ' (api)',
                url: assertion.url
            });
        });

        metadata.collections.assertions.forEach(function (assertion) {
            indexData.push({<commit_msg>Make the assertion menu ordering based on the type hierarchy<commit_after>var metalSmith = require('metalsmith');
var expect = require('../lib/');

function idToName(id) {
    return id.replace(/-/g, ' ');
}

function getTypeHierarchy(typeIndex, typeName) {
    var tree = {};
    Object.keys(typeIndex[typeName] || {}).forEach(function (childTypeName) {
        tree[childTypeName] = getTypeHierarchy(typeIndex, childTypeName);
    });

    return tree;
}

function flattenTypeHierarchy(typeHierarchy, assertionsByType) {
    var result = [];
    Object.keys(typeHierarchy).sort(function (a, b) {
        var aChildren = Object.keys(typeHierarchy[a]);
        var bChildren = Object.keys(typeHierarchy[b]);
        if (aChildren.length === 0 && bChildren.length > 0) {
            return -1;
        }
        if (aChildren.length > 0 && bChildren.length === 0) {
            return 1;
        }

        if (assertionsByType[a].length > assertionsByType[b].length) {
            return -1;
        }
        if (assertionsByType[a].length < assertionsByType[b].length) {
            return 1;
        }


        a = a.toLowerCase();
        b = b.toLowerCase();
        if (a < b) { return -1; }
        if (a > b) { return 1; }
        return 0;
    }).forEach(function (typeName) {
        result.push(typeName);
        Array.prototype.push.apply(result, flattenTypeHierarchy(typeHierarchy[typeName], assertionsByType));
    });
    return result;
}

function addTypeToIndex(typeIndex, type) {
    if (type.baseType) {
        typeIndex[type.baseType.name] = typeIndex[type.baseType.name] || {};
        typeIndex[type.baseType.name][type.name] = type.name;
        addTypeToIndex(typeIndex, type.baseType);
    }
}

function sortTypesByHierarchy(assertionsByType) {
    var typeIndex = {};
    var unknownTypes = [];
    Object.keys(assertionsByType).forEach(function (typeName) {
        var type = expect.getType(typeName);
        if (type) {
            addTypeToIndex(typeIndex, type);
        } else {
            unknownTypes.push(typeName);
        }
    });

    var typeHierarchy = { 'any': getTypeHierarchy(typeIndex, 'any') };

    var result = {};

    flattenTypeHierarchy(typeHierarchy, assertionsByType).concat(unknownTypes).forEach(function (typeName) {
        if (assertionsByType[typeName]) {
            result[typeName] = assertionsByType[typeName];
        }
    });

    return result;
}

metalSmith(__dirname)
    .destination('../site-build')
    .source('src')
    .use(require('metalsmith-collections')({
        assertions: {
            pattern: 'assertions/*/*.md'
        },
        apiPages: {
            pattern: 'api/*.md'
        },
        pages: {
            pattern: '*.md'
        }
    }))
    .use(require('./lib/include-static-assets')({ path: require('path').resolve(__dirname, 'static') }))
    // Dynamicly generate metadata for assertion files
    .use(function (files, metalsmith, next) {
        Object.keys(files).filter(function (file) {
            return /\.md$/.test(file);
        }).forEach(function (file) {
            var id = file.match(/([^\/]+)\.md$/)[1];
            var name = idToName(id);

            if (!files[file].title) {
                files[file].title = name;
            }

            if (files[file].collection.indexOf('apiPages') !== -1) {
                files[file].template = 'api.ejs';
            } else if (files[file].collection.indexOf('assertions') !== -1) {
                var type = file.match(/^assertions\/([^\/]+)/)[1];

                files[file].template = 'assertion.ejs';
                files[file].windowTitle = type + ' - ' + name;
                files[file].type = type;
            }

            if (!files[file].windowTitle) {
                files[file].windowTitle = files[file].title;
            }

            files[file].url = '/' + file.replace(/\.md$/, '/');
        });
        next();
    })
    .use(function (files, metalsmith, next) {
        var metadata = metalsmith.metadata();

        // Make sure that the most important types are listed first and in this order:
        var assertionsByType = {};
        metadata.collections.assertions.forEach(function (assertion) {
            assertionsByType[assertion.type] = assertionsByType[assertion.type] || [];
            assertionsByType[assertion.type].push(assertion);
        });

        assertionsByType = sortTypesByHierarchy(assertionsByType);
        Object.keys(assertionsByType).forEach(function (type) {
            assertionsByType[type].sort(function (a, b) {
                if (a.name < b.name) return -1;
                if (a.name > b.name) return 1;
                return 0;
            });
        });
        metadata.assertionsByType = assertionsByType;
        next();
    })
    .use(function (files, metalsmith, next) {
        var metadata = metalsmith.metadata();
        var indexData = [];

        metadata.collections.apiPages.forEach(function (assertion) {
            indexData.push({
                label: assertion.title + ' (api)',
                url: assertion.url
            });
        });

        metadata.collections.assertions.forEach(function (assertion) {
            indexData.push({
{'fraction_non_alphanumeric': 0.14964831433422265, 'fraction_numerical': 0.003031772980839195, 'mean_word_length': 1.9453571428571428, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5235278', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2348, 'n_words': 427}