<commit_before>var MarkdownContents,
    Contents = require('contents');

/**
 * @param {string} markdown
 */
MarkdownContents = function MarkdownContents (markdown) {
    var markdownContents;

    if (!(this instanceof MarkdownContents)) {
        return new MarkdownContents(markdown);
    }

    markdownContents = this;

    /**
     * Generate flat index of the headings.
     *
     * @return {Array}
     */
    markdownContents.articles = function () {
        var articles = [],
            headings;

        markdown.replace(/^(#+)(.*$)/mg, function (match, level, name) {
            level = level.length;
            name = name.trim();

            articles.push({
                level: level,
                id: name.toLowerCase().replace(/[^\w]+/g, '-'),
                name: name
            });
        });

        return articles;
    };

    /**
     * Generates hierarchical index of the headings from a flat index.
     *
     * @return {Array}
     */
    markdownContents.tree = function () {
        var articles,
            tree;

        articles = markdownContents.articles();
        tree = MarkdownContents.tree(articles);

        return tree;
    };

    markdownContents.markdown = function () {
        return MarkdownContents.treeToMarkdown(markdownContents.tree());
    };
};

/**
 * Generate markdown contents for an array of contents object definition.
 *
 * @param {Array} tree [{id: '', name: '', descendants: []}]
 * @return {String} markdown
 */
MarkdownContents.treeToMarkdown = function (tree, level) {
    var markdown = '',
        offset = '';

    level = level || 0;

    if (level) {
        offset = new Array(level * 4).join(' ') + ' ';
    }

    tree.forEach(function (article) {
        markdown += offset + '* [' + article.name + '](#' + article.id + ')\n';

        if (article.descendants) {
            markdown += MarkdownContents.treeToMarkdown(article.descendants, level + 1);
        }
    });

    return markdown;
};

/**
 * Makes hierarchical index of the articles from a flat index.
 * 
 * @param {Array} articles Generated using Contents.articles.
 * @return {Array}
 */
MarkdownContents.tree = function (articles) {
    return Contents.tree(articles, []);
};

module.exports = MarkdownContents;<commit_msg>Default to no unique IDs.<commit_after>var MarkdownContents,
    Contents = require('contents');

/**
 * @param {string} markdown
 */
MarkdownContents = function MarkdownContents (markdown) {
    var markdownContents;

    if (!(this instanceof MarkdownContents)) {
        return new MarkdownContents(markdown);
    }

    markdownContents = this;

    /**
     * Generate flat index of the headings.
     *
     * @return {Array}
     */
    markdownContents.articles = function () {
        var articles = [],
            headings;

        markdown.replace(/^(#+)(.*$)/mg, function (match, level, name) {
            level = level.length;
            name = name.trim();

            articles.push({
                level: level,
                id: name.toLowerCase().replace(/[^\w]+/g, '-'),
                name: name
            });
        });

        return articles;
    };

    /**
     * Generates hierarchical index of the headings from a flat index.
     *
     * @return {Array}
     */
    markdownContents.tree = function () {
        var articles,
            tree;

        articles = markdownContents.articles();
        tree = MarkdownContents.tree(articles);

        return tree;
    };

    markdownContents.markdown = function () {
        return MarkdownContents.treeToMarkdown(markdownContents.tree());
    };
};

/**
 * Generate markdown contents for an array of contents object definition.
 *
 * @param {Array} tree [{id: '', name: '', descendants: []}]
 * @return {String} markdown
 */
MarkdownContents.treeToMarkdown = function (tree, level) {
    var markdown = '',
        offset = '';

    level = level || 0;

    if (level) {
        offset = new Array(level * 4).join(' ') + ' ';
    }

    tree.forEach(function (article) {
        markdown += offset + '* [' + article.name + '](#' + article.id + ')\n';

        if (article.descendants) {
            markdown += MarkdownContents.treeToMarkdown(article.descendants, level + 1);
        }
    });

    return markdown;
};

/**
 * Makes hierarchical index of the articles from a flat index.
 * 
 * @param {Array} articles Generated using Contents.articles.
 * @return {Array}
 */
MarkdownContents.tree = function (articles) {
    return Contents.tree(articles, false);
};

module.exports = MarkdownContents;
{'fraction_non_alphanumeric': 0.13830019663535068, 'fraction_numerical': 0.0013109023377758358, 'mean_word_length': 2.237623762376238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2232787', 'n_tokens_mistral': 1336, 'n_tokens_neox': 1229, 'n_words': 314}