<commit_before>/*!
 * markdown-toc <https://github.com/jonschlinkert/markdown-toc>
 *
 * Copyright (c) 2014 <NAME>, contributors.
 * Licensed under the MIT license.
 */

'use strict';

/**
 * Module dependencies
 */

var Remarkable = require('remarkable');
var extend = require('extend-shallow');
var mdu = require('markdown-utils');

/**
 * Load `generate` as a remarkable plugin and
 * expose the `toc` function.
 *
 * @param  {String} `str` String of markdown
 * @param  {Object} `options`
 * @return {String} Markdown-formatted table of contents
 */

module.exports = function toc(str, options) {
  return new Remarkable()
    .use(generate(options))
    .render(str);
};

/**
 * Expose `insert` method
 */

module.exports.insert = require('./lib/insert');

/**
 * Generate a markdown table of contents. This is the
 * function that does all of the main work with Remarkable.
 *
 * @param {Object} `options`
 * @return {String}
 */

function generate(options) {
  var opts = extend({firsth1: true, maxdepth: 6}, options);

  return function(md) {
    md.renderer.render = function (tokens) {
      tokens = tokens.slice();
      var res = [], i = 0, h = 0;
      var len = tokens.length;
      var tocstart = -1;

      while (len--) {
        var token = tokens[i++];
        if (/<!--[ \t]*toc[ \t]*-->/.test(token.content)) {
          tocstart = token.lines[1];
        }

        if (token.type === 'heading_open') {
          var lvl = tokens[i].lvl = tokens[i - 1].hLevel;
          res.push(tokens[i]);
        }
      }

      // exclude headings that come before the actual
      // table of contents.
      res = res.reduce(function(acc, token) {
        if (token.lines[0] > tocstart) { acc.push(token); }
        token = linkify(token, opts);
        return acc;
      }, []);

      opts.highest = highest(res);
      return {
        highest: opts.highest,
        content: bullets(res, opts),
        tokens: tokens
      };
    };
  };
}

/**
 * Render markdown list bullets
 *
 * @param  {Array} `arr` Array of listitem objects
 * @param  {Object} `opts`
 * @return {String}
 */

function bullets(arr, opts) {
  var unindent = 0;

  // Keep the first h1? This is `true` by default
  if(opts && opts.firsth1 === false) {
    unindent = 1;
    arr.shift();
  }

  var len = arr.length;
  var res = [];
  var i = 0;

  while (i < len) {
    var ele = arr[i++];
    ele.lvl -= unindent;

    res.push(mdu.listitem(ele.content, ele.lvl, opts));

    // break if heading level is greater than maxdepth
    if (ele.lvl === opts.maxdepth) {
      break;
    }
  }

  return res.join('\n');
}

/**
 * Get the highest heading level in the array, so
 * we can un-indent the proper number of levels.
 *
 * @param {Array} `arr` Array of tokens
 * @return {Number} Highest level
 */<commit_msg>expose `json` property<commit_after>/*!
 * markdown-toc <https://github.com/jonschlinkert/markdown-toc>
 *
 * Copyright (c) 2014-2015, <NAME>.
 * Licensed under the MIT license.
 */

'use strict';

/**
 * Module dependencies
 */

var Remarkable = require('remarkable');
var extend = require('extend-shallow');
var mdu = require('markdown-utils');
var pick = require('object.pick');

/**
 * Load `generate` as a remarkable plugin and
 * expose the `toc` function.
 *
 * @param  {String} `str` String of markdown
 * @param  {Object} `options`
 * @return {String} Markdown-formatted table of contents
 */

module.exports = function toc(str, options) {
  return new Remarkable()
    .use(generate(options))
    .render(str);
};

/**
 * Expose `insert` method
 */

module.exports.insert = require('./lib/insert');

/**
 * Generate a markdown table of contents. This is the
 * function that does all of the main work with Remarkable.
 *
 * @param {Object} `options`
 * @return {String}
 */

function generate(options) {
  var opts = extend({firsth1: true, maxdepth: 6}, options);

  return function(md) {
    md.renderer.render = function (tokens) {
      tokens = tokens.slice();
      var res = [], i = 0, h = 0;
      var len = tokens.length;
      var tocstart = -1;

      while (len--) {
        var token = tokens[i++];
        if (/<!--[ \t]*toc[ \t]*-->/.test(token.content)) {
          tocstart = token.lines[1];
        }

        if (token.type === 'heading_open') {
          var lvl = tokens[i].lvl = tokens[i - 1].hLevel;
          res.push(tokens[i]);
        }
      }

      var results = [];
      var json = [];

      // exclude headings that come before the actual
      // table of contents.
      res.forEach(function(token) {
        if (token.lines[0] > tocstart) {
          json.push(pick(token, ['content', 'lvl']));
          results.push(linkify(token, opts));
        }
      });

      opts.highest = highest(results);
      return {
        tokens: tokens,
        highest: opts.highest,
        content: bullets(results, opts),
        json: json
      };
    };
  };
}

/**
 * Render markdown list bullets
 *
 * @param  {Array} `arr` Array of listitem objects
 * @param  {Object} `opts`
 * @return {String}
 */

function bullets(arr, opts) {
  var unindent = 0;

  // Keep the first h1? This is `true` by default
  if(opts && opts.firsth1 === false) {
    unindent = 1;
    arr.shift();
  }

  var len = arr.length;
  var res = [];
  var i = 0;

  while (i < len) {
    var ele = arr[i++];
    ele.lvl -= unindent;

    res.push(mdu.listitem(ele.content, ele.lvl, opts));
    if (ele.lvl === opts.maxdepth) {
      break;
    }
  }

  return res.join('\n');
}

/**
 * Get the highest heading level in the array, so
 * we can un-indent the proper number of levels.
 *
 * @param {Array} `arr` Array of tokens
 * @return {Number} Highest level
 */
{'fraction_non_alphanumeric': 0.15888673595208738, 'fraction_numerical': 0.006693676237449357, 'mean_word_length': 2.4706601466992666, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604534', 'n_tokens_mistral': 1913, 'n_tokens_neox': 1837, 'n_words': 531}