<commit_before>const Renderer = require('./Renderer.js');
const Slugger = require('./Slugger.js');
const InlineLexer = require('./InlineLexer.js');
const TextRenderer = require('./TextRenderer.js');
const { defaults } = require('./defaults.js');
const {
  merge,
  unescape
} = require('./helpers.js');

/**
 * Parsing & Compiling
 */
module.exports = class Parser {
  constructor(options) {
    this.tokens = [];
    this.token = null;
    this.options = options || defaults;
    this.options.renderer = this.options.renderer || new Renderer();
    this.renderer = this.options.renderer;
    this.renderer.options = this.options;
    this.slugger = new Slugger();
  }

  /**
   * Static Parse Method
   */
  static parse(src, options) {
    const parser = new Parser(options);
    return parser.parse(src);
  };

  /**
   * Parse Loop
   */
  parse(src) {
    this.inline = new InlineLexer(src.links, this.options);
    // use an InlineLexer with a TextRenderer to extract pure text
    this.inlineText = new InlineLexer(
      src.links,
      merge({}, this.options, { renderer: new TextRenderer() })
    );
    this.tokens = src.reverse();

    let out = '';
    while (this.next()) {
      out += this.tok();
    }

    return out;
  };

  /**
   * Next Token
   */<commit_msg>rename parse argument to tokens<commit_after>const Renderer = require('./Renderer.js');
const Slugger = require('./Slugger.js');
const InlineLexer = require('./InlineLexer.js');
const TextRenderer = require('./TextRenderer.js');
const { defaults } = require('./defaults.js');
const {
  merge,
  unescape
} = require('./helpers.js');

/**
 * Parsing & Compiling
 */
module.exports = class Parser {
  constructor(options) {
    this.tokens = [];
    this.token = null;
    this.options = options || defaults;
    this.options.renderer = this.options.renderer || new Renderer();
    this.renderer = this.options.renderer;
    this.renderer.options = this.options;
    this.slugger = new Slugger();
  }

  /**
   * Static Parse Method
   */
  static parse(tokens, options) {
    const parser = new Parser(options);
    return parser.parse(tokens);
  };

  /**
   * Parse Loop
   */
  parse(tokens) {
    this.inline = new InlineLexer(tokens.links, this.options);
    // use an InlineLexer with a TextRenderer to extract pure text
    this.inlineText = new InlineLexer(
      tokens.links,
      merge({}, this.options, { renderer: new TextRenderer() })
    );
    this.tokens = tokens.reverse();

    let out = '';
    while (this.next()) {
      out += this.tok();
    }

    return out;
  };

  /**
   * Next Token
   */
{'fraction_non_alphanumeric': 0.15692307692307692, 'fraction_numerical': 0.0, 'mean_word_length': 2.830633284241532, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2074252', 'n_tokens_mistral': 821, 'n_tokens_neox': 815, 'n_words': 188}