<commit_before>var EOL = require('os').EOL;
var util = require('util');
var marked = require('marked');
var MarkedRenderer = marked.Renderer;
var repeat = require('cli-util').repeat;
var links = require('../links');
var dnl = EOL + EOL;

var unescape = require('../escape').unescape;

function header(text) {
  return text + EOL + repeat(text.length, '=');
}

function Renderer(options) {
  MarkedRenderer.apply(this, arguments);
  this.lists = [];
  this.depth = -1;
}

util.inherits(Renderer, MarkedRenderer);

Renderer.prototype.code = function(code, lang, escaped) {
  return '```' + (lang || '') + EOL + code + EOL + '```' + dnl;
};

Renderer.prototype.blockquote = function(quote) {
  return '> ' + quote;
};

Renderer.prototype.html = function(html) {
  return html;
};

Renderer.prototype.heading = function(text, level, raw) {
  if(level == 1) return header(text) + dnl;
  return repeat(level, '#') + ' ' + text + dnl;
};

Renderer.prototype.hr = function() {
  return repeat(80, '-');
};

Renderer.prototype.after = function(text, token, parser) {
  // crude way of removing escapes
  return unescape(text);
}

Renderer.prototype.token = function(token, parser) {
  switch(token.type) {
    case 'list_start':
      this.depth = token.depth - 1;
      if(token.ordered && !this.lists[this.depth]) {
        this.lists[this.depth] = {counter: 0, ordered: token.ordered};
      }
      break;
    case 'list_item_end':
      if(this.lists[this.depth]) this.lists[this.depth].counter++;
      break;
    case 'list_end':
      var next = parser.peek();
      // not a consecutive list
      if(next.type !== 'list_start') {
        // reset depth so that li elements for ul elements
        // are restored
        this.depth = -1;
        this.lists = [];
      }
      break;<commit_msg>Guard against undefined text in after().<commit_after>var EOL = require('os').EOL;
var util = require('util');
var marked = require('marked');
var MarkedRenderer = marked.Renderer;
var repeat = require('cli-util').repeat;
var links = require('../links');
var dnl = EOL + EOL;

var unescape = require('../escape').unescape;

function header(text) {
  return text + EOL + repeat(text.length, '=');
}

function Renderer(options) {
  MarkedRenderer.apply(this, arguments);
  this.lists = [];
  this.depth = -1;
}

util.inherits(Renderer, MarkedRenderer);

Renderer.prototype.code = function(code, lang, escaped) {
  return '```' + (lang || '') + EOL + code + EOL + '```' + dnl;
};

Renderer.prototype.blockquote = function(quote) {
  return '> ' + quote;
};

Renderer.prototype.html = function(html) {
  return html;
};

Renderer.prototype.heading = function(text, level, raw) {
  if(level == 1) return header(text) + dnl;
  return repeat(level, '#') + ' ' + text + dnl;
};

Renderer.prototype.hr = function() {
  return repeat(80, '-');
};

Renderer.prototype.after = function(text, token, parser) {
  if(!text) return text;
  // crude way of removing escapes
  return unescape(text);
}

Renderer.prototype.token = function(token, parser) {
  switch(token.type) {
    case 'list_start':
      this.depth = token.depth - 1;
      if(token.ordered && !this.lists[this.depth]) {
        this.lists[this.depth] = {counter: 0, ordered: token.ordered};
      }
      break;
    case 'list_item_end':
      if(this.lists[this.depth]) this.lists[this.depth].counter++;
      break;
    case 'list_end':
      var next = parser.peek();
      // not a consecutive list
      if(next.type !== 'list_start') {
        // reset depth so that li elements for ul elements
        // are restored
        this.depth = -1;
        this.lists = [];
      }
      break;
{'fraction_non_alphanumeric': 0.1676728869374314, 'fraction_numerical': 0.0038419319429198683, 'mean_word_length': 3.2090069284064664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422080', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1172, 'n_words': 275}