<commit_before>#!/usr/bin/env node

/**
 * marked tests
 * Copyright (c) 2011-2013, <NAME>. (MIT Licensed)
 * https://github.com/markedjs/marked
 */

/**
 * Modules
 */

var fs = require('fs'),
    path = require('path'),
    fm = require('front-matter'),
    g2r = require('glob-to-regexp'),
    marked = require('../'),
    markedMin = require('../marked.min.js');

/**
 * Load Tests
 */

function load(options) {
  options = options || {};
  var dir = path.join(__dirname, 'compiled_tests'),
      files = {},
      list,
      file,
      name,
      content,
      glob = g2r(options.glob || '*', { extended: true }),
      i,
      l;

  list = fs
    .readdirSync(dir)
    .filter(function(file) {
      return path.extname(file) === '.md';
    })
    .sort();

  l = list.length;

  for (i = 0; i < l; i++) {
    name = path.basename(list[i], '.md');
    if (glob.test(name)) {
      file = path.join(dir, list[i]);
      content = fm(fs.readFileSync(file, 'utf8'));

      files[name] = {
        options: content.attributes,
        text: content.body,
        html: fs.readFileSync(file.replace(/[^.]+$/, 'html'), 'utf8')
      };
    }
  }

  if (options.bench || options.time) {
    if (!options.glob) {
      // Change certain tests to allow
      // comparison to older benchmark times.
      fs.readdirSync(path.join(__dirname, 'new')).forEach(function(name) {
        if (path.extname(name) === '.html') return;
        if (name === 'main.md') return;
        delete files[name];
      });
    }

    if (files['backslash_escapes.md']) {
      files['backslash_escapes.md'] = {
        text: 'hello world \\[how](are you) today'
      };
    }

    if (files['main.md']) {
      files['main.md'].text = files['main.md'].text.replace('* * *\n\n', '');
    }
  }

  return files;
}

/**
 * Test Runner
 */

function runTests(engine, options) {
  if (typeof engine !== 'function') {
    options = engine;
    engine = null;
  }

  engine = engine || marked;
  options = options || {};
  var succeeded = 0,
      failed = 0,
      files = options.files || load(options),
      filenames = Object.keys(files),
      len = filenames.length,
      success,
      i,
      filename,
      file;

  if (options.marked) {
    marked.setOptions(options.marked);
  }

  for (i = 0; i < len; i++) {
    filename = filenames[i];
    file = files[filename];

    var before = process.hrtime();
    success = testFile(engine, file, filename, i + 1);
    var elapsed = process.hrtime(before);
    var tookLessThanOneSec = (elapsed[0] === 0);

    if (success && tookLessThanOneSec) {
      succeeded++;
    } else {
      failed++;
      if (options.stop) {
        break;
      }
    }
  }

  console.log('%d/%d tests completed successfully.', succeeded, len);
  if (failed) console.log('%d/%d tests failed.', failed, len);

  return !failed;
}

/**
 * Test a file
 */

function testFile(engine, file, filename, index) {
  var opts = Object.keys(file.options),
      text,
      html,
      j,
      l,
      before,
      elapsed;

  if (marked._original) {
    marked.defaults = marked._original;
    delete marked._original;
  }

  console.log('#%d. Test %s', index, filename);

  if (opts.length) {
    marked._original = marked.defaults;
    marked.defaults = {};
    Object.keys(marked._original).forEach(function(key) {
      marked.defaults[key] = marked._original[key];
    });
    opts.forEach(function(key) {
      if (marked.defaults.hasOwnProperty(key)) {
        marked.defaults[key] = file.options[key];
      }
    });
  }

  before = process.hrtime();
  try {
    text = engine(file.text).replace(/\s/g, '');
    html = file.html.replace(/\s/g, '');
  } catch (e) {
    elapsed = process.hrtime(before);
    console.log('    failed in %dms', prettyElapsedTime(elapsed));
    throw e;
  }

  elapsed = process.hrtime(before);

  l = html.length;

  for (j = 0; j < l; j++) {
    if (text[j] !== html[j]) {
      text = text.substring(
        Math.max(j - 30, 0),
        Math.min(j + 30, text.length));

      html = html.substring(
        Math.max(j - 30, 0),
        Math.min(j + 30, l));

      console.log('    failed in %dms at offset %d. Near: "%s".\n', prettyElapsedTime(elapsed), j, text);

      console.log('\nGot:\n%s\n', text.trim() || text);
      console.log('\nExpected:\n%s\n', html.trim() || html);

      return false;
    }
  }

  console.log('    passed in %dms', prettyElapsedTime(elapsed));
  return true;
}

/**<commit_msg>show failing test when original tests takes > 1s<commit_after>#!/usr/bin/env node

/**
 * marked tests
 * Copyright (c) 2011-2013, <NAME>. (MIT Licensed)
 * https://github.com/markedjs/marked
 */

/**
 * Modules
 */

var fs = require('fs'),
    path = require('path'),
    fm = require('front-matter'),
    g2r = require('glob-to-regexp'),
    marked = require('../'),
    markedMin = require('../marked.min.js');

/**
 * Load Tests
 */

function load(options) {
  options = options || {};
  var dir = path.join(__dirname, 'compiled_tests'),
      files = {},
      list,
      file,
      name,
      content,
      glob = g2r(options.glob || '*', { extended: true }),
      i,
      l;

  list = fs
    .readdirSync(dir)
    .filter(function(file) {
      return path.extname(file) === '.md';
    })
    .sort();

  l = list.length;

  for (i = 0; i < l; i++) {
    name = path.basename(list[i], '.md');
    if (glob.test(name)) {
      file = path.join(dir, list[i]);
      content = fm(fs.readFileSync(file, 'utf8'));

      files[name] = {
        options: content.attributes,
        text: content.body,
        html: fs.readFileSync(file.replace(/[^.]+$/, 'html'), 'utf8')
      };
    }
  }

  if (options.bench || options.time) {
    if (!options.glob) {
      // Change certain tests to allow
      // comparison to older benchmark times.
      fs.readdirSync(path.join(__dirname, 'new')).forEach(function(name) {
        if (path.extname(name) === '.html') return;
        if (name === 'main.md') return;
        delete files[name];
      });
    }

    if (files['backslash_escapes.md']) {
      files['backslash_escapes.md'] = {
        text: 'hello world \\[how](are you) today'
      };
    }

    if (files['main.md']) {
      files['main.md'].text = files['main.md'].text.replace('* * *\n\n', '');
    }
  }

  return files;
}

/**
 * Test Runner
 */

function runTests(engine, options) {
  if (typeof engine !== 'function') {
    options = engine;
    engine = null;
  }

  engine = engine || marked;
  options = options || {};
  var succeeded = 0,
      failed = 0,
      files = options.files || load(options),
      filenames = Object.keys(files),
      len = filenames.length,
      success,
      i,
      filename,
      file;

  if (options.marked) {
    marked.setOptions(options.marked);
  }

  for (i = 0; i < len; i++) {
    filename = filenames[i];
    file = files[filename];

    success = testFile(engine, file, filename, i + 1);

    if (success) {
      succeeded++;
    } else {
      failed++;
      if (options.stop) {
        break;
      }
    }
  }

  console.log('%d/%d tests completed successfully.', succeeded, len);
  if (failed) console.log('%d/%d tests failed.', failed, len);

  return !failed;
}

/**
 * Test a file
 */

function testFile(engine, file, filename, index) {
  var opts = Object.keys(file.options),
      text,
      html,
      j,
      l,
      before,
      elapsed;

  if (marked._original) {
    marked.defaults = marked._original;
    delete marked._original;
  }

  console.log('#%d. Test %s', index, filename);

  if (opts.length) {
    marked._original = marked.defaults;
    marked.defaults = {};
    Object.keys(marked._original).forEach(function(key) {
      marked.defaults[key] = marked._original[key];
    });
    opts.forEach(function(key) {
      if (marked.defaults.hasOwnProperty(key)) {
        marked.defaults[key] = file.options[key];
      }
    });
  }

  before = process.hrtime();
  try {
    text = engine(file.text).replace(/\s/g, '');
    html = file.html.replace(/\s/g, '');
  } catch (e) {
    elapsed = process.hrtime(before);
    console.log('    failed in %dms', prettyElapsedTime(elapsed));
    throw e;
  }

  elapsed = process.hrtime(before);

  l = html.length;

  for (j = 0; j < l; j++) {
    if (text[j] !== html[j]) {
      text = text.substring(
        Math.max(j - 30, 0),
        Math.min(j + 30, text.length));

      html = html.substring(
        Math.max(j - 30, 0),
        Math.min(j + 30, l));

      console.log('    failed in %dms at offset %d. Near: "%s".\n', prettyElapsedTime(elapsed), j, text);

      console.log('\nGot:\n%s\n', text.trim() || text);
      console.log('\nExpected:\n%s\n', html.trim() || html);

      return false;
    }
  }

  if (elapsed[0] !== 0) {
    console.log('    failed because it took too long.\n\n    passed in %dms', prettyElapsedTime(elapsed));
    return false;
  }

  console.log('    passed in %dms', prettyElapsedTime(elapsed));
  return true;
}

/**
{'fraction_non_alphanumeric': 0.1815446339017051, 'fraction_numerical': 0.006798172294661763, 'mean_word_length': 2.327400815721172, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3044437', 'n_tokens_mistral': 3162, 'n_tokens_neox': 3030, 'n_words': 666}