<commit_before>var fs = require('fs')
  , util = require('util')
  , execSync = require('child_process').execSync
  , ansi = require('ttycolor').ansi
  , tempfile = require('tempfile')
  , printer = require('./print')
  , State = require('../../state')

/**
 *  Returns a closure that may be added to the crawler 
 *  `fetchcomplete` event.
 */
function fetch(info, req, crawler) {

  /**
   *  Handler for the fetch complete event.
   *
   *  For once we actually *want* to use the synchronous methods 
   *  otherwise the crawler keeps on going and it is hard to parse the 
   *  output.
   *
   *  Unfortunately the simplecrawler async logic does not pause or block 
   *  crawling so lots of messages would still be printed interrupting the 
   *  output from the validation.
   *
   *  Therefore we use the synchronous filesystem and child_process methods to 
   *  block and keep the program output in sequence.
   */
  return function onFetchComplete(item, buf, res) {

    // print the output of the request (info)
    req.print(new State(info, req, crawler, item, buf, res));

    // TODO: check contentType includes text/html

    if(buf && buf.length) {
      var file = tempfile('.html')
        , output = file + '.json'
        , cmd
        , opts
        , result
        , pass = false
        , sleep
        , map = {};

      this.log.debug('write file: %s', file);
      fs.writeFileSync(file, buf);

      // track files so we can map
      // .html file paths to the remote URL
      map['file:' + file] = {
        url: item.url,
        output: output,
        item: item
      }

      // setup command options
      cmd = util.format('java -jar %s --format json %s', info.jar, file);
      opts = {env: process.env, stdio: [0, 1, fs.openSync(output, 'w')]};
      this.log.debug('%s', cmd);

      try {
        execSync(cmd, opts);
      }catch(e) {
        this.log.error('validation failed on %s', item.url); 
      }

      // read in validation output and parse
      try {
        result = '' + fs.readFileSync(output); 
      }catch(e) {
        this.log.error('failed to read output file %s', output); <commit_msg>Pass --errors-only to validator jar.<commit_after>var fs = require('fs')
  , util = require('util')
  , execSync = require('child_process').execSync
  , ansi = require('ttycolor').ansi
  , tempfile = require('tempfile')
  , printer = require('./print')
  , State = require('../../state')

/**
 *  Returns a closure that may be added to the crawler 
 *  `fetchcomplete` event.
 */
function fetch(info, req, crawler) {

  /**
   *  Handler for the fetch complete event.
   *
   *  For once we actually *want* to use the synchronous methods 
   *  otherwise the crawler keeps on going and it is hard to parse the 
   *  output.
   *
   *  Unfortunately the simplecrawler async logic does not pause or block 
   *  crawling so lots of messages would still be printed interrupting the 
   *  output from the validation.
   *
   *  Therefore we use the synchronous filesystem and child_process methods to 
   *  block and keep the program output in sequence.
   */
  return function onFetchComplete(item, buf, res) {

    // print the output of the request (info)
    req.print(new State(info, req, crawler, item, buf, res));

    // TODO: check contentType includes text/html

    if(buf && buf.length) {
      var file = tempfile('.html')
        , output = file + '.json'
        , cmd
        , opts
        , result
        , pass = false
        , sleep
        , map = {}
        , args = [];

      if(this.errorsOnly) {
        args.push('--errors-only'); 
      }

      this.log.debug('write file: %s', file);
      fs.writeFileSync(file, buf);

      // track files so we can map
      // .html file paths to the remote URL
      map['file:' + file] = {
        url: item.url,
        output: output,
        item: item
      }

      // setup command options
      args = util.format(' %s ', args.join(' '));
      cmd = util.format(
        'java -jar %s%s --format json %s', info.jar, args, file);
      opts = {env: process.env, stdio: [0, 1, fs.openSync(output, 'w')]};
      this.log.debug('%s', cmd);

      try {
        execSync(cmd, opts);
      }catch(e) {
        this.log.error('validation failed on %s', item.url); 
      }

      // read in validation output and parse
      try {
        result = '' + fs.readFileSync(output); 
      }catch(e) {
        this.log.error('failed to read output file %s', output); 
{'fraction_non_alphanumeric': 0.12806782686300758, 'fraction_numerical': 0.000892458723784025, 'mean_word_length': 2.2136200716845877, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2477607', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1325, 'n_words': 449}