<commit_before>/* eslint-env phantomjs */
/* eslint no-extend-native: 0, space-before-function-paren: 0 */

var fs = require('fs');
var webpage = require('webpage');
var args = require('system').args;

if (args.length !== 2) {
  phantom.exit(2);
}

var demoRootPath = args[1];

if (!String.prototype.endsWith) {
  String.prototype.endsWith = function(searchString, position) {
    var subjectString = this.toString();
    if (typeof position !== 'number' || !isFinite(position) || Math.floor(position) !== position || position > subjectString.length) {
      position = subjectString.length;
    }
    position -= searchString.length;
    var lastIndex = subjectString.indexOf(searchString, position);
    return lastIndex !== -1 && lastIndex === position;
  };
}

function listFiles(dir, files) {
  var entries = fs.list(dir);
  entries.forEach(function(entry) {
    if (entry === '.' || entry === '..') {
      return;
    }
    var path = [dir, entry].join('/');
    if (fs.isDirectory(path)) {
      listFiles(path, files);
    } else {
      files.push(path);
    }
  });
}

function list(dir) {
  var files = [];
  listFiles(dir, files);
  return files;
}

function asyncEach(iterableList, callback, done) {
  var i = -1, length = iterableList.length;

  function loop() {
    i++;
    if (i === length) {
      done();
      return;
    }
    callback(iterableList[i], loop);
  }
  loop();
}

function report(file, errorString) {
  if (errorString) {
    console.log('  FAILED: ' + file);
    console.log('  ' + errorString);
  } else {
    console.log('  SUCCESS: ' + file);
  }
}

function exit() {
  phantom.exit();
}

function check(file, next) {
  var page = webpage.create();

  page.onError = function(msg, trace) {
    var msgStack = ['ERROR: ' + msg];

    if (trace && trace.length) {
      msgStack.push('TRACE:');
      trace.forEach(function(t) {
        msgStack.push(' -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function + '")' : ''));
      });
    }

    report(file, msgStack.join('\n'));

    next();
  };

  page.open(file, function(status) {
    if (status !== 'success') {
      report(file, status);
    } else {
      report(file);
    }

    next();
  });

}
<commit_msg>fix exit code on error<commit_after>/* eslint-env phantomjs */
/* eslint no-extend-native: 0, space-before-function-paren: 0 */

var fs = require('fs');
var webpage = require('webpage');
var args = require('system').args;

if (args.length !== 2) {
  phantom.exit(2);
}

var demoRootPath = args[1];

if (!String.prototype.endsWith) {
  String.prototype.endsWith = function(searchString, position) {
    var subjectString = this.toString();
    if (typeof position !== 'number' || !isFinite(position) || Math.floor(position) !== position || position > subjectString.length) {
      position = subjectString.length;
    }
    position -= searchString.length;
    var lastIndex = subjectString.indexOf(searchString, position);
    return lastIndex !== -1 && lastIndex === position;
  };
}

function listFiles(dir, files) {
  var entries = fs.list(dir);
  entries.forEach(function(entry) {
    if (entry === '.' || entry === '..') {
      return;
    }
    var path = [dir, entry].join('/');
    if (fs.isDirectory(path)) {
      listFiles(path, files);
    } else {
      files.push(path);
    }
  });
}

function list(dir) {
  var files = [];
  listFiles(dir, files);
  return files;
}

function asyncEach(iterableList, callback, done) {
  var i = -1, length = iterableList.length;
  var status = true;

  function loop(pass) {
    if (pass === false && status) {
      status = false;
    }
    i++;
    if (i === length) {
      done(status);
      return;
    }
    callback(iterableList[i], loop);
  }
  loop();
}

function report(file, errorString) {
  if (errorString) {
    console.log('  FAILED: ' + file);
    console.log('  ' + errorString);
  } else {
    console.log('  SUCCESS: ' + file);
  }
}

function exit(status) {
  var code = status === false ? 1 : 0;
  phantom.exit(code);
}

function check(file, next) {
  var page = webpage.create();

  page.onError = function(msg, trace) {
    var msgStack = ['ERROR: ' + msg];

    if (trace && trace.length) {
      msgStack.push('TRACE:');
      trace.forEach(function(t) {
        msgStack.push(' -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function + '")' : ''));
      });
    }

    report(file, msgStack.join('\n'));

    next(false);
  };

  page.open(file, function(status) {
    if (status !== 'success') {
      report(file, status);
      next(false);
    } else {
      report(file);
      next();
    }
  });

}

{'fraction_non_alphanumeric': 0.1823478636167458, 'fraction_numerical': 0.0034527406128614588, 'mean_word_length': 2.8496677740863787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2703420', 'n_tokens_mistral': 1572, 'n_tokens_neox': 1534, 'n_words': 336}