<commit_before>/*!
 * fengine
 * Version: 0.0.1
 * Date: 2016/8/2
 * https://github.com/Nuintun/fengine
 *
 * This is licensed under the MIT License (MIT).
 * For details, see: https://github.com/Nuintun/fengine/blob/master/LICENSE
 */

'use strict';

// Import lib
var os = require('os');
var fs = require('fs');
var url = require('url');
var path = require('path');
var http = require('http');
var util = require('./util');
var colors = require('colors');
var cluster = require('cluster');
var mime = require('mime-types');
var FileSend = require('file-send');
var pkg = require('../package.json');
var Transform = require('./transform');

// Variable declaration
var CWD = process.cwd();
var NUMCPUS = os.cpus().length;
var LOGLEVELS = {
  INFO: 'info',
  WARN: 'warn',
  ERROR: 'error'
};
var FAVICON = path.join(__dirname, '../favicon.ico');
var FAVICONSIZE = fs.lstatSync(FAVICON).size;

/**
 * Fengine
 * @param options
 * @constructor
 */
function Fengine(options) {
  this.options = options;

  this.run();
}

/**
 * prototype
 *
 * @type {{
 *   dir: Fengine.dir,
 *   error: Fengine.error,
 *   log: Fengine.log,
 *   run: Fengine.run
 * }}
 */
Fengine.prototype = {
  dir: function(files, dir) {
    var parent = util.normalize(path.join(dir, '../'));
    var upLink = '      <span>—</span>\n'
      + '      <a href="' + parent + '" title="Back to parent directory."><h1>up</h1></a>\n';
    var up = dir === '/' ? '' : upLink;
    var html = '<!DOCTYPE html>\n'
      + '<html>\n'
      + '  <head>\n'
      + '    <meta name="renderer" content="webkit" />\n'
      + '    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />\n'
      + '    <meta content="text/html; charset=utf-8" http-equiv="content-type" />\n'
      + '    <title>' + dir + '</title>\n'
      + '    <style>\n'
      + '      html, body, p {\n'
      + '        margin: 0; padding: 0;\n'
      + '        font-family: "Lucida Console", Consolas, "Liberation Mono", Menlo, Courier, monospace;\n'
      + '      }\n'
      + '      h1 { margin: 10px 0; }\n'
      + '      a { display: inline-block; color: #0e90d2; text-decoration: none; }\n'
      + '      a:focus, a:hover { color: #095f8a; }\n'
      + '      a:focus { outline: thin dotted; }\n'
      + '      a:active, a:hover { outline: 0; }\n'
      + '      a:hover { text-decoration: underline; }\n'
      + '      .ui-dir {\n'
      + '         text-indent: 10px; border-bottom: 1px solid #ddd; margin-bottom: 10px; background: #f5f5f5;\n'
      + '      }\n'
      + '      .ui-dir > span + a h1 { text-indent: 0; }\n'
      + '      .ui-file { padding: 4px 30px; }\n'
      + '      .ui-file > span { color: #080; }\n'
      + '    </style>\n'
      + '  </head>\n'
      + '  <body>\n'
      + '    <div class="ui-dir">\n'
      + '      <a href="' + dir + '" title="' + dir + '"><h1>' + dir + '</h1></a>\n' + up
      + '    </div>\n';

    var width = (files.length + 1).toString().length;

    files.forEach(function(file, index) {
      var href = dir + file;

      html += ''
        + '    <div class="ui-file">\n'
        + '      <span>' + util.pad(++index, width) + ':</span>\n'
        + '      <a href="' + href + '" title="' + href + '">' + file + '</a></div>\n'
        + '    </div>\n';
    });

    html += '  </body>\n'
      + '</html>\n';

    return html;
  },
  error: function(response, statusCode, next) {
    var html = '<!DOCTYPE html>\n'
      + '<html>\n'
      + '  <head>\n'
      + '    <meta name="renderer" content="webkit" />\n'
      + '    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />\n'
      + '    <meta content="text/html; charset=utf-8" http-equiv="content-type" />\n'
      + '    <title>' + statusCode + '</title>\n'
      + '    <style>\n'
      + '      html, body, div, p {\n'
      + '        margin: 0; padding: 0;\n'
      + '        text-align: center;\n'
      + '        font-family: Calibri, "Lucida Console", Consolas, "Liberation Mono", Menlo, Courier, monospace;\n'
      + '      }\n'
      + '      p { color: #0e90d2; line-height: 100%; }\n'
      + '      .ui-code { font-size: 200px; font-weight: bold; margin-top: 66px; }\n'
      + '      .ui-message { font-size: 80px; }\n'
      + '    </style>\n'
      + '  </head>\n'
      + '  <body>\n'
      + '    <p class="ui-code">' + statusCode + '</p>\n'
      + '    <p class="ui-message">' + http.STATUS_CODES[statusCode] + '</p>\n'
      + '  </body>\n'
      + '</html>\n';

    if (util.fn(next)) {
      next(html);
    } else {
      response.statusCode = statusCode;

      response.setHeader('Content-Type', 'text/html; charset=UTF-8');
      response.end(html);
    }
  },
  log: function(data) {
    var date = util.dateFormat(new Date(), 'yyyy-MM-dd hh:mm:ss');
    var message = colors.green.bold('[' + date + '] ');

    switch (data.type) {
      case LOGLEVELS.INFO:
        message += colors.cyan.bold(data.message);
        break;
      case LOGLEVELS.WARN:
        message += colors.yellow.bold(data.message);
        break;
      case LOGLEVELS.ERROR:
        message += colors.red.bold(data.message);
        break;
    }
<commit_msg>clean code<commit_after>/*!
 * fengine
 * Version: 0.0.1
 * Date: 2016/8/2
 * https://github.com/Nuintun/fengine
 *
 * This is licensed under the MIT License (MIT).
 * For details, see: https://github.com/Nuintun/fengine/blob/master/LICENSE
 */

'use strict';

// Import lib
var os = require('os');
var fs = require('fs');
var url = require('url');
var path = require('path');
var http = require('http');
var util = require('./util');
var colors = require('colors');
var cluster = require('cluster');
var mime = require('mime-types');
var FileSend = require('file-send');
var pkg = require('../package.json');
var Transform = require('./transform');

// Variable declaration
var CWD = process.cwd();
var NUMCPUS = os.cpus().length;
var LOGLEVELS = {
  INFO: 'info',
  WARN: 'warn',
  ERROR: 'error'
};
var FAVICON = path.join(__dirname, '../favicon.ico');
var FAVICONSIZE = fs.lstatSync(FAVICON).size;

/**
 * Fengine
 * @param options
 * @constructor
 */
function Fengine(options) {
  this.options = options;

  this.run();
}

/**
 * prototype
 *
 * @type {{
 *   dir: Fengine.dir,
 *   error: Fengine.error,
 *   log: Fengine.log,
 *   run: Fengine.run
 * }}
 */
Fengine.prototype = {
  dir: function(files, dir) {
    var parent = util.normalize(path.join(dir, '../'));
    var upLink = '      <span>—</span>\n'
      + '      <a href="' + parent + '" title="Back to parent directory."><h1>up</h1></a>\n';
    var up = dir === '/' ? '' : upLink;
    var html = '<!DOCTYPE html>\n'
      + '<html>\n'
      + '  <head>\n'
      + '    <meta name="renderer" content="webkit" />\n'
      + '    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />\n'
      + '    <meta content="text/html; charset=utf-8" http-equiv="content-type" />\n'
      + '    <title>' + dir + '</title>\n'
      + '    <style>\n'
      + '      html, body, p {\n'
      + '        margin: 0; padding: 0;\n'
      + '        font-family: "Lucida Console", Consolas, "Liberation Mono", Menlo, Courier, monospace;\n'
      + '      }\n'
      + '      h1 { margin: 10px 0; }\n'
      + '      a { display: inline-block; color: #0e90d2; text-decoration: none; }\n'
      + '      a:focus, a:hover { color: #095f8a; }\n'
      + '      a:focus { outline: thin dotted; }\n'
      + '      a:active, a:hover { outline: 0; }\n'
      + '      a:hover { text-decoration: underline; }\n'
      + '      .ui-dir {\n'
      + '         text-indent: 10px; border-bottom: 1px solid #ddd; margin-bottom: 10px; background: #f5f5f5;\n'
      + '      }\n'
      + '      .ui-dir > span + a h1 { text-indent: 0; }\n'
      + '      .ui-file { padding: 4px 30px; }\n'
      + '      .ui-file > span { color: #080; }\n'
      + '    </style>\n'
      + '  </head>\n'
      + '  <body>\n'
      + '    <div class="ui-dir">\n'
      + '      <a href="' + dir + '" title="' + dir + '"><h1>' + dir + '</h1></a>\n' + up
      + '    </div>\n';

    var width = (files.length + 1).toString().length;

    files.forEach(function(file, index) {
      var href = dir + file;

      html += ''
        + '    <div class="ui-file">\n'
        + '      <span>' + util.pad(++index, width) + ':</span>\n'
        + '      <a href="' + href + '" title="' + href + '">' + file + '</a></div>\n'
        + '    </div>\n';
    });

    html += '  </body>\n'
      + '</html>\n';

    return html;
  },
  error: function(response, statusCode, next) {
    var html = '<!DOCTYPE html>\n'
      + '<html>\n'
      + '  <head>\n'
      + '    <meta name="renderer" content="webkit" />\n'
      + '    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />\n'
      + '    <meta content="text/html; charset=utf-8" http-equiv="content-type" />\n'
      + '    <title>' + statusCode + '</title>\n'
      + '    <style>\n'
      + '      html, body, div, p {\n'
      + '        margin: 0; padding: 0;\n'
      + '        text-align: center;\n'
      + '        font-family: Calibri, "Lucida Console", Consolas, "Liberation Mono", Menlo, Courier, monospace;\n'
      + '      }\n'
      + '      p { color: #0e90d2; line-height: 100%; }\n'
      + '      .ui-code { font-size: 200px; font-weight: bold; margin-top: 66px; }\n'
      + '      .ui-message { font-size: 80px; }\n'
      + '    </style>\n'
      + '  </head>\n'
      + '  <body>\n'
      + '    <p class="ui-code">' + statusCode + '</p>\n'
      + '    <p class="ui-message">' + http.STATUS_CODES[statusCode] + '</p>\n'
      + '  </body>\n'
      + '</html>\n';

    if (util.fn(next)) {
      next(html);
    } else {
      response.statusCode = statusCode;

      response.setHeader('Content-Type', 'text/html; charset=UTF-8');
      response.end(html);
    }
  },
  log: function(data) {
    var date = util.dateFormat(new Date());
    var message = colors.green.bold('[' + date + '] ');

    switch (data.type) {
      case LOGLEVELS.INFO:
        message += colors.cyan.bold(data.message);
        break;
      case LOGLEVELS.WARN:
        message += colors.yellow.bold(data.message);
        break;
      case LOGLEVELS.ERROR:
        message += colors.red.bold(data.message);
        break;
    }

{'fraction_non_alphanumeric': 0.21183466562682784, 'fraction_numerical': 0.012868005459153831, 'mean_word_length': 2.0789315726290516, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 109, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249433', 'n_tokens_mistral': 3838, 'n_tokens_neox': 3688, 'n_words': 777}