<commit_before>var fs = require('fs');
var log = require('util').log;
var url = require('url');

var mode = process.env.NODE_ENV || 'development';
module.exports.mode = mode;

function debugLog(label) {
  function toString(value) {
    if (typeof value === 'function') {
      return '[function]';
    }
    return value.toString ? value.toString() : value;
  }

  var string;
  if (arguments.length > 2) {
    string = Array.prototype.slice.call(arguments, 1).map(toString).join('  ');
  } else {
    string = toString(arguments[1]);
  }
  string = label.toUpperCase() +': '+ string;
  if (string.indexOf('\n') !== -1) {
    string = '\n\n'+ string;
  }

  log(string +'\n');
}
module.exports.log = (mode !== 'development') ? function() {} : debugLog;

module.exports.normalizeLink = function(link) {
  var parsed = url.parse(link);
  return parsed.host + parsed.pathname;
};

module.exports.patterns = {
  domain: /:\/\/(?:www\.)?([^\/]+)/,
  line: /\n/g,
  createFromTokens: function(escapedTokens) {
    return new RegExp('\\b('+ escapedTokens.join('|') +')\\b');
  }
};

function handleFileError(delegate, retry, error) {
  if (error.code === 'ENOENT') {
    fs.openSync(delegate.file, 'a');
    retry(delegate);
  } else {
    throw error;
  }<commit_msg>Allow logging with one argument<commit_after>var fs = require('fs');
var log = require('util').log;
var url = require('url');

var mode = process.env.NODE_ENV || 'development';
module.exports.mode = mode;

function debugLog(label) {
  function toString(value) {
    if (typeof value === 'function') {
      return '[function]';
    }
    return value.toString ? value.toString() : value;
  }

  var string;
  if (arguments.length > 1) {
    if (arguments.length > 2) {
      string = Array.prototype.slice.call(arguments, 1).map(toString).join('  ');
    } else {
      string = toString(arguments[1]);
    }
    string = label.toUpperCase() +': '+ string;
  } else {
    string = 'LOG: '+ arguments[0];    
  }

  if (string.indexOf('\n') !== -1) {
    string = '\n\n'+ string;
  }

  log(string +'\n');
}
module.exports.log = (mode !== 'development') ? function() {} : debugLog;

module.exports.normalizeLink = function(link) {
  var parsed = url.parse(link);
  return parsed.host + parsed.pathname;
};

module.exports.patterns = {
  domain: /:\/\/(?:www\.)?([^\/]+)/,
  line: /\n/g,
  createFromTokens: function(escapedTokens) {
    return new RegExp('\\b('+ escapedTokens.join('|') +')\\b');
  }
};

function handleFileError(delegate, retry, error) {
  if (error.code === 'ENOENT') {
    fs.openSync(delegate.file, 'a');
    retry(delegate);
  } else {
    throw error;
  }
{'fraction_non_alphanumeric': 0.2001516300227445, 'fraction_numerical': 0.0037907505686125853, 'mean_word_length': 3.3333333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4170750', 'n_tokens_mistral': 912, 'n_tokens_neox': 901, 'n_words': 189}