<commit_before>var path = require('path');
var fs = require('fs');
var crypto = require('crypto');

var externalRx = /^(\s*[a-z0-9\-]+:)?\/\//i;
var queryAndHashRx = /[\?\#].*$/;
var slashRx = /\\/g;

var textFileExt = ['.css', '.js', '.json', '.tmpl', '.txt', '.svg', '.html'];
var textType = ['script', 'style', 'template', 'json', 'l10n', 'culture', 'xml', 'svg', 'text'];

var typeByExt = {
  '.js': 'script',
  '.css': 'style',
  '.tmpl': 'template',
  '.html': 'html',
  '.json': 'json',
  '.l10n': 'json',
  '.xml': 'xml',
  '.svg': 'svg',
  '.cur': 'image',
  '.ico': 'image',
  '.bmp': 'image',
  '.gif': 'image',
  '.png': 'image',
  '.jpg': 'image',
  '.jpeg': 'image',
  '.tiff': 'image',
  '.ttf': 'font',
  '.woff': 'font',
  '.eot': 'font'
};

var typeNotFoundHandler = {
  '.js': '/* Javascript file {filename} not found */',
  '.css': '/* CSS file {filename} not found */'
};

function getFileContentOnFailure(filename){
  return (typeNotFoundHandler[path.extname(filename)] || '').replace(/\{filename\}/, filename);
}

function unixpath(filename){
  return path.normalize(filename).replace(slashRx, '/').replace(/^[a-z]\:/i, '');
}

function abspath(baseURI, filename){
  return unixpath(path.resolve(baseURI, filename.replace(queryAndHashRx, '')).replace(/^[a-z]\:/i, ''));
}

function isExternal(uri){
  return externalRx.test(uri);
}

function getRef(seed, type){
  seed[type] = (type in seed ? seed[type] : -1) + 1;
  return seed[type].toString(36) + type;
}


/**
 * @class File
 */

function File(manager, cfg){
  this.manager = manager;
  this.linkTo = [];
  this.linkBack = [];

  for (var key in cfg)
    this[key] = cfg[key];

  if (!this.type)
    this.type = manager.typeByExt[this.ext] || typeByExt[this.ext] || 'unknown';

  if (!this.filename)
  {
    var type = (this.generated ? 'generated-' : '') + this.type;
    manager.inline[type] = (manager.inline[type] || 0) + 1;
    this.inlineId = '[' + (this.inline ? 'inline ' : '') + type + '#' + manager.inline[type] + ']';
  }
};

File.prototype = {
  resolve: function(filename){
    if (isExternal(filename))
      return filename;

    // remove everything after ? (query string) or # (hash)
    filename = filename.replace(queryAndHashRx, '');

    return unixpath(path.resolve(this.baseURI, filename));
  },

  // input filename
  get dirname(){
    return this.filename
      ? path.dirname(this.filename)
      : '';
  },
  get basename(){
    return this.filename
      ? path.basename(this.filename)
      : '';
  },
  get name(){
    return this.filename
      ? path.basename(this.filename, path.extname(this.filename))
      : '';
  },
  get ext(){
    return this.filename
      ? path.extname(this.filename)
      : '';
  },
  get relpath(){
    return this.filename
      ? unixpath(path.relative(this.manager.baseURI, this.filename))
      : this.sourceFilename || this.inlineId;
  },
  get reldir(){
    return this.filename
      ? path.dirname(this.relpath)
      : this.sourceFilename || this.inlineId;
  },
  get fsFilename(){
    return this.filename
      ? this.manager.getFSFilename(this.filename)
      : this.sourceFilename || this.inlineId;
  },

  // input baseURI
  get baseURI(){
    if (!this.baseURI_)
      this.baseURI_ = unixpath(this.filename ? path.dirname(this.filename) + '/' : '');
    return this.baseURI_;
  },
  set baseURI(uri){
    if (!this.filename)
      this.baseURI_ = unixpath(path.resolve(this.manager.baseURI, uri) + '/');
  },

  // output filename
  get outputFilename(){
    return this.outputFilename_;
  },
  set outputFilename(filename){
    this.outputFilename_ = filename && unixpath(filename);
    this.inline = false;
  },
  get relOutputFilename(){
    return this.outputFilename || ('outputContent' in this ? this.inlineId : '[no output]');
  },

  // refs
  get jsRef(){
    if (!this.jsRef_)
      this.jsRef_ = this.manager.getRef(this.ext);
    return this.jsRef_;
  },
  set jsRef(ref){
    this.jsRef_ = ref;
  },

  // links
  link: function(file, token){
    var link = [file, token];
    token = token || null;

    this.linkTo.push(link);
    this.manager.links.push([this, file]);
    file.linkBack.push([this, token]);

    if (token)
    {
      if (!token.links)
        token.links = [];
      token.links.push(link);

      if (file.warn)
      {
        var loc = token.loc || '';
        var warn = {
          fatal: true,
          loc: loc,
          file: this.filename || this.relpath || loc.replace(/:\d+:\d+$/, ''),
          message: file.warn.message
        };

        if (!this.manager.flow.hasWarn(warn))
          this.manager.flow.warn(warn);
      }
    }
  },
  linkFrom: function(file){
    file.link(this);
  },
  unlink: function(file, token){
    token = token || null;

    for (var i = this.linkTo.length - 1; i >= 0; i--)
    {
      var link = this.linkTo[i];
      if (link[0] === file && link[1] === token)<commit_msg>remove redundant code that leads to infinite recursion on CSS minification<commit_after>var path = require('path');
var fs = require('fs');
var crypto = require('crypto');

var externalRx = /^(\s*[a-z0-9\-]+:)?\/\//i;
var queryAndHashRx = /[\?\#].*$/;
var slashRx = /\\/g;

var textFileExt = ['.css', '.js', '.json', '.tmpl', '.txt', '.svg', '.html'];
var textType = ['script', 'style', 'template', 'json', 'l10n', 'culture', 'xml', 'svg', 'text'];

var typeByExt = {
  '.js': 'script',
  '.css': 'style',
  '.tmpl': 'template',
  '.html': 'html',
  '.json': 'json',
  '.l10n': 'json',
  '.xml': 'xml',
  '.svg': 'svg',
  '.cur': 'image',
  '.ico': 'image',
  '.bmp': 'image',
  '.gif': 'image',
  '.png': 'image',
  '.jpg': 'image',
  '.jpeg': 'image',
  '.tiff': 'image',
  '.ttf': 'font',
  '.woff': 'font',
  '.eot': 'font'
};

var typeNotFoundHandler = {
  '.js': '/* Javascript file {filename} not found */',
  '.css': '/* CSS file {filename} not found */'
};

function getFileContentOnFailure(filename){
  return (typeNotFoundHandler[path.extname(filename)] || '').replace(/\{filename\}/, filename);
}

function unixpath(filename){
  return path.normalize(filename).replace(slashRx, '/').replace(/^[a-z]\:/i, '');
}

function abspath(baseURI, filename){
  return unixpath(path.resolve(baseURI, filename.replace(queryAndHashRx, '')).replace(/^[a-z]\:/i, ''));
}

function isExternal(uri){
  return externalRx.test(uri);
}

function getRef(seed, type){
  seed[type] = (type in seed ? seed[type] : -1) + 1;
  return seed[type].toString(36) + type;
}


/**
 * @class File
 */

function File(manager, cfg){
  this.manager = manager;
  this.linkTo = [];
  this.linkBack = [];

  for (var key in cfg)
    this[key] = cfg[key];

  if (!this.type)
    this.type = manager.typeByExt[this.ext] || typeByExt[this.ext] || 'unknown';

  if (!this.filename)
  {
    var type = (this.generated ? 'generated-' : '') + this.type;
    manager.inline[type] = (manager.inline[type] || 0) + 1;
    this.inlineId = '[' + (this.inline ? 'inline ' : '') + type + '#' + manager.inline[type] + ']';
  }
};

File.prototype = {
  resolve: function(filename){
    if (isExternal(filename))
      return filename;

    // remove everything after ? (query string) or # (hash)
    filename = filename.replace(queryAndHashRx, '');

    return unixpath(path.resolve(this.baseURI, filename));
  },

  // input filename
  get dirname(){
    return this.filename
      ? path.dirname(this.filename)
      : '';
  },
  get basename(){
    return this.filename
      ? path.basename(this.filename)
      : '';
  },
  get name(){
    return this.filename
      ? path.basename(this.filename, path.extname(this.filename))
      : '';
  },
  get ext(){
    return this.filename
      ? path.extname(this.filename)
      : '';
  },
  get relpath(){
    return this.filename
      ? unixpath(path.relative(this.manager.baseURI, this.filename))
      : this.sourceFilename || this.inlineId;
  },
  get reldir(){
    return this.filename
      ? path.dirname(this.relpath)
      : this.sourceFilename || this.inlineId;
  },
  get fsFilename(){
    return this.filename
      ? this.manager.getFSFilename(this.filename)
      : this.sourceFilename || this.inlineId;
  },

  // input baseURI
  get baseURI(){
    if (!this.baseURI_)
      this.baseURI_ = unixpath(this.filename ? path.dirname(this.filename) + '/' : '');
    return this.baseURI_;
  },
  set baseURI(uri){
    if (!this.filename)
      this.baseURI_ = unixpath(path.resolve(this.manager.baseURI, uri) + '/');
  },

  // output filename
  get outputFilename(){
    return this.outputFilename_;
  },
  set outputFilename(filename){
    this.outputFilename_ = filename && unixpath(filename);
    this.inline = false;
  },
  get relOutputFilename(){
    return this.outputFilename || ('outputContent' in this ? this.inlineId : '[no output]');
  },

  // refs
  get jsRef(){
    if (!this.jsRef_)
      this.jsRef_ = this.manager.getRef(this.ext);
    return this.jsRef_;
  },
  set jsRef(ref){
    this.jsRef_ = ref;
  },

  // links
  link: function(file, token){
    var link = [file, token];
    token = token || null;

    this.linkTo.push(link);
    this.manager.links.push([this, file]);
    file.linkBack.push([this, token]);

    if (token)
    {
      if (file.warn)
      {
        var loc = token.loc || '';
        var warn = {
          fatal: true,
          loc: loc,
          file: this.filename || this.relpath || loc.replace(/:\d+:\d+$/, ''),
          message: file.warn.message
        };

        if (!this.manager.flow.hasWarn(warn))
          this.manager.flow.warn(warn);
      }
    }
  },
  linkFrom: function(file){
    file.link(this);
  },
  unlink: function(file, token){
    token = token || null;

    for (var i = this.linkTo.length - 1; i >= 0; i--)
    {
      var link = this.linkTo[i];
      if (link[0] === file && link[1] === token)
{'fraction_non_alphanumeric': 0.1988984088127295, 'fraction_numerical': 0.0032639738882088943, 'mean_word_length': 2.8420846394984327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '69946', 'n_tokens_mistral': 3437, 'n_tokens_neox': 3363, 'n_words': 703}