<commit_before>/*
 * Copyright 2013 The Polymer Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style
 * license that can be found in the LICENSE file.
 */

var async = require('async');
var cheerio = require('cheerio');
var cleancss = require('clean-css');
var fs = require('fs');
var path = require('path');
var uglify = require('uglify-js');
var url = require('url');

var constants = require('./constants.js');
var optparser = require('./optparser.js');
var pathresolver = require('./pathresolver');
var utils = require('./utils');

var read = {};
var options = {};

// validate options with boolean return
function setOptions(optHash, callback) {
  optparser.processOptions(optHash, function(err, o) {
    if (err) {
      return callback(err);
    }
    options = o;
    callback();
  });
}

function exclude(regexes, href) {
  return regexes.some(function(r) {
    return r.test(href);
  });
}

function excludeImport(href) {
  return exclude(options.excludes.imports, href);
}

function excludeScript(href) {
  return exclude(options.excludes.scripts, href);
}

function excludeStyle(href) {
  return exclude(options.excludes.styles, href);
}

function setTextContent(node, text) {
  utils.setTextContent(node, text);
}

function readFile(file) {
  var content = fs.readFileSync(file, 'utf8');
  return content.replace(/^\uFEFF/, '');
}

// inline relative linked stylesheets into <style> tags
function inlineSheets($, inputPath, outputPath) {
  $('link[rel="stylesheet"]').each(function() {
    var href = this.attr('href');
    if (href && !excludeStyle(href)) {
      var filepath = path.resolve(options.outputDir, href);
      // fix up paths in the stylesheet to be relative to the location of the style
      var content = pathresolver.rewriteURL(path.dirname(filepath), inputPath, readFile(filepath));
      var styleDoc = cheerio.load('<style>' + content + '</style>');
      // clone attributes
      styleDoc('style').attr(this.attr());
      // don't set href or rel on the <style>
      styleDoc('style').attr('href', null);
      styleDoc('style').attr('rel', null);
      this.replaceWith(styleDoc.html());
    }
  });
}

function inlineScripts($, dir) {
  $(constants.JS_SRC).each(function() {
    var src = this.attr('src');
    if (src && !excludeScript(src)) {
      var filepath = path.resolve(dir, src);
      var content = readFile(filepath);
      // NOTE: reusing UglifyJS's inline script printer (not exported from OutputStream :/)
      content = content.replace(/<\x2fscript([>\/\t\n\f\r ])/gi, "<\\/script$1");
      this.replaceWith('<script>' + content + '</script>');
    }
  });
}

function readDocument(filename) {
  return cheerio.load(readFile(filename));
}

function concat(filename) {
  if (!read[filename]) {
    read[filename] = true;
    var $ = readDocument(filename);
    var dir = path.dirname(filename);
    pathresolver.resolvePaths($, dir, options.outputDir);
    processImports($);
    inlineSheets($, dir, options.outputDir);
    return $.html();
  } else {
    if (options.verbose) {
      console.log('Dependency deduplicated');
    }
  }
}

function processImports($) {
  $(constants.IMPORTS).each(function() {
    var href = this.attr('href');
    if (!excludeImport(href)) {
      this.replaceWith(concat(path.resolve(options.outputDir, href)));
    }
  });
}

function findScriptLocation($) {
  var pos = $('body').last();
  if (!pos.length) {
    pos = $.root();
  }
  return pos;
}

// arguments are (index, node), where index is unnecessary
function isCommentOrEmptyTextNode(_, node) {
  if (node.type === 'comment'){
    return true;
  } else if (node.type === 'text') {
    // return true if the node is only whitespace
    return !((/\S/).test(node.data));
  }
}

function removeCommentsAndWhitespace($) {
  $(constants.JS_INLINE).each(function() {
    var content = this.html();
    var ast = uglify.parse(content);
    this.replaceWith('<script>' + ast.print_to_string({inline_script: true}) + '</script>');
  });
  $(constants.CSS).each(function() {
    setTextContent(this, new cleancss({noAdvanced: true}).minify(this.text()));
  });
  $('head').contents().filter(isCommentOrEmptyTextNode).remove();
  $('body').contents().filter(isCommentOrEmptyTextNode).remove();
  $.root().contents().filter(isCommentOrEmptyTextNode).remove();
}

function handleMainDocument() {
  // reset shared buffers
  read = {};
  var $ = readDocument(options.input);
  var dir = path.dirname(options.input);
  pathresolver.resolvePaths($, dir, options.outputDir);
  processImports($, dir);
  if (options.inline) {
    inlineSheets($, dir, options.outputDir);
  }

  // strip scripts into a separate file
  if (options.csp) {
    if (options.verbose) {
      console.log('Separating scripts into separate file');
    }

    // CSPify main page by removing inline scripts
    var scripts = [];
    $(constants.JS_INLINE).each(function() {
      scripts.push(this.text());
      this.remove();
    });

    // join scripts with ';' to prevent breakages due to EOF semicolon insertion
    var script_name = path.basename(options.output, '.html') + '.js';
    fs.writeFileSync(path.resolve(options.outputDir, script_name), scripts.join(';' + constants.EOL), 'utf8');
  }

  deduplicateImports($);
  if (!options.csp && options.inline) {
    inlineScripts($, options.outputDir);
  }
  if (options.strip) {
    removeCommentsAndWhitespace($);
  }
  var outhtml = $.html();
  fs.writeFileSync(options.output, outhtml, 'utf8');
}

function deduplicateImports($) {
  var imports = {};
  $(constants.IMPORTS).each(function() {
    var href = this.attr('href');
    // TODO(dfreedm): allow a user defined base url?
    var abs = url.resolve('http://', href);
    if (!imports[abs]) {
      imports[abs] = true;
    } else {
      if(options.verbose) {
        console.log('Import Dependency deduplicated');
      }
      this.remove();
    }
  });
}
<commit_msg>Fix #29<commit_after>/*
 * Copyright 2013 The Polymer Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style
 * license that can be found in the LICENSE file.
 */

var async = require('async');
var cheerio = require('cheerio');
var cleancss = require('clean-css');
var fs = require('fs');
var path = require('path');
var uglify = require('uglify-js');
var url = require('url');

var constants = require('./constants.js');
var optparser = require('./optparser.js');
var pathresolver = require('./pathresolver');
var utils = require('./utils');

var read = {};
var options = {};

// validate options with boolean return
function setOptions(optHash, callback) {
  optparser.processOptions(optHash, function(err, o) {
    if (err) {
      return callback(err);
    }
    options = o;
    callback();
  });
}

function exclude(regexes, href) {
  return regexes.some(function(r) {
    return r.test(href);
  });
}

function excludeImport(href) {
  return exclude(options.excludes.imports, href);
}

function excludeScript(href) {
  return exclude(options.excludes.scripts, href);
}

function excludeStyle(href) {
  return exclude(options.excludes.styles, href);
}

function setTextContent(node, text) {
  utils.setTextContent(node, text);
}

function readFile(file) {
  var content = fs.readFileSync(file, 'utf8');
  return content.replace(/^\uFEFF/, '');
}

// inline relative linked stylesheets into <style> tags
function inlineSheets($, inputPath, outputPath) {
  $('link[rel="stylesheet"]').each(function() {
    var href = this.attr('href');
    if (href && !excludeStyle(href)) {
      var filepath = path.resolve(options.outputDir, href);
      // fix up paths in the stylesheet to be relative to the location of the style
      var content = pathresolver.rewriteURL(path.dirname(filepath), inputPath, readFile(filepath));
      var styleDoc = cheerio.load('<style>' + content + '</style>');
      // clone attributes
      styleDoc('style').attr(this.attr());
      // don't set href or rel on the <style>
      styleDoc('style').attr('href', null);
      styleDoc('style').attr('rel', null);
      this.replaceWith(styleDoc.html());
    }
  });
}

function inlineScripts($, dir) {
  $(constants.JS_SRC).each(function() {
    var src = this.attr('src');
    if (src && !excludeScript(src)) {
      var filepath = path.resolve(dir, src);
      var content = readFile(filepath);
      // NOTE: reusing UglifyJS's inline script printer (not exported from OutputStream :/)
      content = content.replace(/<\x2fscript([>\/\t\n\f\r ])/gi, "<\\/script$1");
      this.replaceWith('<script>' + content + '</script>');
    }
  });
}

function readDocument(filename) {
  return cheerio.load(readFile(filename));
}

function concat(filename) {
  if (!read[filename]) {
    read[filename] = true;
    var $ = readDocument(filename);
    var dir = path.dirname(filename);
    pathresolver.resolvePaths($, dir, options.outputDir);
    processImports($);
    inlineSheets($, dir, options.outputDir);
    return $.html();
  } else {
    if (options.verbose) {
      console.log('Dependency deduplicated');
    }
  }
}

function processImports($) {
  $(constants.IMPORTS).each(function() {
    var href = this.attr('href');
    if (!excludeImport(href)) {
      this.replaceWith(concat(path.resolve(options.outputDir, href)));
    }
  });
}

function findScriptLocation($) {
  var pos = $('body').last();
  if (!pos.length) {
    pos = $.root();
  }
  return pos;
}

// arguments are (index, node), where index is unnecessary
function isCommentOrEmptyTextNode(_, node) {
  if (node.type === 'comment'){
    return true;
  } else if (node.type === 'text') {
    // return true if the node is only whitespace
    return !((/\S/).test(node.data));
  }
}

function removeCommentsAndWhitespace($) {
  $(constants.JS_INLINE).each(function() {
    var content = this.html();
    var ast = uglify.parse(content);
    this.replaceWith('<script>' + ast.print_to_string({inline_script: true}) + '</script>');
  });
  $(constants.CSS).each(function() {
    setTextContent(this, new cleancss({noAdvanced: true}).minify(this.text()));
  });
  $('head').contents().filter(isCommentOrEmptyTextNode).remove();
  $('body').contents().filter(isCommentOrEmptyTextNode).remove();
  $.root().contents().filter(isCommentOrEmptyTextNode).remove();
}

function handleMainDocument() {
  // reset shared buffers
  read = {};
  var $ = readDocument(options.input);
  var dir = path.dirname(options.input);
  pathresolver.resolvePaths($, dir, options.outputDir);
  processImports($, dir);
  if (options.inline) {
    inlineSheets($, dir, options.outputDir);
  }

  // strip scripts into a separate file
  if (options.csp) {
    if (options.verbose) {
      console.log('Separating scripts into separate file');
    }

    // CSPify main page by removing inline scripts
    var scripts = [];
    $(constants.JS_INLINE).each(function() {
      scripts.push(this.text());
      this.remove();
    });

    // join scripts with ';' to prevent breakages due to EOF semicolon insertion
    var script_name = path.basename(options.output, '.html') + '.js';
    fs.writeFileSync(path.resolve(options.outputDir, script_name), scripts.join(';' + constants.EOL), 'utf8');
    // insert out-of-lined script into document
    findScriptLocation($).append('<script src="' + script_name + '"></script>');
  }

  deduplicateImports($);
  if (!options.csp && options.inline) {
    inlineScripts($, options.outputDir);
  }
  if (options.strip) {
    removeCommentsAndWhitespace($);
  }
  var outhtml = $.html();
  fs.writeFileSync(options.output, outhtml, 'utf8');
}

function deduplicateImports($) {
  var imports = {};
  $(constants.IMPORTS).each(function() {
    var href = this.attr('href');
    // TODO(dfreedm): allow a user defined base url?
    var abs = url.resolve('http://', href);
    if (!imports[abs]) {
      imports[abs] = true;
    } else {
      if(options.verbose) {
        console.log('Import Dependency deduplicated');
      }
      this.remove();
    }
  });
}

{'fraction_non_alphanumeric': 0.16759869793840246, 'fraction_numerical': 0.0016693097404223353, 'mean_word_length': 3.676814988290398, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916628', 'n_tokens_mistral': 3783, 'n_tokens_neox': 3732, 'n_words': 868}