<commit_before>/**
 * @license
 * Copyright (c) 2014 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
 */
var dom5 = require('dom5')
var escodegen = require('escodegen');
var espree = require('espree');
var through2 = require('through2');
var uglify = require('uglify-js');

/**
 * Transforms all inline scripts in `html` with `filter`.
 */
function filterInlineScript(html, filter) {
  var p = dom5.predicates;
  var parsedHtml = dom5.parse(html);
  var isInlineScript = p.AND(
    p.hasTagName('script'),
    p.NOT(p.hasAttr('src'))
  );
  dom5.queryAll(parsedHtml, isInlineScript)
    .forEach(function(inlineScript) {
      var text = dom5.getTextContent(inlineScript);
      dom5.setTextContent(inlineScript,filter(text));
    });
  return dom5.serialize(parsedHtml);
}

var exports = {
  /**
   * Gulp plugin that removes comments with a parser roundtrip.
   */
  cleanJsComments: function cleanJsComments() {
    return through2.obj(function(file, encoding, cb) {
      var cleaned = filterInlineScript(String(file.contents), function(text) {
          content = espree.parse(text, {attachComment: false, comments: false});
          return escodegen.generate(content);
      });
      file.contents = new Buffer(cleaned);
      this.push(file);
      return cb();
    });
  },

  /**
   * Uglify the stream.
   */
  uglifyJs: function uglifyJs() {
    return through2.obj(function(file, encoding, cb) {
      var cleaned = filterInlineScript(String(file.contents), function(text) {
          return uglify.minify(text, {fromString: true}).code;
      });
      file.contents = new Buffer(cleaned);
      this.push(file);
      return cb();
    });
  }
};

module.exports = exports;<commit_msg>Add a leftAlignJs function<commit_after>/**
 * @license
 * Copyright (c) 2014 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
 */
var dom5 = require('dom5')
var escodegen = require('escodegen');
var espree = require('espree');
var through2 = require('through2');
var uglify = require('uglify-js');

/**
 * Transforms all inline scripts in `html` with `filter`.
 */
function filterInlineScript(html, filter) {
  var p = dom5.predicates;
  var parsedHtml = dom5.parse(html);
  var isInlineScript = p.AND(
    p.hasTagName('script'),
    p.NOT(p.hasAttr('src'))
  );
  dom5.queryAll(parsedHtml, isInlineScript)
    .forEach(function(inlineScript) {
      var text = dom5.getTextContent(inlineScript);
      dom5.setTextContent(inlineScript,filter(text));
    });
  return dom5.serialize(parsedHtml);
}

var exports = {
  /**
   * Gulp plugin that removes comments with a parser roundtrip.
   */
  cleanJsComments: function cleanJsComments() {
    return through2.obj(function(file, encoding, cb) {
      var cleaned = filterInlineScript(String(file.contents), function(text) {
          content = espree.parse(text, {attachComment: false, comments: false});
          return escodegen.generate(content);
      });
      file.contents = new Buffer(cleaned);
      this.push(file);
      return cb();
    });
  },

  /**
   * Left align all the javascript
   * Leave newlines, as they can be useful for debugging
   */
  leftAlignJs: function leftAlignJs() {
    return through2.obj(function(file, encoding, cb) {
      var cleaned = filterInlineScript(String(file.contents), function(text) {
        return uglify.minify(text, {
          fromString: true,
          mangle: false,
          compress: false,
          output: {
            beautify: true,
            indent_level: 0
          }
        }).code;
      });
      file.contents = new Buffer(cleaned);
      this.push(file);
      return cb();
    });
  },

  /**
   * Uglify the stream.
   */
  uglifyJs: function uglifyJs() {
    return through2.obj(function(file, encoding, cb) {
      var cleaned = filterInlineScript(String(file.contents), function(text) {
          return uglify.minify(text, {fromString: true}).code;
      });
      file.contents = new Buffer(cleaned);
      this.push(file);
      return cb();
    });
  }
};

module.exports = exports;
{'fraction_non_alphanumeric': 0.12510187449062754, 'fraction_numerical': 0.0069274653626731865, 'mean_word_length': 3.2246127366609296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1703892', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1455, 'n_words': 399}