<commit_before>var saucy = require('./sourcemaps');
var path = require('path');
var mkdirp = require('mkdirp');
var fs = require('fs');
var asp = require('rsvp').denodeify;
var uglify = require('uglify-js');

function countLines(str) {
  return str.split(/\r\n|\r|\n/).length;
}

// Process compiler outputs, gathering:
//
//   concatOutputs:         list of source strings to concatenate
//   sourceMapsWithOffsets: list of [absolute offset,
//                                   source map string] pairs
//
//  Takes lists as empty references and populates via push.
function processOutputs(outputs) {
  var offset = 0;

  var outputObj = {};

  var sources = outputObj.sources = [];
  var sourceMapsWithOffsets = outputObj.sourceMapsWithOffsets = [];

  if (!outputs)
    return outputObj;

  outputs.forEach(function(output) {
    var source;
    if (typeof output == 'object') {
      source = output.source || '';
      var offset_ = output.sourceMapOffset || 0;
      var map = output.sourceMap;
      if (map) {
        sourceMapsWithOffsets.push([offset + offset_, map]);
      }
    } 
    else if (typeof output == 'string') {
      source = output;
    }
    else {
      throw "Unexpected output format: " + output.toString();
    }
    source = saucy.removeSourceMaps(source || '');
    offset += countLines(source);
    sources.push(source);
  });

  return outputObj;
}

function createOutput(outputs, outFile, createSourceMaps) {
  var outPath = path.dirname(outFile);

  // process output
  var sourceMap;

  var outputObj = processOutputs(outputs);

  if (createSourceMaps && outputObj.sourceMapsWithOffsets.length)
    sourceMap = saucy.concatenateSourceMaps(outFile, outputObj.sourceMapsWithOffsets, 
        'file:' + path.resolve(outPath));
  
  var output = outputObj.sources.join('\n');

  return {
    source: output,
    sourceMap: sourceMap
  };
}

function minify(output, outFile) {
  var fileName = path.basename(outFile);

  var ast = uglify.parse(output.source, { filename: fileName });
  ast.figure_out_scope();
  ast = ast.transform(uglify.Compressor({ warnings: false }));
  ast.figure_out_scope();
  ast.compute_char_frequency();
  ast.mangle_names({
    except: ['require']
  });

  var sourceMap;
  if (output.sourceMap)
    sourceMap = uglify.SourceMap({
      file: fileName,
      orig: output.sourceMap
    });

  output.source = ast.print_to_string({
    // for some reason non-ascii broke esprima.... this does break unicode though
    ascii_only: true,
    // keep first comment
    comments: function(node, comment) {
      return comment.line == 1 && comment.col == 0;
    },
    source_map: sourceMap
  });
  output.sourceMap = sourceMap.toString();

  return output;
}


exports.writeOutputFile = function(opts, outputs) {
  var output = createOutput(outputs, opts.outFile, opts.sourceMaps);

  if (opts.minify)
    output = minify(output, opts.outFile);

  if (opts.sourceMaps) {
    var sourceMapFile = path.basename(opts.outFile) + '.map';
    output.source += '//# sourceMappingURL=' + sourceMapFile;
  }

  return asp(mkdirp)(path.dirname(opts.outFile))
  .then(function() {
    if (!opts.sourceMaps)
      return;
    return asp(fs.writeFile)(path.resolve(path.dirname(opts.outFile), sourceMapFile), output.sourceMap);
  })
  .then(function() {
    return asp(fs.writeFile)(opts.outFile, output.source);<commit_msg>Put sourcemap comment on new line at EOF<commit_after>var saucy = require('./sourcemaps');
var path = require('path');
var mkdirp = require('mkdirp');
var fs = require('fs');
var asp = require('rsvp').denodeify;
var uglify = require('uglify-js');

function countLines(str) {
  return str.split(/\r\n|\r|\n/).length;
}

// Process compiler outputs, gathering:
//
//   concatOutputs:         list of source strings to concatenate
//   sourceMapsWithOffsets: list of [absolute offset,
//                                   source map string] pairs
//
//  Takes lists as empty references and populates via push.
function processOutputs(outputs) {
  var offset = 0;

  var outputObj = {};

  var sources = outputObj.sources = [];
  var sourceMapsWithOffsets = outputObj.sourceMapsWithOffsets = [];

  if (!outputs)
    return outputObj;

  outputs.forEach(function(output) {
    var source;
    if (typeof output == 'object') {
      source = output.source || '';
      var offset_ = output.sourceMapOffset || 0;
      var map = output.sourceMap;
      if (map) {
        sourceMapsWithOffsets.push([offset + offset_, map]);
      }
    } 
    else if (typeof output == 'string') {
      source = output;
    }
    else {
      throw "Unexpected output format: " + output.toString();
    }
    source = saucy.removeSourceMaps(source || '');
    offset += countLines(source);
    sources.push(source);
  });

  return outputObj;
}

function createOutput(outputs, outFile, createSourceMaps) {
  var outPath = path.dirname(outFile);

  // process output
  var sourceMap;

  var outputObj = processOutputs(outputs);

  if (createSourceMaps && outputObj.sourceMapsWithOffsets.length)
    sourceMap = saucy.concatenateSourceMaps(outFile, outputObj.sourceMapsWithOffsets, 
        'file:' + path.resolve(outPath));
  
  var output = outputObj.sources.join('\n');

  return {
    source: output,
    sourceMap: sourceMap
  };
}

function minify(output, outFile) {
  var fileName = path.basename(outFile);

  var ast = uglify.parse(output.source, { filename: fileName });
  ast.figure_out_scope();
  ast = ast.transform(uglify.Compressor({ warnings: false }));
  ast.figure_out_scope();
  ast.compute_char_frequency();
  ast.mangle_names({
    except: ['require']
  });

  var sourceMap;
  if (output.sourceMap)
    sourceMap = uglify.SourceMap({
      file: fileName,
      orig: output.sourceMap
    });

  output.source = ast.print_to_string({
    // for some reason non-ascii broke esprima.... this does break unicode though
    ascii_only: true,
    // keep first comment
    comments: function(node, comment) {
      return comment.line == 1 && comment.col == 0;
    },
    source_map: sourceMap
  });
  output.sourceMap = sourceMap.toString();

  return output;
}


exports.writeOutputFile = function(opts, outputs) {
  var output = createOutput(outputs, opts.outFile, opts.sourceMaps);

  if (opts.minify)
    output = minify(output, opts.outFile);

  if (opts.sourceMaps) {
    var sourceMapFile = path.basename(opts.outFile) + '.map';
    output.source += '\n//# sourceMappingURL=' + sourceMapFile;
  }

  return asp(mkdirp)(path.dirname(opts.outFile))
  .then(function() {
    if (!opts.sourceMaps)
      return;
    return asp(fs.writeFile)(path.resolve(path.dirname(opts.outFile), sourceMapFile), output.sourceMap);
  })
  .then(function() {
    return asp(fs.writeFile)(opts.outFile, output.source);
{'fraction_non_alphanumeric': 0.13491946209546327, 'fraction_numerical': 0.0011822077730161075, 'mean_word_length': 3.286257124762508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4175443', 'n_tokens_mistral': 2159, 'n_tokens_neox': 2102, 'n_words': 483}