<commit_before>var sourceMapToComment = require("source-map-to-comment");
var sourceMap          = require("source-map");
var chokidar           = require("chokidar");
var path               = require("path");
var util               = require("./util");
var fs                 = require("fs");
var _                  = require("lodash");

module.exports = function (commander, filenames) {
  var results = [];

  var buildResult = function () {
    var map = new sourceMap.SourceMapGenerator({
      file: commander.outFile || "stdout"
    });

    var code = "";
    var offset = 0;

    _.each(results, function (result) {
      var filename = result.filename;
      code += result.code + "\n";

      if (result.map) {
        var consumer = new sourceMap.SourceMapConsumer(result.map);

        map._sources.add(filename);
        map.setSourceContent(filename, result.actual);

        consumer.eachMapping(function (mapping) {
          map._mappings.add({
            generatedLine: mapping.generatedLine + offset,
            generatedColumn: mapping.generatedColumn,
            originalLine: mapping.originalLine,
            originalColumn: mapping.originalColumn,
            source: filename
          });
        });

        offset = code.split("\n").length;
      }
    });

    if (commander.sourceMapsInline || (!commander.outFile && commander.sourceMaps)) {
      code += "\n" + sourceMapToComment(map);
    }

    return {
      map: map,
      code: code
    };
  };

  var output = function () {
    var result = buildResult();

    if (commander.outFile) {
      if (commander.sourceMaps) {
        var mapLoc = commander.outFile + ".map";
        result.code = util.addSourceMappingUrl(result.code, mapLoc);
        fs.writeFileSync(mapLoc, JSON.stringify(result.map));
      }
<commit_msg>use convert-source-map package - closes #960<commit_after>var convertSourceMap = require("convert-source-map");
var sourceMap        = require("source-map");
var chokidar         = require("chokidar");
var path             = require("path");
var util             = require("./util");
var fs               = require("fs");
var _                = require("lodash");

module.exports = function (commander, filenames) {
  var results = [];

  var buildResult = function () {
    var map = new sourceMap.SourceMapGenerator({
      file: commander.outFile || "stdout"
    });

    var code = "";
    var offset = 0;

    _.each(results, function (result) {
      var filename = result.filename;
      code += result.code + "\n";

      if (result.map) {
        var consumer = new sourceMap.SourceMapConsumer(result.map);

        map._sources.add(filename);
        map.setSourceContent(filename, result.actual);

        consumer.eachMapping(function (mapping) {
          map._mappings.add({
            generatedLine: mapping.generatedLine + offset,
            generatedColumn: mapping.generatedColumn,
            originalLine: mapping.originalLine,
            originalColumn: mapping.originalColumn,
            source: filename
          });
        });

        offset = code.split("\n").length;
      }
    });

    if (commander.sourceMapsInline || (!commander.outFile && commander.sourceMaps)) {
      code += "\n" + convertSourceMap.fromObject(map).toComment();
    }

    return {
      map: map,
      code: code
    };
  };

  var output = function () {
    var result = buildResult();

    if (commander.outFile) {
      if (commander.sourceMaps) {
        var mapLoc = commander.outFile + ".map";
        result.code = util.addSourceMappingUrl(result.code, mapLoc);
        fs.writeFileSync(mapLoc, JSON.stringify(result.map));
      }

{'fraction_non_alphanumeric': 0.13076713076713076, 'fraction_numerical': 0.001365001365001365, 'mean_word_length': 2.2597864768683276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5349358', 'n_tokens_mistral': 1036, 'n_tokens_neox': 1006, 'n_words': 216}