<commit_before>let convertSourceMap = require("convert-source-map");
let pathExists       = require("path-exists");
let sourceMap        = require("source-map");
let chokidar         = require("chokidar");
let slash            = require("slash");
let path             = require("path");
let util             = require("./util");
let fs               = require("fs");
let _                = require("lodash");

module.exports = function (commander, filenames, opts) {
  if (commander.sourceMaps === "inline") {
    opts.sourceMaps = true;
  }

  let results = [];

  let buildResult = function () {
    let map = new sourceMap.SourceMapGenerator({
      file: path.basename(commander.outFile) || "stdout",
      sourceRoot: opts.sourceRoot
    });

    let code = "";
    let offset = 0;

    _.each(results, function (result) {
      let filename = result.filename || "stdout";
      code += result.code + "\n";

      if (result.map) {
        let consumer = new sourceMap.SourceMapConsumer(result.map);

        let sourceFilename = filename;
        if (commander.outFile) {
          sourceFilename = path.relative(path.dirname(commander.outFile), sourceFilename);
        }
        sourceFilename = slash(sourceFilename);

        map._sources.add(sourceFilename);
        map.setSourceContent(sourceFilename, result.actual);

        consumer.eachMapping(function (mapping) {
          map._mappings.add({
            generatedLine: mapping.generatedLine + offset,
            generatedColumn: mapping.generatedColumn,
            originalLine: mapping.originalLine,
            originalColumn: mapping.originalColumn,
            source: sourceFilename
          });
        });

        offset = code.split("\n").length;
      }
    });

    // add the inline sourcemap comment if we've either explicitly asked for inline source
    // maps, or we've requested them without any output file
    if (commander.sourceMaps === "inline" || (!commander.outFile && commander.sourceMaps)) {
      code += "\n" + convertSourceMap.fromObject(map).toComment();
    }

    return {
      map: map,
      code: code
    };
  };

  let output = function () {
    let result = buildResult();

    if (commander.outFile) {
      // we've requested for a sorucemap to be written to disk
      if (commander.sourceMaps && commander.sourceMaps !== "inline") {
        let mapLoc = commander.outFile + ".map";
        result.code = util.addSourceMappingUrl(result.code, mapLoc);
        fs.writeFileSync(mapLoc, JSON.stringify(result.map));
      }

      fs.writeFileSync(commander.outFile, result.code);
    } else {
      process.stdout.write(result.code + "\n");
    }
  };

  let stdin = function () {
    let code = "";

    process.stdin.setEncoding("utf8");

    process.stdin.on("readable", function () {
      let chunk = process.stdin.read();
      if (chunk !== null) code += chunk;
    });

    process.stdin.on("end", function () {
      results.push(util.transform(commander.filename, code));
      output();
    });
  };

  let walk = function () {
    let _filenames = [];
    results = [];

    _.each(filenames, function (filename) {
      if (!pathExists.sync(filename)) return;

      let stat = fs.statSync(filename);
      if (stat.isDirectory()) {
        let dirname = filename;

        _.each(util.readdirFilter(filename), function (filename) {
          _filenames.push(path.join(dirname, filename));
        });
      } else {
        _filenames.push(filename);
      }
    });

    _.each(_filenames, function (filename) {
      if (util.shouldIgnore(filename)) return;

      let data = util.compile(filename);
      if (data.ignored) return;
      results.push(data);
    });

    output();
  };

  let files = function () {
    walk();

    if (commander.watch) {
      chokidar.watch(filenames, {
        persistent: true,
        ignoreInitial: true
      }).on("all", function (type, filename) {<commit_msg>Make babel-cli watching only given files and extensions. Use --only, --ignore and --extensions while process filenames with --out-file option.<commit_after>let convertSourceMap = require("convert-source-map");
let pathExists       = require("path-exists");
let sourceMap        = require("source-map");
let chokidar         = require("chokidar");
let slash            = require("slash");
let path             = require("path");
let util             = require("./util");
let fs               = require("fs");
let _                = require("lodash");

module.exports = function (commander, filenames, opts) {
  if (commander.sourceMaps === "inline") {
    opts.sourceMaps = true;
  }

  let results = [];

  let buildResult = function () {
    let map = new sourceMap.SourceMapGenerator({
      file: path.basename(commander.outFile) || "stdout",
      sourceRoot: opts.sourceRoot
    });

    let code = "";
    let offset = 0;

    _.each(results, function (result) {
      let filename = result.filename || "stdout";
      code += result.code + "\n";

      if (result.map) {
        let consumer = new sourceMap.SourceMapConsumer(result.map);

        let sourceFilename = filename;
        if (commander.outFile) {
          sourceFilename = path.relative(path.dirname(commander.outFile), sourceFilename);
        }
        sourceFilename = slash(sourceFilename);

        map._sources.add(sourceFilename);
        map.setSourceContent(sourceFilename, result.actual);

        consumer.eachMapping(function (mapping) {
          map._mappings.add({
            generatedLine: mapping.generatedLine + offset,
            generatedColumn: mapping.generatedColumn,
            originalLine: mapping.originalLine,
            originalColumn: mapping.originalColumn,
            source: sourceFilename
          });
        });

        offset = code.split("\n").length;
      }
    });

    // add the inline sourcemap comment if we've either explicitly asked for inline source
    // maps, or we've requested them without any output file
    if (commander.sourceMaps === "inline" || (!commander.outFile && commander.sourceMaps)) {
      code += "\n" + convertSourceMap.fromObject(map).toComment();
    }

    return {
      map: map,
      code: code
    };
  };

  let output = function () {
    let result = buildResult();

    if (commander.outFile) {
      // we've requested for a sorucemap to be written to disk
      if (commander.sourceMaps && commander.sourceMaps !== "inline") {
        let mapLoc = commander.outFile + ".map";
        result.code = util.addSourceMappingUrl(result.code, mapLoc);
        fs.writeFileSync(mapLoc, JSON.stringify(result.map));
      }

      fs.writeFileSync(commander.outFile, result.code);
    } else {
      process.stdout.write(result.code + "\n");
    }
  };

  let stdin = function () {
    let code = "";

    process.stdin.setEncoding("utf8");

    process.stdin.on("readable", function () {
      let chunk = process.stdin.read();
      if (chunk !== null) code += chunk;
    });

    process.stdin.on("end", function () {
      results.push(util.transform(commander.filename, code));
      output();
    });
  };

  let walk = function () {
    let _filenames = [];
    results = [];

    _.each(filenames, function (filename) {
      if (!pathExists.sync(filename)) return;

      let stat = fs.statSync(filename);
      if (stat.isDirectory()) {
        let dirname = filename;

        _.each(util.readdirFilter(filename), function (filename) {
          _filenames.push(path.join(dirname, filename));
        });
      } else {
        _filenames.push(filename);
      }
    });

    _.each(_filenames, function (filename) {
      if (util.shouldIgnore(filename)) return;

      let data = util.compile(filename);
      if (data.ignored) return;
      results.push(data);
    });

    output();
  };

  let files = function () {
    walk();

    if (commander.watch) {
      chokidar.watch(filenames, {
        persistent: true,
        ignoreInitial: true
      }).on("all", function (type, filename) {
        if (util.shouldIgnore(filename) || !util.canCompile(filename, commander.extensions)) return;

{'fraction_non_alphanumeric': 0.13653774867169158, 'fraction_numerical': 0.0004942542938341777, 'mean_word_length': 2.439864003399915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5307937', 'n_tokens_mistral': 2344, 'n_tokens_neox': 2272, 'n_words': 519}