<commit_before>#!/usr/src/app/bin/node_gc

var CODE_DIR = "/code";
process.chdir(CODE_DIR);

// Redirect `console.log` so that we are the only ones
// writing to STDOUT
var stdout = console.log;
console.log = console.error;

var CLIEngine = require("eslint").CLIEngine;
var docs = require("eslint").docs;
var fs = require("fs");
var glob = require("glob");
var options = { extensions: [".js"], ignore: true, reset: false, useEslintrc: true };
var cli = new CLIEngine(options);
var debug = false;
var checks = require("../lib/checks");
var validateConfig = require("../lib/validate_config");

// a wrapper for emitting perf timing
function runWithTiming(name, fn) {
  var start = new Date()
    , rv = fn()
    , duration = (new Date() - start) / 1000;
  if (debug) {
    console.error("eslint.timing." + name + ": " + duration + "s");
  }
  return rv;
}

function contentBody(check) {
  var content = docs.get(check) || "For more information visit ";
  return content + "Source: http://eslint.org/docs/rules/\n";
}

function buildIssueJson(message, path) {
  // ESLint doesn't emit a ruleId in the
  // case of a fatal error (such as an invalid
  // token)
  var checkName = message.ruleId;
  if(message.fatal) {
    checkName = "fatal";
  }

  var issue = {
    type: "issue",
    categories: checks.categories(checkName),
    check_name: checkName,
    description: message.message,
    content: {
      body: contentBody(checkName)
    },
    location: {
      path: path,
      positions: {
        begin: {
          line: message.line || 1,
          column: message.column || 1
        },
        end: {
          line: message.line || 1,
          column: message.column || 1
        }
      }
    },
    remediation_points: checks.remediationPoints(checkName, message, cli.getConfigForFile(path))
  };
  return JSON.stringify(issue);
}

function isFileWithMatchingExtension(file, extensions) {
  var stats = fs.lstatSync(file);
  var extension = "." + file.split(".").pop();
  return (
    stats.isFile() &&
    !stats.isSymbolicLink()
    && extensions.indexOf(extension) >= 0
  );
}

function isFileIgnoredByLibrary(file) {
  var ignored = cli.isPathIgnored(file);
  if (ignored) {
    var output = "File `" + file + "` ignored because of your .eslintignore file." + "\n";
    process.stderr.write(output);
  }
  return ignored;
}

function prunePathsWithinSymlinks(paths) {
  // Extracts symlinked paths and filters them out, including any child paths
  var symlinks = paths.filter(function(path) {
    return fs.lstatSync(path).isSymbolicLink();
  });

  return paths.filter(function(path) {
    var withinSymlink = false;
    symlinks.forEach(function(symlink) {
      if (path.indexOf(symlink) === 0) {
        withinSymlink = true;
      }
    });
    return !withinSymlink;
  });
}

function inclusionBasedFileListBuilder(includePaths) {
  // Uses glob to expand the files and directories in includePaths, filtering
  // down to match the list of desired extensions.
  return function(extensions) {
    var analysisFiles = [];

    includePaths.forEach(function(fileOrDirectory, i) {
      if ((/\/$/).test(fileOrDirectory)) {
        // if it ends in a slash, expand and push
        var filesInThisDirectory = glob.sync(
          fileOrDirectory + "/**/**"
        );
        prunePathsWithinSymlinks(filesInThisDirectory).forEach(function(file, j){
          if (!isFileIgnoredByLibrary(file) && isFileWithMatchingExtension(file, extensions)) {
            analysisFiles.push(file);
          }
        });
      } else {
        if (!isFileIgnoredByLibrary(fileOrDirectory) && isFileWithMatchingExtension(fileOrDirectory, extensions)) {
          analysisFiles.push(fileOrDirectory);
        }
      }
    });

    return analysisFiles;
  };
}

var buildFileList;
runWithTiming("engineConfig", function () {
  if (fs.existsSync("/config.json")) {
    var engineConfig = JSON.parse(fs.readFileSync("/config.json"));

    if (engineConfig.include_paths) {
      buildFileList = inclusionBasedFileListBuilder(
        engineConfig.include_paths
      );
    } else {
      // No explicit includes, let's try with everything
      buildFileList = inclusionBasedFileListBuilder(["./"]);
    }

    var userConfig = engineConfig.config || {};
    if (userConfig.config) {
      options.configFile = CODE_DIR + "/" + userConfig.config;
    }

    if (userConfig.extensions) {
      options.extensions = userConfig.extensions;
    }

    if (userConfig.debug) {
      debug = true;
    }
  }
});

var analysisFiles = runWithTiming("buildFileList", function() {
  return buildFileList(options.extensions);
});

function analyzeFiles() {
  var batchNum = 0
    , batchSize = 10
    , batchFiles
    , batchReport;

  while(analysisFiles.length > 0) {
    batchFiles = analysisFiles.splice(0, batchSize);

    if (debug) {
      process.stderr.write("Analyzing: " + batchFiles + "\n");
    }

    runWithTiming("analyze-batch-" + batchNum, function() {
       batchReport = cli.executeOnFiles(batchFiles);
    });
    runWithTiming("report-batch" + batchNum, function() {
      batchReport.results.forEach(function(result) {
        var path = result.filePath.replace(/^\/code\//, "");

        result.messages.forEach(function(message) {
          var issueJson = buildIssueJson(message, path);<commit_msg>Delay instantiation of cli variable<commit_after>#!/usr/src/app/bin/node_gc

var CODE_DIR = "/code";
process.chdir(CODE_DIR);

// Redirect `console.log` so that we are the only ones
// writing to STDOUT
var stdout = console.log;
console.log = console.error;

var CLIEngine = require("eslint").CLIEngine;
var docs = require("eslint").docs;
var fs = require("fs");
var glob = require("glob");
var options = { extensions: [".js"], ignore: true, reset: false, useEslintrc: true };
var cli; // instantiation delayed until after options are (potentially) modified
var debug = false;
var checks = require("../lib/checks");
var validateConfig = require("../lib/validate_config");

// a wrapper for emitting perf timing
function runWithTiming(name, fn) {
  var start = new Date()
    , rv = fn()
    , duration = (new Date() - start) / 1000;
  if (debug) {
    console.error("eslint.timing." + name + ": " + duration + "s");
  }
  return rv;
}

function contentBody(check) {
  var content = docs.get(check) || "For more information visit ";
  return content + "Source: http://eslint.org/docs/rules/\n";
}

function buildIssueJson(message, path) {
  // ESLint doesn't emit a ruleId in the
  // case of a fatal error (such as an invalid
  // token)
  var checkName = message.ruleId;
  if(message.fatal) {
    checkName = "fatal";
  }

  var issue = {
    type: "issue",
    categories: checks.categories(checkName),
    check_name: checkName,
    description: message.message,
    content: {
      body: contentBody(checkName)
    },
    location: {
      path: path,
      positions: {
        begin: {
          line: message.line || 1,
          column: message.column || 1
        },
        end: {
          line: message.line || 1,
          column: message.column || 1
        }
      }
    },
    remediation_points: checks.remediationPoints(checkName, message, cli.getConfigForFile(path))
  };
  return JSON.stringify(issue);
}

function isFileWithMatchingExtension(file, extensions) {
  var stats = fs.lstatSync(file);
  var extension = "." + file.split(".").pop();
  return (
    stats.isFile() &&
    !stats.isSymbolicLink()
    && extensions.indexOf(extension) >= 0
  );
}

function isFileIgnoredByLibrary(file) {
  var ignored = cli.isPathIgnored(file);
  if (ignored) {
    var output = "File `" + file + "` ignored because of your .eslintignore file." + "\n";
    process.stderr.write(output);
  }
  return ignored;
}

function prunePathsWithinSymlinks(paths) {
  // Extracts symlinked paths and filters them out, including any child paths
  var symlinks = paths.filter(function(path) {
    return fs.lstatSync(path).isSymbolicLink();
  });

  return paths.filter(function(path) {
    var withinSymlink = false;
    symlinks.forEach(function(symlink) {
      if (path.indexOf(symlink) === 0) {
        withinSymlink = true;
      }
    });
    return !withinSymlink;
  });
}

function inclusionBasedFileListBuilder(includePaths) {
  // Uses glob to expand the files and directories in includePaths, filtering
  // down to match the list of desired extensions.
  return function(extensions) {
    var analysisFiles = [];

    includePaths.forEach(function(fileOrDirectory, i) {
      if ((/\/$/).test(fileOrDirectory)) {
        // if it ends in a slash, expand and push
        var filesInThisDirectory = glob.sync(
          fileOrDirectory + "/**/**"
        );
        prunePathsWithinSymlinks(filesInThisDirectory).forEach(function(file, j){
          if (!isFileIgnoredByLibrary(file) && isFileWithMatchingExtension(file, extensions)) {
            analysisFiles.push(file);
          }
        });
      } else {
        if (!isFileIgnoredByLibrary(fileOrDirectory) && isFileWithMatchingExtension(fileOrDirectory, extensions)) {
          analysisFiles.push(fileOrDirectory);
        }
      }
    });

    return analysisFiles;
  };
}

var buildFileList;
runWithTiming("engineConfig", function () {
  if (fs.existsSync("/config.json")) {
    var engineConfig = JSON.parse(fs.readFileSync("/config.json"));

    if (engineConfig.include_paths) {
      buildFileList = inclusionBasedFileListBuilder(
        engineConfig.include_paths
      );
    } else {
      // No explicit includes, let's try with everything
      buildFileList = inclusionBasedFileListBuilder(["./"]);
    }

    var userConfig = engineConfig.config || {};
    if (userConfig.config) {
      options.configFile = CODE_DIR + "/" + userConfig.config;
    }

    if (userConfig.extensions) {
      options.extensions = userConfig.extensions;
    }

    if (userConfig.debug) {
      debug = true;
    }
  }

  cli = new CLIEngine(options);
});

var analysisFiles = runWithTiming("buildFileList", function() {
  return buildFileList(options.extensions);
});

function analyzeFiles() {
  var batchNum = 0
    , batchSize = 10
    , batchFiles
    , batchReport;

  while(analysisFiles.length > 0) {
    batchFiles = analysisFiles.splice(0, batchSize);

    if (debug) {
      process.stderr.write("Analyzing: " + batchFiles + "\n");
    }

    runWithTiming("analyze-batch-" + batchNum, function() {
       batchReport = cli.executeOnFiles(batchFiles);
    });
    runWithTiming("report-batch" + batchNum, function() {
      batchReport.results.forEach(function(result) {
        var path = result.filePath.replace(/^\/code\//, "");

        result.messages.forEach(function(message) {
          var issueJson = buildIssueJson(message, path);
{'fraction_non_alphanumeric': 0.13419631216241384, 'fraction_numerical': 0.002793816353138387, 'mean_word_length': 3.0432981927710845, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1143995', 'n_tokens_mistral': 3326, 'n_tokens_neox': 3229, 'n_words': 744}