<commit_before>/**
 * grunt-plugins.js - gets the plugin list from npm
 * server.js serves the plugin list via the website.
 */
var request = require('request');
var _ = require('lodash');
var async = require('async');
var ent = require('ent');
var fs = require('fs');

require('date-utils');

// A list of grunt plugins that have wronged!
var bannedPlugins = [
  'grunt-contrib-jst-2', // Reason: unofficial contrib plugin
  'loadnpmtasks',  // Reason: not a plugin
  'dp-grunt-contrib-copy',
  'grunt-contrib-yohtml', // Reason: unofficial contrib plugin
  'grunt-contrib-quickstart', // Reason: unofficial contrib plugin
  'grunt-contrib-coffeeify', // Reason: unofficial contrib plugin
  'grunt-contrib-nodefy', // Reason: unofficial contrib plugin
  'grunt-contrib-handlebars-rhengles', // Reason: unofficial contrib plugin
  'grunt-contrib-juicepress', // Reason: unofficial contrib plugin
  'grunt-contrib-build-crx', // Reason: unofficial contrib plugin
  'grunt-test', // Reason: no real plugin
  'grunt-testingoscar123', // Reason: no real plugin
  'assemble-less-variables',
  'grunt-contrib-eslint', // Reason: unofficial contrib plugin
  'grunt-contrib-htmlone', // Reason: unofficial contrib plugin
  'grunt-contrib-coffeeify', // Reason: unofficial contrib plugin
  'grunt-contrib-compass-sourcemap', // Reason: unofficial contrib plugin
  'grunt-contrib-spritify', // Reason: unofficial contrib plugin
  'grunt-contrib-include', // Reason: unofficial contrib plugin
  'grunt-contrib-juicepress', // Reason: unofficial contrib plugin
  'grunt-contrib-hogan', // Reason: unofficial contrib plugin
  'grunt-contrib-stylus2tss', // Reason: unofficial contrib plugin, no documentation
  'grunt-contrib-smartdoc', // Reason: unofficial contrib plugin
  'grunt-contrib-cjsc', // Reason: unofficial contrib plugin
  'grunt-contrib-compressor', // Reason: unofficial contrib plugin
  'grunt-contrib-license-report', // Reason: unofficial contrib plugin
  'grunt-contrib-tishadow', // Reason: unofficial contrib plugin
  'grunt-contrib-smartsprites', // Reason: unofficial contrib plugin, no documentation
  'grunt-contrib-rubylint', // Reason: unofficial contrib plugin
  'grunt-contrib-lualint', // Reason: unofficial contrib plugin
  'grunt-contrib-levin-usemin', // Reason: unofficial contrib plugin
  'grunt-contrib-quickstart', // Reason: unofficial contrib plugin
  'grunt-contrib-i18next', // Reason: unofficial contrib plugin
  'grunt-contrib-ftpush',  // Reason: unofficial contrib plugin
  'grunt-mindirect', // Reason: duplicate of contrib-uglify
  'private-grunt-contrib-uglify', // Reason: duplicate of contrib-uglify
  'grunt-handlebars-static', // Reason: duplicate of compile-handlebars
  'grunt-contrib-jshint-reporter-tweaks', // Reason: unofficial contrib plugin
  'grunt-sass-fork', // Reason: duplicate of grunt-contrib-sass
  'grunt-gh-deploy', // Reason: duplicate of grunt-gh-pages
  'grunt-github-pages', // Reason: duplicate of grunt-gh-pages
  'grunt-contrib-cssmin-pre-2.1.0', // Reason: unofficial contrib plugin
  'private-grunt-contrib-cssmin', // Reason: unofficial contrib plugin
  'private-grunt-contrib-stylus', // Reason: unofficial contrib plugin
  'grunt-contrib-zopfli', // Reason: unofficial contrib plugin, duplicate of grunt-zopfli
  'grunt-htmlmin', // Reason: duplicate of contrib-htmlmin
  'grunt-symbolic-link', // Reason: duplicate of contrib-symlink
  'grunt-symlink', // Reason: duplicate of contrib-symlink
  'grunt-symlinks', // Reason: duplicate of contrib-symlink
  'grunt-coffee', // Reason: duplicate of contrib-coffee
  'grunt-coffee-lint', // Reason: duplicate of grunt-coffeelint, missing documentation
  'grunt-less', // Reason: duplicate of grunt-contrib-less
  'grunt-scsslint', // Reason: duplicate of grunt-scss-lint
  'grunt-css', // Reason: deprecated
  'ngbp-contrib-lintcss', // Reason: missing documentation, duplicate of grunt-contrib-csslint
  'ngbp-contrib-mincss', // Reason: missing documentation, duplicate of grunt-contrib-cssmin
  'grunt-init-test' // Reason: missing documentation, does nothing
];

var pluginFile = 'build/plugin-list.json';

function getPlugin(item, callback) {
  var name = item.key[1];
  var url = 'https://skimdb.npmjs.com/registry/' + name;
  request({url: url, json: true}, function handlePlugin(error, response, body) {
    if (!error && response.statusCode == 200) {
      callback(null, condensePlugin(body));
    } else {
      console.log('Failed to get data for:', name);
      callback(null, null);
    }<commit_msg>Updates to blacklist<commit_after>/**
 * grunt-plugins.js - gets the plugin list from npm
 * server.js serves the plugin list via the website.
 */
var request = require('request');
var _ = require('lodash');
var async = require('async');
var ent = require('ent');
var fs = require('fs');

require('date-utils');

// A list of grunt plugins that have wronged!
var bannedPlugins = [
  'grunt-contrib-jst-2', // Reason: unofficial contrib plugin
  'loadnpmtasks',  // Reason: not a plugin
  'dp-grunt-contrib-copy',
  'grunt-contrib-yohtml', // Reason: unofficial contrib plugin
  'grunt-contrib-quickstart', // Reason: unofficial contrib plugin
  'grunt-contrib-coffeeify', // Reason: unofficial contrib plugin
  'grunt-contrib-nodefy', // Reason: unofficial contrib plugin
  'grunt-contrib-handlebars-rhengles', // Reason: unofficial contrib plugin
  'grunt-contrib-juicepress', // Reason: unofficial contrib plugin
  'grunt-contrib-build-crx', // Reason: unofficial contrib plugin
  'grunt-test', // Reason: no real plugin
  'grunt-testingoscar123', // Reason: no real plugin
  'assemble-less-variables',
  'grunt-contrib-eslint', // Reason: unofficial contrib plugin
  'grunt-contrib-htmlone', // Reason: unofficial contrib plugin
  'grunt-contrib-coffeeify', // Reason: unofficial contrib plugin
  'grunt-contrib-compass-sourcemap', // Reason: unofficial contrib plugin
  'grunt-contrib-spritify', // Reason: unofficial contrib plugin
  'grunt-contrib-include', // Reason: unofficial contrib plugin
  'grunt-contrib-juicepress', // Reason: unofficial contrib plugin
  'grunt-contrib-hogan', // Reason: unofficial contrib plugin
  'grunt-contrib-stylus2tss', // Reason: unofficial contrib plugin, no documentation
  'grunt-contrib-smartdoc', // Reason: unofficial contrib plugin
  'grunt-contrib-cjsc', // Reason: unofficial contrib plugin
  'grunt-contrib-compressor', // Reason: unofficial contrib plugin
  'grunt-contrib-license-report', // Reason: unofficial contrib plugin
  'grunt-contrib-tishadow', // Reason: unofficial contrib plugin
  'grunt-contrib-smartsprites', // Reason: unofficial contrib plugin, no documentation
  'grunt-contrib-rubylint', // Reason: unofficial contrib plugin
  'grunt-contrib-lualint', // Reason: unofficial contrib plugin
  'grunt-contrib-levin-usemin', // Reason: unofficial contrib plugin
  'grunt-contrib-quickstart', // Reason: unofficial contrib plugin
  'grunt-contrib-i18next', // Reason: unofficial contrib plugin
  'grunt-contrib-ftpush',  // Reason: unofficial contrib plugin
  'grunt-mindirect', // Reason: duplicate of contrib-uglify
  'private-grunt-contrib-uglify', // Reason: duplicate of contrib-uglify
  'grunt-handlebars-static', // Reason: duplicate of compile-handlebars
  'grunt-contrib-jshint-reporter-tweaks', // Reason: unofficial contrib plugin
  'grunt-sass-fork', // Reason: duplicate of grunt-contrib-sass
  'grunt-gh-deploy', // Reason: duplicate of grunt-gh-pages
  'grunt-github-pages', // Reason: duplicate of grunt-gh-pages
  'grunt-contrib-cssmin-pre-2.1.0', // Reason: unofficial contrib plugin
  'private-grunt-contrib-cssmin', // Reason: unofficial contrib plugin
  'private-grunt-contrib-stylus', // Reason: unofficial contrib plugin
  'grunt-contrib-zopfli', // Reason: unofficial contrib plugin, duplicate of grunt-zopfli
  'grunt-htmlmin', // Reason: duplicate of contrib-htmlmin
  'grunt-symbolic-link', // Reason: duplicate of contrib-symlink
  'grunt-symlink', // Reason: duplicate of contrib-symlink
  'grunt-symlinks', // Reason: duplicate of contrib-symlink
  'grunt-coffee', // Reason: duplicate of contrib-coffee
  'grunt-coffee-lint', // Reason: duplicate of grunt-coffeelint, missing documentation
  'grunt-less', // Reason: duplicate of grunt-contrib-less
  'grunt-scsslint', // Reason: duplicate of grunt-scss-lint
  'grunt-css', // Reason: deprecated
  'ngbp-contrib-lintcss', // Reason: missing documentation, duplicate of grunt-contrib-csslint
  'ngbp-contrib-mincss', // Reason: missing documentation, duplicate of grunt-contrib-cssmin
  'grunt-init-test', // Reason: missing documentation, does nothing
  'grunt-contrib-githooks', // Reason: unofficial contrib plugin
  'grunt-contrib-stylus-map', // Reason: unofficial contrib plugin
  'grunt-contrib-dox', // Reason: unofficial contrib plugin
  'grunt-contrib-requiregrep', // Reason: unofficial contrib plugin
  'grunt-contrib-litchi' // Reason: unofficial contrib plugin
];

var pluginFile = 'build/plugin-list.json';

function getPlugin(item, callback) {
  var name = item.key[1];
  var url = 'https://skimdb.npmjs.com/registry/' + name;
  request({url: url, json: true}, function handlePlugin(error, response, body) {
    if (!error && response.statusCode == 200) {
      callback(null, condensePlugin(body));
    } else {
      console.log('Failed to get data for:', name);
      callback(null, null);
    }
{'fraction_non_alphanumeric': 0.13471778487752928, 'fraction_numerical': 0.002981895633652822, 'mean_word_length': 4.154226125137212, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '867200', 'n_tokens_mistral': 3149, 'n_tokens_neox': 2755, 'n_words': 759}