<commit_before>#!/usr/bin/env node

/* global require, module */

// require()s
var path = require('path');
var fs = require('fs');

var _ = require('lodash');
var rimraf = require('rimraf');
var json = require('bower-json');
var bowerDirectory = require('bower-directory');
var debug = require('debug')('bowcat');

var opts = require('minimist')(process.argv.slice(2));

var usage = 'Usage: bowcat [<input-dir>] [-o <output-dir>] [--min | -m]\n\n'
          + 'Quickly concatenate bower dependencies.\n\n'
          + 'Options:\n'
          + '  -o <output-dir>  Where to write the concatenated dependencies to\n'
          + '  --min, -m        Include only minified files [default: false]\n'

// opt parsing
if (opts.help) {
  process.stdout.write(usage);
  process.exit(0);
}

var includeMins = opts.min || opts.m;
var inputDir = opts._[0] || '.';
var outputDir = opts.o || path.join('.', 'build');

inputDir = bowerDirectory.sync({ cwd: path.resolve(inputDir) });

outputDir = path.resolve(outputDir);

var pkgs = fs.readdirSync(inputDir);

pkgs = _.map(pkgs, function (p) {
  return path.join(inputDir, p);
});

var concatedPkgs = [];

// constructFileList: create a list of files to concat for a particular
// package. 'dir' is the path to the package, 'mains' is the 'main' field
// in the package's bower.json, 'minified' is whether or not to include
// minified files
function constructFileList (dir, mains, minified) {
  var files = fs.readdirSync(dir);

  files = _.map(files, function (f) {
    return path.join(dir, f);
  });

  _.each(files, function (f, i, l) {
    if (fs.statSync(f).isDirectory())
      files = files.concat(constructFileList(f, mains, minified));
  });

  files = _.filter(files, function (f) {
    if (fs.statSync(f).isDirectory()) return false;

    var fname = path.basename(f).split('.').slice(0, -1).join('.');

    var include = _.some(mains, function (m) {
      m = path.basename(m).split('.').slice(0, -1).join('.');
      return m === fname;
    });

    if (minified)
      include = f.indexOf('.min.js') === (f.length - 7)
             || f.indexOf('.min.css') === (f.length - 8);

    return include;
  });

  return files;
}

// concatPackage: concatenate a single package. 'package' is the full path to
// the package directory, 'outDir' is the output directory, 'minified'
// is whether or not to include minified files
function concatPackage (pack, outDir, minified) {
  if (_.contains(concatedPkgs, path.basename(pack))) return;

  var bowerFile = 'bower.json';

  if (! fs.existsSync(path.join(pack, bowerFile)))
    bowerFile = '.bower.json';

  var regularJSON = JSON.parse(
    fs.readFileSync(path.join(pack, bowerFile))
  );
  var bowerJSON = json.normalize(regularJSON);
  var deps = bowerJSON.dependencies || {};
  var mains = bowerJSON.main || [];

  concatedPkgs.push(path.basename(pack));

  _.each(Object.keys(deps), function (pkg, i, l) {<commit_msg>when using the minified option, bowcat will now add the non-minified version if there's no minified version present (as in the case of firebase) in order to prevent missing dependencies<commit_after>#!/usr/bin/env node

/* global require, module */

// require()s
var path = require('path');
var fs = require('fs');

var _ = require('lodash');
var rimraf = require('rimraf');
var json = require('bower-json');
var bowerDirectory = require('bower-directory');
var debug = require('debug')('bowcat');

var opts = require('minimist')(process.argv.slice(2));

var usage = 'Usage: bowcat [<input-dir>] [-o <output-dir>] [--min | -m]\n\n'
          + 'Quickly concatenate bower dependencies.\n\n'
          + 'Options:\n'
          + '  -o <output-dir>  Where to write the concatenated dependencies to\n'
          + '  --min, -m        Include only minified files [default: false]\n'

// opt parsing
if (opts.help) {
  process.stdout.write(usage);
  process.exit(0);
}

var includeMins = opts.min || opts.m;
var inputDir = opts._[0] || '.';
var outputDir = opts.o || path.join('.', 'build');

inputDir = bowerDirectory.sync({ cwd: path.resolve(inputDir) });

outputDir = path.resolve(outputDir);

var pkgs = fs.readdirSync(inputDir);

pkgs = _.map(pkgs, function (p) {
  return path.join(inputDir, p);
});

var concatedPkgs = [];

// constructFileList: create a list of files to concat for a particular
// package. 'dir' is the path to the package, 'mains' is the 'main' field
// in the package's bower.json, 'minified' is whether or not to include
// minified files
function constructFileList (dir, mains, minified) {
  var files = fs.readdirSync(dir);

  files = _.map(files, function (f) {
    return path.join(dir, f);
  });

  _.each(files, function (f, i, l) {
    if (fs.statSync(f).isDirectory())
      files = files.concat(constructFileList(f, mains, minified));
  });

  files = _.filter(files, function (f) {
    if (fs.statSync(f).isDirectory()) return false;

    var fname = path.basename(f).split('.').slice(0, -1).join('.');

    var include = _.some(mains, function (m) {
      m = path.basename(m).split('.').slice(0, -1).join('.');
      return m === fname;
    });

    if (minified)
      include = f.indexOf('.min.js') === (f.length - 7)
             || f.indexOf('.min.css') === (f.length - 8);

    return include;
  });

  if (minified && files.length === 0) {
    return constructFileList(dir, mains, false);
  }

  return files;
}

// concatPackage: concatenate a single package. 'package' is the full path to
// the package directory, 'outDir' is the output directory, 'minified'
// is whether or not to include minified files
function concatPackage (pack, outDir, minified) {
  if (_.contains(concatedPkgs, path.basename(pack))) return;

  var bowerFile = 'bower.json';

  if (! fs.existsSync(path.join(pack, bowerFile)))
    bowerFile = '.bower.json';

  var regularJSON = JSON.parse(
    fs.readFileSync(path.join(pack, bowerFile))
  );
  var bowerJSON = json.normalize(regularJSON);
  var deps = bowerJSON.dependencies || {};
  var mains = bowerJSON.main || [];

  concatedPkgs.push(path.basename(pack));

  _.each(Object.keys(deps), function (pkg, i, l) {
{'fraction_non_alphanumeric': 0.16918032786885245, 'fraction_numerical': 0.003114754098360656, 'mean_word_length': 3.463057790782736, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4952279', 'n_tokens_mistral': 2097, 'n_tokens_neox': 2036, 'n_words': 557}