<commit_before>'use strict';

var async = require('async'),
    concat = require('concat-stream'),
    hyperquest = require('hyperquest'),
    groupsOf = require('in-groups-of');

var debug = require('diagnostics')('footage');

var footage = module.exports = function (user) {
  var searchUri = 'https://skimdb.npmjs.com/registry/_design/app/_view/byUser?key='

  hyperquest.get(searchUri + '%22' + user + '%22')
    .pipe(concat({ encoding: 'string' }, function (data) {
      footage.downloads(JSON.parse(data));
    }));
};

footage.downloads = function (pkgs) {
  var baseUri = 'https://api.npmjs.org/downloads/range';
  var groups = groupsOf(pkgs.rows.map(function (row) {
    return row.id;
  }).sort(), 50);

  //
  // Does a simple reduce on the download
  // keys of a single npm downloads request
  //
  function sumDownloads(names, stats) {
    names.forEach(function (name) {
      if (!stats[name]) { return; }
      stats[name].total = Object.keys(stats[name].downloads || {})
        .reduce(function (sum, key) {
          return sum + stats[name].downloads[key].downloads;
        }, 0);
    });

    return stats;
  }

  //
  // Gets the statistics for the specified
  // set of packages.
  //
  function getGroupStats(query) {
    return function (pkgs, next) {
      var uri = [baseUri, query, pkgs.join(',')].join('/');
      debug('GET ' + uri);
      hyperquest.get(uri)
        .pipe(concat({ encoding: 'string' }, function (data) {
          next(null, sumDownloads(pkgs, JSON.parse(data)));
        }));
    };
  }

  async.mapLimit(groups, 5, getGroupStats('last-month'), function (err, mapped) {
    var all = mapped.reduce(function (acc, group) {
      Object.keys(group).forEach(function (name) {
        acc[name] = group[name];
      });

      return acc;
    }, {});

    Object.keys(all)
      .sort(function (lname, rname) {
        return all[lname].total - all[rname].total;
      })
      .forEach(function (name) {
        console.log(name, all[name].total);
      });
  });
};<commit_msg>[api] Include the user total.<commit_after>'use strict';

var async = require('async'),
    concat = require('concat-stream'),
    hyperquest = require('hyperquest'),
    groupsOf = require('in-groups-of');

var debug = require('diagnostics')('footage');

var footage = module.exports = function (user) {
  var searchUri = 'https://skimdb.npmjs.com/registry/_design/app/_view/byUser?key='

  hyperquest.get(searchUri + '%22' + user + '%22')
    .pipe(concat({ encoding: 'string' }, function (data) {
      footage.downloads(JSON.parse(data), user);
    }));
};

footage.downloads = function (pkgs, user) {
  var baseUri = 'https://api.npmjs.org/downloads/range';
  var groups = groupsOf(pkgs.rows.map(function (row) {
    return row.id;
  }).sort(), 50);

  //
  // Does a simple reduce on the download
  // keys of a single npm downloads request
  //
  function sumDownloads(names, stats) {
    names.forEach(function (name) {
      if (!stats[name]) { return; }
      stats[name].total = Object.keys(stats[name].downloads || {})
        .reduce(function (sum, key) {
          return sum + stats[name].downloads[key].downloads;
        }, 0);
    });

    return stats;
  }

  //
  // Gets the statistics for the specified
  // set of packages.
  //
  function getGroupStats(query) {
    return function (pkgs, next) {
      var uri = [baseUri, query, pkgs.join(',')].join('/');
      debug('GET ' + uri);
      hyperquest.get(uri)
        .pipe(concat({ encoding: 'string' }, function (data) {
          next(null, sumDownloads(pkgs, JSON.parse(data)));
        }));
    };
  }

  async.mapLimit(groups, 5, getGroupStats('last-month'), function (err, mapped) {
    var all = mapped.reduce(function (acc, group) {
      Object.keys(group).forEach(function (name) {
        acc[name] = group[name];
      });

      return acc;
    }, {});

    var userTotal = 0;
    Object.keys(all)
      .sort(function (lname, rname) {
        return all[lname].total - all[rname].total;
      })
      .forEach(function (name) {
        userTotal += all[name].total;
        console.log(name, all[name].total);
      });

    console.log(user, userTotal);
  });
};
{'fraction_non_alphanumeric': 0.17360114777618366, 'fraction_numerical': 0.0040650406504065045, 'mean_word_length': 2.7958257713248638, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7257678', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1335, 'n_words': 289}