<commit_before>// Copyright © 2016 <NAME>. All rights reserved.
// The following code is covered by the AGPL-3.0 license.

var db = require('./db');


// Set a value for a metric.

function set (object, metric, value) {

  var data = object.data;

  if (!data) {
    data = object.data = {};
  }

  data[metric] = value;
  db.save();

}

exports.set = set;


// Push a value into a metric array.

function push (object, metric, value) {

  var data = object.data;

  if (!data || !data[metric]) {
    set(object, metric, []);
    data = object.data;
  }

  data[metric].push(value);
  db.save();

}

exports.push = push;


// Get all available metrics.

function get (callback) {

  var data = {
    users: getUserData(),
    projects: getProjectData(),
    contributions: getContributionData()
  };

  callback(data);

}

exports.get = get;


// Get metrics about all users.

function getUserData () {

  var data = {};

  ['users', 'waitlist'].forEach(function (metric) {
    var object = db.get(metric);
    data[metric] = Object.keys(object).length;
  });

  return data;

}

exports.getUserData = getUserData;


// Get metrics about all projects.

function getProjectData () {

  var data = [];
  var projects = db.get('projects');

  for (var projectId in projects) {
    var project = projects[projectId];
    data.push({
      id: projectId,
      data: project.data
    });
  }


  return data;

}

exports.getProjectData = getProjectData;


// Get metrics about all contributions.

function getContributionData () {

  var data = {
    'new': 0,
    'build-failed': 0,
    'built': 0,
    'start-failed': 0,
    'started': 0,
    'merged': 0
  };
  var users = db.get('users');

  for (var email in users) {
    var machines = users[email].machines;

    for (var projectId in machines) {
      machines[projectId].forEach(function (machine) {
        data[machine.status]++;
      });
    }
  }

  var total = 0;
  for (var status in data) {
    total += data[status];
  }
  data.total = total;

  return data;

}

exports.getContributionData = getContributionData;
<commit_msg>Measure data collection time.<commit_after>// Copyright © 2016 <NAME>. All rights reserved.
// The following code is covered by the AGPL-3.0 license.

var db = require('./db');
var log = require('./log');


// Set a value for a metric.

function set (object, metric, value) {

  var data = object.data;

  if (!data) {
    data = object.data = {};
  }

  data[metric] = value;
  db.save();

}

exports.set = set;


// Push a value into a metric array.

function push (object, metric, value) {

  var data = object.data;

  if (!data || !data[metric]) {
    set(object, metric, []);
    data = object.data;
  }

  data[metric].push(value);
  db.save();

}

exports.push = push;


// Get all available metrics.

function get (callback) {

  var time = Date.now();
  var data = {
    users: getUserData(),
    projects: getProjectData(),
    contributions: getContributionData()
  };

  callback(data);
  log('data collection took', Date.now() - time, 'ms.');

}

exports.get = get;


// Get metrics about all users.

function getUserData () {

  var data = {};

  ['users', 'waitlist'].forEach(function (metric) {
    var object = db.get(metric);
    data[metric] = Object.keys(object).length;
  });

  return data;

}

exports.getUserData = getUserData;


// Get metrics about all projects.

function getProjectData () {

  var data = [];
  var projects = db.get('projects');

  for (var projectId in projects) {
    var project = projects[projectId];
    data.push({
      id: projectId,
      data: project.data
    });
  }


  return data;

}

exports.getProjectData = getProjectData;


// Get metrics about all contributions.

function getContributionData () {

  var data = {
    'new': 0,
    'build-failed': 0,
    'built': 0,
    'start-failed': 0,
    'started': 0,
    'merged': 0
  };
  var users = db.get('users');

  for (var email in users) {
    var machines = users[email].machines;

    for (var projectId in machines) {
      machines[projectId].forEach(function (machine) {
        data[machine.status]++;
      });
    }
  }

  var total = 0;
  for (var status in data) {
    total += data[status];
  }
  data.total = total;

  return data;

}

exports.getContributionData = getContributionData;

{'fraction_non_alphanumeric': 0.14730483271375464, 'fraction_numerical': 0.0060408921933085506, 'mean_word_length': 2.782952548330404, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '55422', 'n_tokens_mistral': 1481, 'n_tokens_neox': 1403, 'n_words': 401}