<commit_before>/*******************************************************************************
 * fetchFromGithub.js - quick node module to populate allData.js from Github
 *
 * Note: in order for this to work, you'll need Node.js and npm installed.
 * You'll also need to run:
 * $ npm install request
 * $ npm install underscore
 * To get the necessary dependencies. After that, just run
 * $ node fetchFromGithub.js
 * To refresh allData.js
 *
 * This data is pulled from Github's Commits API. See
 * https://developer.github.com/v3/repos/commits for details.
 ******************************************************************************/

var request = require( 'request' );
var us = require( 'underscore' );

/*
Feel free to customize these to fetch from a different repo
*/
var OWNER = 'twbs',
    REPO = 'bootstrap',
    TOKEN = '', // See https://developer.github.com/v3/auth/#basic-authentication
                // for details on personal access tokens
    URL = 'https://api.github.com/repos/' + OWNER + '/' + REPO + '/commits';

// Variables required by this script
var _commitList = [];

/*
Helper functions
*/
function _getFromGithub( requestUrl, callback ) {
  var options = {
    url: requestUrl,
    headers: { 'User-Agent': 'ui-assignment' },
    auth: { 'user': TOKEN, pass: '' }
  };

  request( options, function( err, response, body ) {
    if( err || response.statusCode != 200 ) {
      console.error( 'Could not fetch data from Github, exiting.' );
      console.error( err ? err.stack : body );
      process.exit( 1 );
    }

    callback( JSON.parse( body ) );
  });
}

function _handleListResults( data ) {
  us.each( data, function( element ) {
    _getFromGithub( URL + '/' + element.sha, _addToCommitList );
  });
}

function _addToCommitList( data ) {
  _commitList.push( data );

  /*
  This is brittle, but will work for our purposes
  */
  if( _commitList.length >= 200 ) {
    console.log( _commitList );
  }
}

// Get 200 commits from the API
_getFromGithub( URL + '?per_page=100', _handleListResults );
_getFromGithub( URL + '?page=2&per_page=100', _handleListResults );<commit_msg>Changed script to output properly stringified data.<commit_after>/*******************************************************************************
 * fetchFromGithub.js - quick node module to populate allData.js from Github
 *
 * Note: in order for this to work, you'll need Node.js and npm installed.
 * You'll also need to run:
 * $ npm install request
 * $ npm install underscore
 * To get the necessary dependencies. After that, just run
 * $ node fetchFromGithub.js
 * To refresh allData.js
 *
 * This data is pulled from Github's Commits API. See
 * https://developer.github.com/v3/repos/commits for details.
 ******************************************************************************/

var request = require( 'request' );
var us = require( 'underscore' );

/*
Feel free to customize these to fetch from a different repo
*/
var OWNER = 'twbs',
    REPO = 'bootstrap',
    TOKEN = '', // See https://developer.github.com/v3/auth/#basic-authentication
                // for details on personal access tokens
    URL = 'https://api.github.com/repos/' + OWNER + '/' + REPO + '/commits';

// Variables required by this script
var _commitList = [];

/*
Helper functions
*/
function _getFromGithub( requestUrl, callback ) {
  var options = {
    url: requestUrl,
    headers: { 'User-Agent': 'ui-assignment' },
    auth: { 'user': TOKEN, pass: '' }
  };

  request( options, function( err, response, body ) {
    if( err || response.statusCode != 200 ) {
      console.error( 'Could not fetch data from Github, exiting.' );
      console.error( err ? err.stack : body );
      process.exit( 1 );
    }

    callback( JSON.parse( body ) );
  });
}

function _handleListResults( data ) {
  us.each( data, function( element ) {
    _getFromGithub( URL + '/' + element.sha, _addToCommitList );
  });
}

function _addToCommitList( data ) {
  _commitList.push( data );

  /*
  This is brittle, but will work for our purposes
  */
  if( _commitList.length >= 200 ) {
    console.log( JSON.stringify( _commitList ) );
  }
}

// Get 200 commits from the API
_getFromGithub( URL + '?per_page=100', _handleListResults );
_getFromGithub( URL + '?page=2&per_page=100', _handleListResults );
{'fraction_non_alphanumeric': 0.19609665427509293, 'fraction_numerical': 0.008828996282527882, 'mean_word_length': 3.498432601880878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3791965', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1256, 'n_words': 393}