<commit_before>// Example of a distributed computing problem that uses the Rally API
// mapData and reduceResults functions may use lodash methods
const _ = require('lodash');

/*
  EXAMPLE USE CASE OF RALLY API

  The following example uses the Rally distributed computing system
  to generate an array of all prime numbers between 1 and 1,000,000.
  This example intentionally uses a brute-force algorithm.

  Rally uses Project constructor to instantiate new calculation projects.
  The Project constructor takes a single 'options' object, which has the 
  following structure:

    options = {
      title: STRING,
      dataSet: ARRAY,
      generateDataSet: FUNCTION,
      mapData: FUNCTION,
      reduceResults: FUNCTION
    }

  'dataSet' is an array of items that you wish to be used in the distributed
  calculation. dataSet can be an array of arrays. You can provide the dataSet
  in the options object, or provide a 'generateDataSet' function that can be
  used to generate the desired dataSet. At the moment, generateDataSet cannot
  take in any parameters.

  The Project constructor will use 'dataSet' or 'generateDataSet' to initialize
  the Jobs that are available for our workers. Project constructor defaults to
  using the dataSet array if both dataSet and generateDataSet are provided in options.

  mapData is a function that your workers will run on every item of dataSet.
  This function takes a single parameter (dataSubset) and returns the result
  of the mapData calculation. Client worker sends the result back to the 
  server to be stored in an array. The results will be stored in the same
  order of their input dataSet. Suppose that your mapData function simply 
  doubles every number it finds. Then your original dataSet of [1, 2, 3, 4]
  will create a results array of [2, 4, 6, 8].

  reduceResults is fun on the array of final results once all jobs have been
  completed. Using the above example, if the reduceResults function sums up
  all numbers in an array, the final result of the distributed calculation 
  will be 2 + 4 + 6 + 8, which is 20.
*/
const testOptions = {
  title: 'Primes',
  projectType: 'primes',

  dataSet: null,

  generateDataSet: () => {
    var dataSet = [];

    for (var i = 0; i < 5; i++) {
      dataSet.push( [i * 50000, i * 50000 + 49999]);
    }
    return dataSet;
  },  

  mapData: (dataSubset) => {<commit_msg>Changed number of jobs from 5 to 20<commit_after>// Example of a distributed computing problem that uses the Rally API
// mapData and reduceResults functions may use lodash methods
const _ = require('lodash');

/*
  EXAMPLE USE CASE OF RALLY API

  The following example uses the Rally distributed computing system
  to generate an array of all prime numbers between 1 and 1,000,000.
  This example intentionally uses a brute-force algorithm.

  Rally uses Project constructor to instantiate new calculation projects.
  The Project constructor takes a single 'options' object, which has the 
  following structure:

    options = {
      title: STRING,
      dataSet: ARRAY,
      generateDataSet: FUNCTION,
      mapData: FUNCTION,
      reduceResults: FUNCTION
    }

  'dataSet' is an array of items that you wish to be used in the distributed
  calculation. dataSet can be an array of arrays. You can provide the dataSet
  in the options object, or provide a 'generateDataSet' function that can be
  used to generate the desired dataSet. At the moment, generateDataSet cannot
  take in any parameters.

  The Project constructor will use 'dataSet' or 'generateDataSet' to initialize
  the Jobs that are available for our workers. Project constructor defaults to
  using the dataSet array if both dataSet and generateDataSet are provided in options.

  mapData is a function that your workers will run on every item of dataSet.
  This function takes a single parameter (dataSubset) and returns the result
  of the mapData calculation. Client worker sends the result back to the 
  server to be stored in an array. The results will be stored in the same
  order of their input dataSet. Suppose that your mapData function simply 
  doubles every number it finds. Then your original dataSet of [1, 2, 3, 4]
  will create a results array of [2, 4, 6, 8].

  reduceResults is fun on the array of final results once all jobs have been
  completed. Using the above example, if the reduceResults function sums up
  all numbers in an array, the final result of the distributed calculation 
  will be 2 + 4 + 6 + 8, which is 20.
*/
const testOptions = {
  title: 'Primes',
  projectType: 'primes',

  dataSet: null,

  generateDataSet: () => {
    var dataSet = [];

    for (var i = 0; i < 20; i++) {
      dataSet.push( [i * 50000, i * 50000 + 49999]);
    }
    return dataSet;
  },  

  mapData: (dataSubset) => {
{'fraction_non_alphanumeric': 0.05503439649781113, 'fraction_numerical': 0.017094017094017096, 'mean_word_length': 3.45910780669145, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2195870', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1247, 'n_words': 689}