<commit_before>#!/usr/bin/env node
'use strict';

// spreadsheet key is the long id in the sheets URL
var GoogleSpreadsheet = require('google-spreadsheet');
var Sheet = new GoogleSpreadsheet('1LI4OBEGMPKxBwtGY_xJ1yT_G-rrCJxl8FaC0q0DaBl0');

var Firebase = require('firebase');

var Moment = require('moment-timezone');

const TS_PATH = './node_modules/jstrueskill/lib/racingjellyfish/jstrueskill';
var TrueSkillCalculator = require(`${TS_PATH}/TrueSkillCalculator`);
var GameInfo = require(`${TS_PATH}/GameInfo`).getDefaultGameInfo();
var Player = require(`${TS_PATH}/Player`);
var Team = require(`${TS_PATH}/Team`);

var _ = require('lodash');

function getSheetRows(sheet, tab) {
  return new Promise((resolve,reject) => {
    Sheet.getRows(tab, (err, rows) => {
      if (err !== null) {reject(err);}
      resolve(rows);
    });
  });
}

//Participants
function getParticipantMap(rows) {
  return rows.reduce((participantMap, row) => {
    participantMap.set(Number.parseInt(row.participantid), {
      fullname: `${row.last}, ${row.first}`,
      driverSkill: GameInfo.getDefaultRating()
    });
    return participantMap;
  }, new Map());
}

//Races
function getRaceMap(rows) {
  return rows.reduce((raceMap, row) => {
    raceMap.set(row.raceid, {
      raceId: row.raceid,
      name: `${row.year} ${row.racename}`,
      theme: row.theme,
      abbreviation: row.raceabbr,
      date: Moment.tz(row.date, 'MM/DD/YYYY', 'America/New_York').toJSON(),
      numLegs: Number.parseInt(row.numlegs),
      legs: [],
      toScore: row.toscore === 'TRUE' ? true : false,
      start: row.start,
      finish: row.finish,
      distance: row.distance
    });
    return raceMap;
  }, new Map());
}

function getLegResults(rows, raceMap) {
  let results = rows.map((row) => {
    const RACE_DATE = Moment(raceMap.get(row.raceid).date);
    return {
      raceId: row.raceid,
      leg: Number.parseInt(row.leg),
      driverId: Number.parseInt(row.driverid),
      navigatorId: 'n/a',
      start: RACE_DATE.clone().add(Moment.duration(row.starttime)).toJSON(),
      end: RACE_DATE.clone().add(Moment.duration(row.finishtime)).toJSON(),
      duration: Moment.duration(row.legduration).toJSON(),
      rank: null, //placeholder
      dnf: row.dnf === 'TRUE' ? true : false,
      unknownTime: row.unknown === 'TRUE' ? true : false
    };
  });
  return results;
}

function resultCompare(r1, r2) {
  let compareVal = (r => {
    if (r.dnf === true) { return Number.MAX_VALUE / 2; }
    if (r.unknownTime === true) { return Number.MAX_VALUE; }
    return Moment.duration(r.duration).asMilliseconds();
  });
  return compareVal(r1) - compareVal(r2);
}

function getRaceResults(legResults, participantMap, raceMap) {
  let raceResults = _.chain(legResults)
  //first get leg rankings by doing some cool sorting work
  .sort((r1, r2) => {
    if (r1.raceId !== r2.raceId) { return new Date(r1.start) - new Date(r2.start); }
    if (r1.leg !== r2.leg) { return r1.leg - r2.leg; }
    return resultCompare(r1, r2);
  })
  //rank the individual race legs
  .map((curResult, index, allResults) => {
    let rank = 1;
    if (index !== 0) {
      let prevResult = allResults[index - 1];
      if ((prevResult.raceId === curResult.raceId) && (prevResult.leg === curResult.leg)) {
        //curResult is in the same leg
        if (resultCompare(curResult, prevResult) === 0) {
          rank = prevResult.rank;
        }  else {
          rank = prevResult.rank + 1;
        }
      }
    }
    curResult.rank = rank;
    return curResult;
  })

  //then start to process it into a race result table
  .reduce((races, curLeg) => {
    if (!races.hasOwnProperty(curLeg.raceId)) {
      races[curLeg.raceId] = {};
    }
    if (!races[curLeg.raceId].hasOwnProperty(curLeg.driverId)) {
      races[curLeg.raceId][curLeg.driverId] = {
        driverId: curLeg.driverId,
        driverName: participantMap.get(curLeg.driverId).fullname,
        navigatorName: 'n/a',
        duration: Moment.duration(0).toJSON(),
        rank: null,
        legs: {},
        dnf: false,
        unknownTime: false
      };
    }
    races[curLeg.raceId][curLeg.driverId].legs[curLeg.leg] = _.pick(curLeg,
      ['start','end', 'duration', 'rank', 'dnf', 'unknownTime', 'toScore']);

    Moment.duration(races[curLeg.raceId][curLeg.driverId].duration).add(curLeg.duration);
    if (curLeg.dnf === true) { races[curLeg.raceId][curLeg.driverId].dnf = true; }
    if (curLeg.unknownTime === true) { races[curLeg.raceId][curLeg.driverId].unknownTime = true; }
    return races;
  }, {})

  //create sorted table array of results
  .reduce((resultsMap, raceResult, raceId) => {
    resultsMap[raceId] = _(raceResult).values()
    .sort(resultCompare)
    .map((result, index, prevResults) => {
      if (index === 0) {
        result.rank = 1;
      } else {
        let prevResult = prevResults[index - 1];
        if (resultCompare(result, prevResult) === 0) {
          result.rank = prevResult.rank;
        } else {
          result.rank = prevResult.rank + 1;
        }
      }
      return result;
    })
    .value();
    return resultsMap;
  }, {})
  .value();

  let retVal = _(Array.from(raceMap.values()))
  .map(race => {
    var raceDetail = _.pick(race,
        ['raceId', 'name', 'theme', 'date', 'start', 'end', 'distance', 'numLegs', 'toScore']);
    let raceId = raceDetail.raceId;
    if (raceResults[raceId] !== undefined) {
      let results = raceResults[raceId];
      raceDetail.results = results;
      raceDetail.time = results[0].duration;
      raceDetail.winner = results[0].driverName;
    }
    return raceDetail;
  })
  .value();<commit_msg>more lodash refactoring... lots to go<commit_after>#!/usr/bin/env node
'use strict';

// spreadsheet key is the long id in the sheets URL
var GoogleSpreadsheet = require('google-spreadsheet');
var Sheet = new GoogleSpreadsheet('1LI4OBEGMPKxBwtGY_xJ1yT_G-rrCJxl8FaC0q0DaBl0');

var Firebase = require('firebase');

var Moment = require('moment-timezone');

const TS_PATH = './node_modules/jstrueskill/lib/racingjellyfish/jstrueskill';
var TrueSkillCalculator = require(`${TS_PATH}/TrueSkillCalculator`);
var GameInfo = require(`${TS_PATH}/GameInfo`).getDefaultGameInfo();
var Player = require(`${TS_PATH}/Player`);
var Team = require(`${TS_PATH}/Team`);

var _ = require('lodash');

function getSheetRows(sheet, tab) {
  return new Promise((resolve,reject) => {
    Sheet.getRows(tab, (err, rows) => {
      if (err !== null) {reject(err);}
      resolve(rows);
    });
  });
}

//Participants
function getParticipantMap(rows) {
  return rows.reduce((participantMap, row) => {
    participantMap.set(Number.parseInt(row.participantid), {
      fullname: `${row.last}, ${row.first}`,
      driverSkill: GameInfo.getDefaultRating()
    });
    return participantMap;
  }, new Map());
}

//Races
function getRaceMap(rows) {
  return rows.reduce((raceMap, row) => {
    raceMap.set(row.raceid, {
      raceId: row.raceid,
      name: `${row.year} ${row.racename}`,
      theme: row.theme,
      abbreviation: row.raceabbr,
      date: Moment.tz(row.date, 'MM/DD/YYYY', 'America/New_York').toJSON(),
      numLegs: Number.parseInt(row.numlegs),
      legs: [],
      toScore: row.toscore === 'TRUE' ? true : false,
      start: row.start,
      finish: row.finish,
      distance: row.distance
    });
    return raceMap;
  }, new Map());
}

function getLegResults(rows, raceMap) {
  let results = rows.map((row) => {
    const RACE_DATE = Moment(raceMap.get(row.raceid).date);
    return {
      raceId: row.raceid,
      leg: Number.parseInt(row.leg),
      driverId: Number.parseInt(row.driverid),
      navigatorId: 'n/a',
      start: RACE_DATE.clone().add(Moment.duration(row.starttime)).toJSON(),
      end: RACE_DATE.clone().add(Moment.duration(row.finishtime)).toJSON(),
      duration: Moment.duration(row.legduration).toJSON(),
      rank: null, //placeholder
      dnf: row.dnf === 'TRUE' ? true : false,
      unknownTime: row.unknown === 'TRUE' ? true : false
    };
  });
  return results;
}

function getResultCompare(r) {
  if (r.dnf === true) { return Number.MAX_VALUE / 2; }
  if (r.unknownTime === true) { return Number.MAX_VALUE; }
  return Moment.duration(r.duration).asMilliseconds();
}

function resultCompare(r1, r2) {
  return getResultCompare(r1) - getResultCompare(r2);
}

function getRaceResults(legResults, participantMap, raceMap) {
  let raceResults = _.chain(legResults)
  //first get leg rankings by doing some cool sorting work
  .sort((r1, r2) => {
    if (r1.raceId !== r2.raceId) { return new Date(r1.start) - new Date(r2.start); }
    if (r1.leg !== r2.leg) { return r1.leg - r2.leg; }
    return resultCompare(r1, r2);
  })
  //rank the individual race legs
  .map((curResult, index, allResults) => {
    let rank = 1;
    if (index !== 0) {
      let prevResult = allResults[index - 1];
      if ((prevResult.raceId === curResult.raceId) && (prevResult.leg === curResult.leg)) {
        //curResult is in the same leg
        if (resultCompare(curResult, prevResult) === 0) {
          rank = prevResult.rank;
        }  else {
          rank = prevResult.rank + 1;
        }
      }
    }
    curResult.rank = rank;
    return curResult;
  })

  //then start to process it into a race result table
  .reduce((races, curLeg) => {
    if (!races.hasOwnProperty(curLeg.raceId)) {
      races[curLeg.raceId] = {};
    }
    if (!races[curLeg.raceId].hasOwnProperty(curLeg.driverId)) {
      races[curLeg.raceId][curLeg.driverId] = {
        driverId: curLeg.driverId,
        driverName: participantMap.get(curLeg.driverId).fullname,
        navigatorName: 'n/a',
        duration: Moment.duration(0).toJSON(),
        rank: null,
        legs: {},
        dnf: false,
        unknownTime: false
      };
    }
    races[curLeg.raceId][curLeg.driverId].legs[curLeg.leg] = _.pick(curLeg,
      ['start','end', 'duration', 'rank', 'dnf', 'unknownTime', 'toScore']);

    Moment.duration(races[curLeg.raceId][curLeg.driverId].duration).add(curLeg.duration);
    if (curLeg.dnf === true) { races[curLeg.raceId][curLeg.driverId].dnf = true; }
    if (curLeg.unknownTime === true) { races[curLeg.raceId][curLeg.driverId].unknownTime = true; }
    return races;
  }, {})

  //create sorted table array of results
  .reduce((resultsMap, raceResult, raceId) => {
    resultsMap[raceId] = _(raceResult)
    .values() //flatten to an array
    .groupBy(getResultCompare)
    .toPairs()
    .sort((g1, g2) => {return g1[0] - g2[0];})
    .reduce((prevResults, curResultGroup) => {
      let rankedResults = _.map(curResultGroup, result => {
        result.rank = prevResults.length + 1;
      });
      return _.concatenate(prevResults, rankedResults);
    }, []);
    return resultsMap;
  }, {})
  .value();

  let retVal = _(Array.from(raceMap.values()))
  .map(race => {
    var raceDetail = _.pick(race,
        ['raceId', 'name', 'theme', 'date', 'start', 'end', 'distance', 'numLegs', 'toScore']);
    let raceId = raceDetail.raceId;
    if (raceResults[raceId] !== undefined) {
      let results = raceResults[raceId];
      raceDetail.results = results;
      raceDetail.time = results[0].duration;
      raceDetail.winner = results[0].driverName;
    }
    return raceDetail;
  })
  .value();
{'fraction_non_alphanumeric': 0.1625222024866785, 'fraction_numerical': 0.00674955595026643, 'mean_word_length': 3.1476979742173112, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5675165', 'n_tokens_mistral': 3903, 'n_tokens_neox': 3806, 'n_words': 762}