<commit_before>const _ = require('lodash');
const fs = require('fs');
const path = require('path');
const chalk = require('chalk');
const puppeteer = require('puppeteer');

const teams = require('../../src/resources/teams.json');

const red = chalk.bold.red;
const green = chalk.bold.green;

const OUTPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../data/polls');

const CURRENT_YEAR = 2019;
const AP_POLL_YEARS = _.range(1936, CURRENT_YEAR + 1);

const unknownTeams = new Set();

process.setMaxListeners(Infinity);

let browser;

const teamNamesMap = {
  Pitt: 'Pittsburgh',
  'Miami (FL)': 'Maimi',
  'Texas Christian': 'TCU',
  SMU: 'Southern Methodist',
  'North Carolina State': 'NC State',
};

const normalizeTeamName = (teamName) => {
  return teamNamesMap[teamName] || teamName;
};

const getText = async (element) => {
  return await (await element.getProperty('textContent')).jsonValue();
};

const parsePollRows = async (pollRows) => {
  const pollResults = [];

  for (const row of pollRows) {
    const headerCells = await row.$$('th');
    if (headerCells.length === 1) {
      const rowValues = [await getText(headerCells[0])];
      const rowCells = await row.$$('td');

      for (const cell of rowCells) {
        let cellText = await getText(cell);
        rowValues.push(cellText.trim());
      }

      const weekIndex = Number(rowValues[0]) - 1;
      const date = rowValues[1];
      const ranking = Number(rowValues[2]);
      const teamNameAndRecord = rowValues[3];
      const previousRanking = rowValues[4] || 'NR';
      const conference = rowValues[6];<commit_msg>Fixed issue with scrape polls script<commit_after>const _ = require('lodash');
const fs = require('fs');
const path = require('path');
const chalk = require('chalk');
const puppeteer = require('puppeteer');

const teams = require('../../src/resources/teams.json');

const red = chalk.bold.red;
const green = chalk.bold.green;

const OUTPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../data/polls');

const CURRENT_YEAR = 2019;
const AP_POLL_YEARS = _.range(1936, CURRENT_YEAR + 1);

const unknownTeams = new Set();

process.setMaxListeners(Infinity);

let browser;

const teamNamesMap = {
  Pitt: 'Pittsburgh',
  'Miami (FL)': 'Maimi',
  'Texas Christian': 'TCU',
  SMU: 'Southern Methodist',
  'North Carolina State': 'NC State',
};

const normalizeTeamName = (teamName) => {
  return teamNamesMap[teamName] || teamName;
};

const getText = async (element) => {
  return await (await element.getProperty('textContent')).jsonValue();
};

const parsePollRows = async (pollRows) => {
  const pollResults = [];

  for (const row of pollRows) {
    const headerCells = await row.$$('th');
    if (headerCells.length === 1) {
      const rowValues = [await getText(headerCells[0])];
      const rowCells = await row.$$('td');

      for (const cell of rowCells) {
        let cellText = await getText(cell);
        rowValues.push(cellText.trim());
      }

      const weekIndex = Number(rowValues[0]) - 1;
      const date = rowValues[1];
      const ranking = Number(rowValues[2]);
      const teamNameAndRecord = rowValues[3];
      const previousRanking = rowValues[4] || 'NR';
      const conference = rowCells.length === 8 ? rowValues[7] : rowValues[6];
{'fraction_non_alphanumeric': 0.15427166564228642, 'fraction_numerical': 0.011677934849416103, 'mean_word_length': 3.924357034795764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3934675', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1030, 'n_words': 267}