<commit_before>'use strict';

module.exports = function(league, testObj) {

  const config = require('./config');
  const twitter = require('./lib/twitter')(config, league.leagueName);
  const request = require('./lib/request');
  const _ = require('lodash');
  const moment = require('moment');

  const fetchNextFromWeb = function() {
    return league.urls().then((urls) => {
      const requests = urls.map((url) => {
        return request.get(url);
      });
      return Promise.all(requests).then((res) => {
        const next = league.Scores.parse(res);
        return Promise.resolve(next);
      });
    });
  };

  const final = (closeDB, db, res) => {
    return (closeDB ? db.end() : Promise.resolve()).then(() => {
      return Promise.resolve(res);
    });
  };

  const getFromStore = (db, allTime) => {
    let closeDB = false;
    if (!db) {
      db = require('./lib/db')();
      closeDB = true;
    }
    const sql = `
    select * from score_tweet.Leagues t1
    where League = ?
    and not exists (
      select 1 from score_tweet.Leagues t2
      where t2.League = t1.League
      and t2.id > t1.id
    );`;
    return db.query(sql, [league.leagueName]).then((res) => {
      if (!res[0]) { return final(closeDB, db, null); }

      const createdOn = moment.unix(res[0].CreatedOn);
      const timeSinceLast = moment.duration(moment().diff(createdOn));
      if (timeSinceLast.asMinutes() >= 20 && !allTime && !league.occasionalFetch) {
        console.log('Skipping because existing item is too old');
        return final(closeDB, db, null);
      }
      const scores = JSON.parse(res[0].Data);<commit_msg>fix dumb query<commit_after>'use strict';

module.exports = function(league, testObj) {

  const ENABLE_LOGGING = false;
  const log = (msg) => {
    if (ENABLE_LOGGING) {
      console.log(msg);
    }
  };

  const config = require('./config');
  const twitter = require('./lib/twitter')(config, league.leagueName);
  const request = require('./lib/request');
  const _ = require('lodash');
  const moment = require('moment');

  const fetchNextFromWeb = function() {
    log('Fetching scores from league');
    return league.urls().then((urls) => {
      const requests = urls.map((url) => {
        return request.get(url);
      });
      return Promise.all(requests).then((res) => {
        const next = league.Scores.parse(res);
        return Promise.resolve(next);
      });
    });
  };

  const final = (closeDB, db, res) => {
    return (closeDB ? db.end() : Promise.resolve()).then(() => {
      return Promise.resolve(res);
    });
  };

  const getFromStore = (db, allTime) => {
    let closeDB = false;
    if (!db) {
      db = require('./lib/db')();
      closeDB = true;
    }
    const sql = `
    select * from score_tweet.Leagues t1
    where League = ?
    order by id desc limit 1;`;
    log('Fetching scores from database');
    return db.query(sql, [league.leagueName]).then((res) => {
      if (!res[0]) { return final(closeDB, db, null); }

      const createdOn = moment.unix(res[0].CreatedOn);
      const timeSinceLast = moment.duration(moment().diff(createdOn));
      if (timeSinceLast.asMinutes() >= 20 && !allTime && !league.occasionalFetch) {
        console.log('Skipping because existing item is too old');
        return final(closeDB, db, null);
      }
      const scores = JSON.parse(res[0].Data);
{'fraction_non_alphanumeric': 0.1651348947524459, 'fraction_numerical': 0.0056329676845538095, 'mean_word_length': 2.8081264108352144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2031780', 'n_tokens_mistral': 1079, 'n_tokens_neox': 1052, 'n_words': 244}