<commit_before>var debug = require('debug');
log = debug('bot'),
logPoll = debug('bot:poll'),
logYoutube = debug('bot:youtube'),
logReddit = debug('bot:reddit'),
logYoutubeSetup = debug('bot:youtube:setup'),
logRedditSetup = debug('bot:reddit:setup'),
logYoutubePoll = debug('bot:youtube:poll'),
logRedditPoll = debug('bot:reddit:poll'),
logRedditPost = debug('bot:reddit:post');
log('Starting requires');
var config = require('./config');
var Snoocore = require('snoocore');
var gapi = require('googleapis');
var youtube = gapi.youtube('v3');
var async = require('async');
var moment = require('moment');
// var util = require('util');
log('Finished');

var reddit = new Snoocore({
  userAgent: config.userAgent,
  login: {username: config.reddit.username, password: <PASSWORD>},
  oauth: {
    type: 'script',
    duration: "permanent",
    consumerKey: config.reddit.oauth.consumerKey,
    consumerSecret: config.reddit.oauth.consumerSecret,
    scope: config.reddit.scope
  }
});

function setupReddit(cb) {
  logRedditSetup("Started to set up Reddit");
  reddit.auth().then(function() {
    return reddit('/api/needs_captcha.json').get();
 }, function(error){
    logRedditSetup("Erred in setting up Reddit: Auth: " + error)
    if (cb) {
      cb(error);
    }
    return null;
 }).then(function(needs_captcha){
    if (!cb) {
      logRedditSetup("No call back, got " + needs_captcha + ' for testing.');
      return;
    }
    if (needs_captcha) {
      cb("Error: I've been hit with captchas!!!!");
      return;
    }
    logRedditSetup("Finished setting up Reddit.")
    cb(null, !needs_captcha);
    // find where to pin the after mark here.
    // redditHoneInOnTime(null, null, cb);
    // reddit('/r/' + config.reddit.subreddit + '/top').get({count: 5, t: 'hour'}).then(function(data){
    //   cb(null, JSON.stringify(data));
    // }, cb);
  }, function(error) {
    logRedditSetup("Erred in setting up Reddit: Checking captchas: " + error)
    if (cb) {
      cb(error);
    }
  })
};

var redditTokenInterval = setInterval(setupReddit, 55 * 60 * 1000);

// function redditHoneInOnTime(after, before, cb) {
//   opts = {}
//   if (typeof after == 'string') {
//     opts.after = after;
//   }
//   if (typeof before == 'string') {
//     opts.before = before;
//   }
//   reddit('/r/' + config.reddit.subreddit + '/top').get({count: 2, t: 'week'}).then(function(data){
//     // console.log(util.inspect(data.data.children[0].data));
//     cb(null,data);
//   }, function(error) {
//     cb(error);
//   })
// }

function setupYoutube(cb) {
  logYoutubeSetup('Started to set up YouTube')
  gapi.options({auth: config.youtube.apiKey})
  youtube.channels.list({part: 'contentDetails', forUsername: config.youtube.channel}, function getChannelDetails (err, response) {
    if (err) {
      cb("YouTube Error: " + err);
      return;
    }
    if (response.pageInfo.totalResults != 1) {
      logYoutubeSetup(response);
      cb("Well poop. Got " + response.pageInfo.totalResults + " channels some how, expected 1.");
      return;
    }
    var res = response.items[0].id;
    logYoutubeSetup("Finished setting up Youtube.")
    cb(null, res);
  });
};

function eliminateDuplicates(arr) {
  var i,
      len=arr.length,
      out=[],
      obj={};
  for (i=0;i<len;i++) {
    obj[arr[i]]=0;
  }
  for (i in obj) {
    out.push(i);
  }
  return out;
}

function pollYouTube(cb) {
  logYoutubePoll("Started polling YouTube");
  youtube.activities.list({publishedAfter: moment().subtract(1, 'hour').add(5, 'minutes').toISOString() ,part: 'contentDetails,snippet', channelId: channelId, maxResults: 50}, function (err, results) {
    if (err) {
      logYoutubePoll("YouTube polling failed at fetching play list items")
      cb("YouTube Error: " + err);
      return;
    }
    var data = results.items.filter(function(item){
      return item.snippet.type === "upload" || item.snippet.type === "playlistItem";
    }).map(function(item){
      if (item.contentDetails.upload !== undefined) {
        return item.contentDetails.upload.videoId;
      } else if (item.contentDetails.playlistItem !== undefined) {
        return item.contentDetails.playlistItem.resourceId.videoId;
      }
      // return item.contentDetails.videoId;
    })
    data = eliminateDuplicates(data);
    // logYoutubePoll(data);
    youtube.videos.list({part: 'snippet,contentDetails', fields: "items(contentDetails,id,snippet)", id: data.join(',')}, function(err, results) {
      if (err) {
        logYoutubePoll("YouTube polling failed at getting content Details")
        cb(err);
        return;
      }
      var time = moment().subtract(1, 'hour').add(5, 'minutes');
      var finalResults = results.items.filter(function(item){
        return item.snippet.channelId === channelId; // For sanity's sake. Blame YouTube.
      }).map(function(item){
        logYoutubePoll(item.snippet.title + " - [" + formatTime(item.contentDetails.duration) + "]");
        return {
          length: item.contentDetails.duration,
          title: item.snippet.title,
          id: item.id,
          publishedAt: item.snippet.publishedAt
        }
      })
      logYoutubePoll("Finished polling YouTube")
      cb(err, finalResults);
    })
  })
}
// TODO: make sure that I do get every thing in the past hour.
function pollReddit(cb) {
  logRedditPoll("Started polling Reddit.");
  // cb(null,"Poll Reddit is unfinished!");
  // return null;
  reddit('/r/' + config.reddit.subreddit + '/new').get({limit: config.reddit.limitResults, /*count: 1,*/ t: 'hour'}).then(function(data){
    // logRedditPoll(data.data);
    var results = data.data.children.map(function (item) {
      return item.data
    }).filter(function(item){
      logRedditPoll(item.title + " - " + moment.unix(item.created_utc).fromNow() + ' - ' + item.name);
      // logRedditPoll(item)
      return item.domain.match(/youtu(be\.(com|ca)|\.be)/)
    }).map(function (item){
      return item.url.match(/^https?:\/\/(?:www.)?youtu(?:be.(?:(?:com|ca)\/watch\?v=)|\.be\/)(.+)$/im)[1];
    })
    logRedditPoll("Finished polling Reddit");
    cb(null, results);
  }, function(error) {
    logRedditPoll("Reddit Polling failed!")
    cb(error);<commit_msg>Tone down the logging in pollReddit function.<commit_after>var debug = require('debug');
log = debug('bot'),
logPoll = debug('bot:poll'),
logYoutube = debug('bot:youtube'),
logReddit = debug('bot:reddit'),
logYoutubeSetup = debug('bot:youtube:setup'),
logRedditSetup = debug('bot:reddit:setup'),
logYoutubePoll = debug('bot:youtube:poll'),
logRedditPoll = debug('bot:reddit:poll'),
logRedditPost = debug('bot:reddit:post');
log('Starting requires');
var config = require('./config');
var Snoocore = require('snoocore');
var gapi = require('googleapis');
var youtube = gapi.youtube('v3');
var async = require('async');
var moment = require('moment');
// var util = require('util');
log('Finished');

var reddit = new Snoocore({
  userAgent: config.userAgent,
  login: {username: config.reddit.username, password: <PASSWORD>},
  oauth: {
    type: 'script',
    duration: "permanent",
    consumerKey: config.reddit.oauth.consumerKey,
    consumerSecret: config.reddit.oauth.consumerSecret,
    scope: config.reddit.scope
  }
});

function setupReddit(cb) {
  logRedditSetup("Started to set up Reddit");
  reddit.auth().then(function() {
    return reddit('/api/needs_captcha.json').get();
 }, function(error){
    logRedditSetup("Erred in setting up Reddit: Auth: " + error)
    if (cb) {
      cb(error);
    }
    return null;
 }).then(function(needs_captcha){
    if (!cb) {
      logRedditSetup("No call back, got " + needs_captcha + ' for testing.');
      return;
    }
    if (needs_captcha) {
      cb("Error: I've been hit with captchas!!!!");
      return;
    }
    logRedditSetup("Finished setting up Reddit.")
    cb(null, !needs_captcha);
    // find where to pin the after mark here.
    // redditHoneInOnTime(null, null, cb);
    // reddit('/r/' + config.reddit.subreddit + '/top').get({count: 5, t: 'hour'}).then(function(data){
    //   cb(null, JSON.stringify(data));
    // }, cb);
  }, function(error) {
    logRedditSetup("Erred in setting up Reddit: Checking captchas: " + error)
    if (cb) {
      cb(error);
    }
  })
};

var redditTokenInterval = setInterval(setupReddit, 55 * 60 * 1000);

// function redditHoneInOnTime(after, before, cb) {
//   opts = {}
//   if (typeof after == 'string') {
//     opts.after = after;
//   }
//   if (typeof before == 'string') {
//     opts.before = before;
//   }
//   reddit('/r/' + config.reddit.subreddit + '/top').get({count: 2, t: 'week'}).then(function(data){
//     // console.log(util.inspect(data.data.children[0].data));
//     cb(null,data);
//   }, function(error) {
//     cb(error);
//   })
// }

function setupYoutube(cb) {
  logYoutubeSetup('Started to set up YouTube')
  gapi.options({auth: config.youtube.apiKey})
  youtube.channels.list({part: 'contentDetails', forUsername: config.youtube.channel}, function getChannelDetails (err, response) {
    if (err) {
      cb("YouTube Error: " + err);
      return;
    }
    if (response.pageInfo.totalResults != 1) {
      logYoutubeSetup(response);
      cb("Well poop. Got " + response.pageInfo.totalResults + " channels some how, expected 1.");
      return;
    }
    var res = response.items[0].id;
    logYoutubeSetup("Finished setting up Youtube.")
    cb(null, res);
  });
};

function eliminateDuplicates(arr) {
  var i,
      len=arr.length,
      out=[],
      obj={};
  for (i=0;i<len;i++) {
    obj[arr[i]]=0;
  }
  for (i in obj) {
    out.push(i);
  }
  return out;
}

function pollYouTube(cb) {
  logYoutubePoll("Started polling YouTube");
  youtube.activities.list({publishedAfter: moment().subtract(1, 'hour').add(5, 'minutes').toISOString() ,part: 'contentDetails,snippet', channelId: channelId, maxResults: 50}, function (err, results) {
    if (err) {
      logYoutubePoll("YouTube polling failed at fetching play list items")
      cb("YouTube Error: " + err);
      return;
    }
    var data = results.items.filter(function(item){
      return item.snippet.type === "upload" || item.snippet.type === "playlistItem";
    }).map(function(item){
      if (item.contentDetails.upload !== undefined) {
        return item.contentDetails.upload.videoId;
      } else if (item.contentDetails.playlistItem !== undefined) {
        return item.contentDetails.playlistItem.resourceId.videoId;
      }
      // return item.contentDetails.videoId;
    })
    data = eliminateDuplicates(data);
    // logYoutubePoll(data);
    youtube.videos.list({part: 'snippet,contentDetails', fields: "items(contentDetails,id,snippet)", id: data.join(',')}, function(err, results) {
      if (err) {
        logYoutubePoll("YouTube polling failed at getting content Details")
        cb(err);
        return;
      }
      var time = moment().subtract(1, 'hour').add(5, 'minutes');
      var finalResults = results.items.filter(function(item){
        return item.snippet.channelId === channelId; // For sanity's sake. Blame YouTube.
      }).map(function(item){
        logYoutubePoll(item.snippet.title + " - [" + formatTime(item.contentDetails.duration) + "]");
        return {
          length: item.contentDetails.duration,
          title: item.snippet.title,
          id: item.id,
          publishedAt: item.snippet.publishedAt
        }
      })
      logYoutubePoll("Finished polling YouTube")
      cb(err, finalResults);
    })
  })
}
// TODO: make sure that I do get every thing in the past hour.
function pollReddit(cb) {
  logRedditPoll("Started polling Reddit.");
  // cb(null,"Poll Reddit is unfinished!");
  // return null;
  reddit('/r/' + config.reddit.subreddit + '/new').get({limit: config.reddit.limitResults, /*count: 1,*/ t: 'hour'}).then(function(data){
    // logRedditPoll(data.data);
    var results = data.data.children.map(function (item) {
      return item.data
    }).filter(function(item){
      // logRedditPoll(item.title + " - " + moment.unix(item.created_utc).fromNow() + ' - ' + item.name);
      // logRedditPoll(item)
      return item.domain.match(/youtu(be\.(com|ca)|\.be)/)
    }).map(function (item){
      return item.url.match(/^https?:\/\/(?:www.)?youtu(?:be.(?:(?:com|ca)\/watch\?v=)|\.be\/)(.+)$/im)[1];
    })
    logRedditPoll("Found " + results.length + " item(s) of interest.");
    logRedditPoll("Finished polling Reddit");
    cb(null, results);
  }, function(error) {
    logRedditPoll("Reddit Polling failed!")
    cb(error);
{'fraction_non_alphanumeric': 0.1741353143222302, 'fraction_numerical': 0.003993929227574087, 'mean_word_length': 3.317241379310345, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6447949', 'n_tokens_mistral': 4192, 'n_tokens_neox': 4113, 'n_words': 836}