<commit_before>'use strict'

const differenceInDays = require('date-fns/difference_in_days')
const debug = require('debug')('fetch-timeline')
const subDays = require('date-fns/sub_days')
const isAfter = require('date-fns/is_after')
const from = require('from2').obj

const int64decrement = require('./int64-decrement')
const twitterClient = require('./twitter-client')
const isEmpty = value => !value || !value.length
const last = arr => arr[arr.length - 1]
const first = arr => arr[0]

const logTweet = (tweet, index) => {
  const screeName = tweet.user.screen_name
  const tweetId = tweet.id_str
  const tweetUrl = `https://twitter.com/${screeName}/status/${tweetId}`

  debug(`
    #${++index}
    id=${tweet.id}
    createdAt=${tweet.created_at}
    url=${tweetUrl}
    text=${tweet.text}
  `)
}

function fetchTimeline (params, opts) {
  const {credentials, limit, limitDays} = opts
  const twitterParams = twitterClient.setupParams(params)
  const fetchChunk = twitterClient.create(credentials)

  const info = {apiCalls: 0, count: 0, timestamp: Date.now()}
  let isFirstChunk = true
  let isUnderLimit = true
  let cursor = false

  const hasFetch = () => {
    return isFirstChunk || (isUnderLimit && cursor)
  }

  function fetchTweets (cb) {
    ++info.apiCalls
    if (cursor) twitterParams.max_id = cursor

    fetchChunk(twitterParams, function (err, chunk, res) {
      debug(`fetchChunk #${info.apiCalls} of ${chunk.length} tweets`)

      if (err) {
        err.headers = res.headers
        return cb(err, res)
      }

      // user without tweets
      // user with protected account
      // user with no more chunk of tweets
      if (isEmpty(chunk)) return cb()

      if (isFirstChunk) {
        const firstChunk = first(chunk)
        info.user = firstChunk.user
        info.newerTweetDate = new Date(firstChunk.created_at)
        isFirstChunk = false
      }

      return cb(null, chunk)
    })
  }

  const stream = from(function (size, next) {
    const finish = () => {
      this.emit('info', info)
      return next(null, null)
    }

    if (!hasFetch()) return finish()

    fetchTweets((err, tweets) => {
      if (err) return next(err)

      // excedeed tweets
      if (limit && info.count + tweets.length > limit) {
        const maxTweets = limit - info.count
        debug(`resizing, limit excedeed by ${maxTweets} tweets`)
        tweets = tweets.slice(0, maxTweets)
        isUnderLimit = false
      }

      // excedeed tweets by date
      if (limitDays) {
        const olderTweetDate = new Date(last(tweets).created_at)
        const diffDays = differenceInDays(info.timestamp, olderTweetDate)

        if (diffDays > limitDays) {
          debug(`resizing, limit excedeed by ${diffDays} days`)
          const markDate = subDays(info.timestamp, limitDays)
          tweets = tweets.filter(tweet => {
            const tweetDate = new Date(tweet.created_at)
            return isAfter(tweetDate, markDate)
          })
          isUnderLimit = false
        }
      }

      if (isEmpty(tweets)) return finish()

      const lastTweet = tweets.pop()
      tweets.forEach(tweet => {
        logTweet(tweet, info.count++)
        this.push(tweet)
      })

      if (isUnderLimit) cursor = int64decrement(lastTweet.id_str)
      info.olderTweetDate = new Date(lastTweet.created_at)<commit_msg>Ensure tweets is not empty<commit_after>'use strict'

const differenceInDays = require('date-fns/difference_in_days')
const debug = require('debug')('fetch-timeline')
const subDays = require('date-fns/sub_days')
const isAfter = require('date-fns/is_after')
const from = require('from2').obj

const int64decrement = require('./int64-decrement')
const twitterClient = require('./twitter-client')
const isEmpty = value => !value || !value.length
const last = arr => arr[arr.length - 1]
const first = arr => arr[0]

const logTweet = (tweet, index) => {
  const screeName = tweet.user.screen_name
  const tweetId = tweet.id_str
  const tweetUrl = `https://twitter.com/${screeName}/status/${tweetId}`

  debug(`
    #${++index}
    id=${tweet.id}
    createdAt=${tweet.created_at}
    url=${tweetUrl}
    text=${tweet.text}
  `)
}

function fetchTimeline (params, opts) {
  const {credentials, limit, limitDays} = opts
  const twitterParams = twitterClient.setupParams(params)
  const fetchChunk = twitterClient.create(credentials)

  const info = {apiCalls: 0, count: 0, timestamp: Date.now()}
  let isFirstChunk = true
  let isUnderLimit = true
  let cursor = false

  const hasFetch = () => {
    return isFirstChunk || (isUnderLimit && cursor)
  }

  function fetchTweets (cb) {
    ++info.apiCalls
    if (cursor) twitterParams.max_id = cursor

    fetchChunk(twitterParams, function (err, chunk, res) {
      debug(`fetchChunk #${info.apiCalls} of ${chunk.length} tweets`)

      if (err) {
        err.headers = res.headers
        return cb(err, res)
      }

      // user without tweets
      // user with protected account
      // user with no more chunk of tweets
      if (isEmpty(chunk)) return cb()

      if (isFirstChunk) {
        const firstChunk = first(chunk)
        info.user = firstChunk.user
        info.newerTweetDate = new Date(firstChunk.created_at)
        isFirstChunk = false
      }

      return cb(null, chunk)
    })
  }

  const stream = from(function (size, next) {
    const finish = () => {
      this.emit('info', info)
      return next(null, null)
    }

    if (!hasFetch()) return finish()

    fetchTweets((err, tweets) => {
      if (err) return next(err)

      // excedeed tweets
      if (limit && info.count + tweets.length > limit) {
        const maxTweets = limit - info.count
        debug(`resizing, limit excedeed by ${maxTweets} tweets`)
        tweets = tweets.slice(0, maxTweets)
        isUnderLimit = false
      }

      // excedeed tweets by date
      if (!isEmpty(tweets) && limitDays) {
        const olderTweetDate = new Date(last(tweets).created_at)
        const diffDays = differenceInDays(info.timestamp, olderTweetDate)

        if (diffDays > limitDays) {
          debug(`resizing, limit excedeed by ${diffDays} days`)
          const markDate = subDays(info.timestamp, limitDays)
          tweets = tweets.filter(tweet => {
            const tweetDate = new Date(tweet.created_at)
            return isAfter(tweetDate, markDate)
          })
          isUnderLimit = false
        }
      }

      if (isEmpty(tweets)) return finish()

      const lastTweet = tweets.pop()
      tweets.forEach(tweet => {
        logTweet(tweet, info.count++)
        this.push(tweet)
      })

      if (isUnderLimit) cursor = int64decrement(lastTweet.id_str)
      info.olderTweetDate = new Date(lastTweet.created_at)
{'fraction_non_alphanumeric': 0.12319380306867272, 'fraction_numerical': 0.0035751526888127515, 'mean_word_length': 2.799660441426146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7503905', 'n_tokens_mistral': 2180, 'n_tokens_neox': 2074, 'n_words': 508}