<commit_before>"""
endorse_bot.py
author: @mbiokyle29
"""
import tweepy
import datetime
import json
import bisect
import logging

log = logging.getLogger(__name__)

class EndorseBot(object):

    def __init__(self, key, secret, query, days_back, leaderboard_size):

        self.key = key
        self.secret = secret
        self.query = query
        
        # time stuff
        self.since_id = -1
        self.days_back = days_back
        self.last_run = None

        # Create authentication token
        self.auth = tweepy.OAuthHandler(self.key, self.secret)
        self.api = tweepy.API(self.auth)

        # data storage
        self.leaderboard = []
        self.leaderboard_size = leaderboard_size

    def __repr__(self):
        return "<EndorseBot| query:{} | daysback: {}| size: {}>".format(
            self.query, self.days_back, self.leaderboard_size)

    def build_leaderboard(self):
        
        log.info("leaderboard build starting")
        log.info("last build: %s", str(self.last_run))

        if self.last_run != None:
            
            time_diff = (datetime.datetime.now() - self.last_run).total_seconds()
            log.info("Time since last build: %i", time_diff)

            # 15 min update
            if time_diff >= (60 * 15):
                log.info("Not building, happened less than 15 mins ago")
                self.last_run = datetime.datetime.now()
                return self.leaderboard
        try:
            if self.since_id != -1:
                for res in tweepy.Cursor(self.api.search, since_id=self.since_id, 
                                         q=self.query).items():

                    text = res.text

                    if "RT" in text:
                        continue

                    followers = res.author.followers_count
                    author = res.author.screen_name
                    created_at = res.created_at
                    tweet_id = res.id

                    tweet = {
                        'followers': followers,
                        'author': author,
                        'text': text,
                        'tweet_id': tweet_id
                    }

                    time_diff = datetime.datetime.now() - res.created_at

                    if time_diff.days == self.days_back:
                        self.since_id = tweet_id
                        self.add_tweet(tweet)
                        break

                    self.add_tweet(tweet)
            else:
                for res in tweepy.Cursor(self.api.search, since_id=self.since_id, 
                                         q=self.query).items():

                    text = res.text
                    followers = res.author.followers_count
                    author = res.author.screen_name
                    created_at = res.created_at
                    tweet_id = res.id

                    tweet = {
                        'followers': followers,
                        'author': author,
                        'text': text,
                        'tweet_id': tweet_id
                    }

                    time_diff = datetime.datetime.now() - res.created_at

                    if time_diff.days == self.days_back:
                        self.since_id = tweet_id
                        self.add_tweet(tweet)
                        break

                    self.add_tweet(tweet)

        except tweepy.TweepError:
            log.info("Twitter useage cutoff -- sending what we have")
            self.last_run = datetime.datetime.now()
            return self.leaderboard

        self.last_run = datetime.datetime.now()
        return self.leaderboard

    def add_tweet(self, tweet):

        # check if we are at capacity
        if len(self.leaderboard) == self.leaderboard_size:
            last = self.leaderboard[0]

            if last['followers'] < tweet['followers']:
                self.leaderboard.pop(0)

            # if its less than the bottom do nothing
            else:
                return

        # add it
        index_to_add = bisect.bisect(self.leaderboard, tweet['followers'])
        self.leaderboard.insert(index_to_add, tweet)

<commit_msg>Added RT ignore in else block aswell<commit_after>"""
endorse_bot.py
author: @mbiokyle29
"""
import tweepy
import datetime
import json
import bisect
import logging

log = logging.getLogger(__name__)

class EndorseBot(object):

    def __init__(self, key, secret, query, days_back, leaderboard_size):

        self.key = key
        self.secret = secret
        self.query = query
        
        # time stuff
        self.since_id = -1
        self.days_back = days_back
        self.last_run = None

        # Create authentication token
        self.auth = tweepy.OAuthHandler(self.key, self.secret)
        self.api = tweepy.API(self.auth)

        # data storage
        self.leaderboard = []
        self.leaderboard_size = leaderboard_size

    def __repr__(self):
        return "< EndorseBot| query:{} | daysback: {}| size: {} >".format(
            self.query, self.days_back, self.leaderboard_size)

    def build_leaderboard(self):
        
        log.info("leaderboard build starting")
        log.info("last build: %s", str(self.last_run))

        if self.last_run != None:
            
            time_diff = (datetime.datetime.now() - self.last_run).total_seconds()
            log.info("Time since last build: %i", time_diff)

            # 15 min update
            if time_diff >= (60 * 15):
                log.info("Not building, happened less than 15 mins ago")
                self.last_run = datetime.datetime.now()
                return self.leaderboard
        try:
            if self.since_id != -1:
                for res in tweepy.Cursor(self.api.search, since_id=self.since_id, 
                                         q=self.query).items():

                    text = res.text

                    if "RT" in text:
                        continue

                    followers = res.author.followers_count
                    author = res.author.screen_name
                    created_at = res.created_at
                    tweet_id = res.id

                    tweet = {
                        'followers': followers,
                        'author': author,
                        'text': text,
                        'tweet_id': tweet_id
                    }

                    time_diff = datetime.datetime.now() - res.created_at

                    if time_diff.days == self.days_back:
                        self.since_id = tweet_id
                        self.add_tweet(tweet)
                        break

                    self.add_tweet(tweet)
            else:
                for res in tweepy.Cursor(self.api.search, since_id=self.since_id, 
                                         q=self.query).items():

                    text = res.text
                    followers = res.author.followers_count
                    author = res.author.screen_name
                    created_at = res.created_at
                    tweet_id = res.id

                    if "RT" in text:
                        continue

                    tweet = {
                        'followers': followers,
                        'author': author,
                        'text': text,
                        'tweet_id': tweet_id
                    }

                    time_diff = datetime.datetime.now() - res.created_at

                    if time_diff.days == self.days_back:
                        self.since_id = tweet_id
                        self.add_tweet(tweet)
                        break

                    self.add_tweet(tweet)

        except tweepy.TweepError:
            log.info("Twitter useage cutoff -- sending what we have")
            self.last_run = datetime.datetime.now()
            return self.leaderboard

        self.last_run = datetime.datetime.now()
        return self.leaderboard

    def add_tweet(self, tweet):

        # check if we are at capacity
        if len(self.leaderboard) == self.leaderboard_size:
            last = self.leaderboard[0]

            if last['followers'] < tweet['followers']:
                self.leaderboard.pop(0)

            # if its less than the bottom do nothing
            else:
                return

        # add it
        index_to_add = bisect.bisect(self.leaderboard, tweet['followers'])
        self.leaderboard.insert(index_to_add, tweet)


{'fraction_non_alphanumeric': 0.08618250412638528, 'fraction_numerical': 0.0033011082291912286, 'mean_word_length': 1.2809895133100295, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166655', 'n_tokens_mistral': 2362, 'n_tokens_neox': 2148, 'n_words': 485}