<commit_before>import argparse
import time
import tweepy

import botornotapi
import db
from config.keys import myauth
from config.test_candidates import SLUGS, FOLLOWER_LIMIT

AUTH = tweepy.OAuthHandler(myauth['consumer_key'], myauth['consumer_secret'])
AUTH.set_access_token(myauth['access_token'], myauth['access_token_secret'])
TWITTER_API = tweepy.API(AUTH, wait_on_rate_limit=True, wait_on_rate_limit_notify=True,\
                         retry_count=3, retry_delay=5, retry_errors=set([401, 404, 500, 503]))

def limit_handled(cursor):
    while True:
        try:
            yield cursor.next()
        except tweepy.RateLimitError:
            timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
            print('Warning: Rate limit reached! ' + timestamp)
            time.sleep(15 * 60)

def save_tweets_with_retweets(screen_name):
    timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
    print(timestamp)
    for tweet in limit_handled(tweepy.Cursor(TWITTER_API.user_timeline, id=screen_name, count=200).items()):
        retweets = get_retweets(tweet.id)
        db.saveRetweets(tweet, retweets)

def save_followers_with_botness(account_handle):
    followers = get_followers("@" + account_handle)
    user = TWITTER_API.get_user(account_handle)
    timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
    print(timestamp)
    print("Save user %s and the followers ..." % account_handle)
    db.saveUser(user)
    for follower in followers:
        follower_handle = follower.screen_name
        if db.hasFollower(fromName=follower_handle, toName=account_handle):
            print("Already checked @" + follower_handle + " ... skipping for now.")
            continue
        follower_botness = botornotapi.get_bot_or_not("@" + follower_handle)
        if follower_botness is not None:
            db.saveFollower(user, follower, follower_botness)
            print("Saved follower @" + follower_handle + " for @" + account_handle + " with botness: " + str(follower_botness))
        else:
            print("Botness is none for @" + follower_handle)

def get_retweets(tweet_id):
    timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
    print(timestamp)
    retweets = []
    for retweet in TWITTER_API.retweets(id=tweet_id, count=200):
        retweets.append(retweet)
    return retweets
<commit_msg>Improve logging of twitter api scraping runs a bit<commit_after>import argparse
import time
import tweepy

import botornotapi
import db
from config.keys import myauth
from config.test_candidates import SLUGS, FOLLOWER_LIMIT

AUTH = tweepy.OAuthHandler(myauth['consumer_key'], myauth['consumer_secret'])
AUTH.set_access_token(myauth['access_token'], myauth['access_token_secret'])
TWITTER_API = tweepy.API(AUTH, wait_on_rate_limit=True, wait_on_rate_limit_notify=True,\
                         retry_count=3, retry_delay=5, retry_errors=set([401, 404, 500, 503]))

def limit_handled(cursor):
    while True:
        try:
            yield cursor.next()
        except tweepy.RateLimitError:
            timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
            print('Warning: Rate limit reached! ' + timestamp)
            time.sleep(15 * 60)

def save_tweets_with_retweets(screen_name):
    timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
    print(timestamp)
    for tweet in limit_handled(tweepy.Cursor(TWITTER_API.user_timeline, id=screen_name, count=200).items()):
        retweets = get_retweets(tweet.id)
        db.saveRetweets(tweet, retweets)

def save_followers_with_botness(account_handle):
    followers = get_followers("@" + account_handle)
    user = TWITTER_API.get_user(account_handle)
    timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
    print(timestamp)
    print("Save user @%s and the followers (twitter reported %d)  ..." % (account_handle, user.followers_count))
    db.saveUser(user)
    for follower in followers:
        follower_handle = follower.screen_name
        if db.hasFollower(fromName=follower_handle, toName=account_handle):
            print("Already checked @" + follower_handle + " ... skipping for now.")
            continue
        follower_botness = botornotapi.get_bot_or_not("@" + follower_handle)
        if follower_botness is not None:
            db.saveFollower(user, follower, follower_botness)
            print("Saved follower @%s for @%s with botness %f" % (follower_handle, account_handle, follower_botness['score']))
        else:
            print("Botness is none for @" + follower_handle)

def get_retweets(tweet_id):
    timestamp = time.strftime("%d.%m.%Y %H:%M:%S", time.localtime())
    print(timestamp)
    retweets = []
    for retweet in TWITTER_API.retweets(id=tweet_id, count=200):
        retweets.append(retweet)
    return retweets

{'fraction_non_alphanumeric': 0.12112443158329889, 'fraction_numerical': 0.009921455146754857, 'mean_word_length': 3.6753623188405795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '486995', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1557, 'n_words': 311}