<commit_before>import tweepy
import logging


def login(consumer_key, consumer_secret, access_token, access_token_secret):
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)
    return api


def get_since_id(api, screen_name):
    timeline = api.user_timeline(screen_name=screen_name)
    try:
        return timeline[-1].id
    except IndexError:
        return 1


def get_unique_statuses(api, screen_name, since_id, cache):
    logger = logging.getLogger("twitterdedupe.get_unique_statuses")
    logger.debug("%s -- since %s" % (screen_name, since_id))

    gather_results = True
    stati = []
    page = 1

    while gather_results:
        timeline = api.user_timeline(screen_name=screen_name, since_id=since_id, page=page)
        for s in timeline:
            for url in s.entities['urls']:
                expanded_url = url['expanded_url']
                url_count = cache.get(expanded_url, None)
                logger.info("%s - %s -- %s" % (s.id, expanded_url, url_count))
                if url_count is not None or 0:
                    cache.set(expanded_url, url_count+1)
                else:
                    stati.append(s)
                    cache.set(expanded_url, 1)
        if len(timeline) > 0:
            page += 1
        else:
            gather_results = False
    return stati<commit_msg>Added key function, corrected cache set/get behavior<commit_after>import tweepy
import logging


def login(consumer_key, consumer_secret, access_token, access_token_secret):
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)
    return api


def get_since_id(api, screen_name):
    timeline = api.user_timeline(screen_name=screen_name)
    try:
        return timeline[-1].id
    except IndexError:
        return 1


def _key(screen_name, url):
    return "%s_%s" % (screen_name, url)


def get_unique_statuses(api, screen_name, since_id, cache, cache_length=604800):
    logger = logging.getLogger("twitterdedupe.get_unique_statuses")
    logger.debug("%s -- since %s" % (screen_name, since_id))

    gather_results = True
    stati = []
    page = 1

    while gather_results:
        timeline = api.user_timeline(screen_name=screen_name, since_id=since_id, page=page)
        for s in timeline:
            for url in s.entities['urls']:
                expanded_url = url['expanded_url']
                key = _key(screen_name, expanded_url)
                url_count = cache.get(key, None)
                logger.info("%s - %s -- %s" % (s.id, expanded_url, url_count))
                if url_count is None:
                    stati.append(s)
                    cache.set(key, 1, cache_length)
        if len(timeline) > 0:
            page += 1
        else:
            gather_results = False
    return stati
{'fraction_non_alphanumeric': 0.09619306594153637, 'fraction_numerical': 0.006798096532970768, 'mean_word_length': 2.474616292798111, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1317153', 'n_tokens_mistral': 958, 'n_tokens_neox': 917, 'n_words': 201}