<commit_before>import re
import datetime
import requests
import simplejson as json

from django.core.cache import cache


def fetch_tweets(username='codeinthehole'):
    """
    Return tweets but with caching
    """
    key = 'tweets_%s' % username
    tweets = cache.get(key)
    if tweets is None:
        tweets = _fetch_tweets(username)
        cache.set(key, tweets, 300)
    return tweets


def _fetch_tweets(username='codeinthehole'):
    """
    Return a list of tweets for a given user
    """
    url = "https://twitter.com/statuses/user_timeline.json?screen_name=%s" % username
    response = requests.get(url)
    raw_tweets = json.loads(response.content)
    processed_tweets = []
    for tweet in raw_tweets:
        # Ignore replies
        if tweet['text'].startswith('@'):
            continue
        data = {
            'text': htmlify(tweet['text']),
            'date_created': datetime.datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y")}
        processed_tweets.append(data)
    return processed_tweets


urlfinder = re.compile(r"(https?://[^ ]+)")
tweeterfinder = re.compile(r"@(\w+)")<commit_msg>Added error handling to tweet fetcher<commit_after>import re
import datetime
import requests
import simplejson as json

from django.core.cache import cache


def fetch_tweets(username='codeinthehole'):
    """
    Return tweets but with caching
    """
    key = 'tweets_%s' % username
    tweets = cache.get(key)
    if tweets is None:
        tweets = _fetch_tweets(username)
        cache.set(key, tweets, 300)
    return tweets


def _fetch_tweets(username='codeinthehole'):
    """
    Return a list of tweets for a given user
    """
    url = "https://twitter.com/statuses/user_timeline.json?screen_name=%s" % username
    response = requests.get(url)
    raw_tweets = json.loads(response.content)
    if 'error' in raw_tweets:
        return []
    processed_tweets = []
    for tweet in raw_tweets:
        # Ignore replies
        if tweet['text'].startswith('@'):
            continue
        data = {
            'text': htmlify(tweet['text']),
            'date_created': datetime.datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y")}
        processed_tweets.append(data)
    return processed_tweets


urlfinder = re.compile(r"(https?://[^ ]+)")
tweeterfinder = re.compile(r"@(\w+)")
{'fraction_non_alphanumeric': 0.13211554800339848, 'fraction_numerical': 0.00594732370433305, 'mean_word_length': 3.0463917525773194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4641370', 'n_tokens_mistral': 805, 'n_tokens_neox': 765, 'n_words': 175}