<commit_before>from datetime import datetime, timedelta
import json
import logging
import re

from django.conf import settings
import httplib2
import oauth2 as oauth

from leapfrog.models import Object, Account, Person, UserStream, Media, UserReplyStream
import leapfrog.poll.embedlam


log = logging.getLogger(__name__)


def account_for_twitter_user(userdata, person=None):
    try:
        account = Account.objects.get(service='twitter.com', ident=str(userdata['id']))
    except Account.DoesNotExist:
        if person is None:
            avatar = Media(
                width=48,
                height=48,
                image_url=userdata['profile_image_url'],
            )
            avatar.save()
            person = Person(
                display_name=userdata['name'],
                avatar=avatar,
                permalink_url='http://twitter.com/%s' % userdata['screen_name'],
            )
            person.save()
        account = Account(
            service='twitter.com',
            ident=str(userdata['id']),
            display_name=userdata['name'],
            person=person,
            status_background_color=userdata.get('profile_background_color') or '',
            status_background_image_url=userdata.get('profile_background_image_url') or '',
            status_background_tile=userdata.get('profile_background_tile') or '',
        )
        account.save()

    return account


def tweet_html(tweetdata):
    tweet = tweetdata['text']
    if 'entities' not in tweetdata:
        return tweet

    mutations = list()
    for urldata in tweetdata['entities'].get('urls', ()):
        url = urldata['expanded_url'] or urldata['url']
        start, end = urldata['indices']
        text = urldata.get('text', tweet[start:end])
        classattr = 'class="%s" ' % urldata['class'] if 'class' in urldata else ''
        mutations.append({
            'indices': (start, end),
            'html': u"""<a %shref="%s">%s</a>""" % (classattr, url, text),
        })
    for mentiondata in tweetdata['entities'].get('user_mentions', ()):
        mutations.append({
            'indices': mentiondata['indices'],
            'html': u"""@<a href="http://twitter.com/%(screen_name)s" title="%(name)s">%(screen_name)s</a>""" % mentiondata,
        })
    for tagdata in tweetdata['entities'].get('hashtags', ()):
        mutations.append({
            'indices': tagdata['indices'],
            'html': u"""<a href="http://twitter.com/search?q=%%23%(text)s">#%(text)s</a>""" % tagdata,
        })

    # Mutate the tweet from the end, so the replacements don't invalidate the remaining indices.
    for mutation in sorted(mutations, key=lambda x: x['indices'][0], reverse=True):
        indices = mutation['indices']
        tweet = tweet[:indices[0]] + mutation['html'] + tweet[indices[1]:]

    return tweet


url_re = re.compile(r"""
    (?: (?<= \A )
      | (?<= [\s.:;?\-\]<\(] ) )

    https?://
    [-\w;/?:@&=+$.!~*'()%,#]+
    [\w/]

    (?= \Z | [\s\.,!:;?\-\[\]>\)] )
""", re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)

mention_re = re.compile(r"""
    (?: (?<= \A ) | (?<= \s ) )
    @ (\w+)
""", re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)

tag_re = re.compile(r"""
    (?: (?<= \A ) | (?<= \s ) )  # BOT or whitespace
    \# (\w\S*\w)      # tag
    (?<! 's )         # but don't end with 's
""", re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)

def synthesize_entities(tweetdata):
    if 'entities' in tweetdata:
        return

    tweettext = tweetdata['text']
    ents = {
        'urls': [],
        'hashtags': [],
        'user_mentions': [],
    }

    for match in url_re.finditer(tweettext):
        ents['urls'].append({
            'expanded_url': None,
            'url': match.group(0),
            'indices': [match.start(), match.end()],
        })

    for match in mention_re.finditer(tweettext):
        ents['user_mentions'].append({
            'id': None,  # we would have to get this if we ever used it
            'name': match.group(1),  # TODO: get this from the API? ugh
            'screen_name': match.group(1),
            'indices': [match.start(), match.end()],
        })

    for match in tag_re.finditer(tweettext):
        ents['hashtags'].append({
            'text': match.group(1),
            'indices': [match.start(), match.end()],
        })

    tweetdata['entities'] = ents


def object_from_twitpic_url(url):
    mo = re.match(r'http://twitpic\.com/(\w+)', url)
    twitpic_id = mo.group(1)

    try:
        return Object.objects.get(service='twitpic.com', foreign_id=twitpic_id)
    except Object.DoesNotExist:
        pass

    h = httplib2.Http()
    resp, content = h.request('http://api.twitpic.com/2/media/show.json?id=%s' % twitpic_id)

    try:
        picdata = json.loads(content)
    except ValueError:
        # Couldn't get twitpic infos... probably because we're banned.
        return None

    userdata = picdata['user']
    # ugh, why did they rename these
    userdata['id'] = userdata['twitter_id']
    userdata['screen_name'] = userdata['username']
    userdata['profile_image_url'] = userdata['avatar_url']

    pic = Media(
        image_url='http://twitpic.com/show/large/%s' % twitpic_id,
        width=int(picdata['width']),
        height=int(picdata['height']),
    )
    pic.save()
    obj = Object(
        service='twitpic.com',
        foreign_id=twitpic_id,
        render_mode='image',
        title=picdata['message'],
        image=pic,
        author=account_for_twitter_user(userdata),
        time=datetime.strptime(picdata['timestamp'], '%Y-%m-%d %H:%M:%S'),
        permalink_url=url,
    )
    obj.save()<commit_msg>Handle if twitpic gives us a JSON response but one with errors<commit_after>from datetime import datetime, timedelta
import json
import logging
import re

from django.conf import settings
import httplib2
import oauth2 as oauth

from leapfrog.models import Object, Account, Person, UserStream, Media, UserReplyStream
import leapfrog.poll.embedlam


log = logging.getLogger(__name__)


def account_for_twitter_user(userdata, person=None):
    try:
        account = Account.objects.get(service='twitter.com', ident=str(userdata['id']))
    except Account.DoesNotExist:
        if person is None:
            avatar = Media(
                width=48,
                height=48,
                image_url=userdata['profile_image_url'],
            )
            avatar.save()
            person = Person(
                display_name=userdata['name'],
                avatar=avatar,
                permalink_url='http://twitter.com/%s' % userdata['screen_name'],
            )
            person.save()
        account = Account(
            service='twitter.com',
            ident=str(userdata['id']),
            display_name=userdata['name'],
            person=person,
            status_background_color=userdata.get('profile_background_color') or '',
            status_background_image_url=userdata.get('profile_background_image_url') or '',
            status_background_tile=userdata.get('profile_background_tile') or '',
        )
        account.save()

    return account


def tweet_html(tweetdata):
    tweet = tweetdata['text']
    if 'entities' not in tweetdata:
        return tweet

    mutations = list()
    for urldata in tweetdata['entities'].get('urls', ()):
        url = urldata['expanded_url'] or urldata['url']
        start, end = urldata['indices']
        text = urldata.get('text', tweet[start:end])
        classattr = 'class="%s" ' % urldata['class'] if 'class' in urldata else ''
        mutations.append({
            'indices': (start, end),
            'html': u"""<a %shref="%s">%s</a>""" % (classattr, url, text),
        })
    for mentiondata in tweetdata['entities'].get('user_mentions', ()):
        mutations.append({
            'indices': mentiondata['indices'],
            'html': u"""@<a href="http://twitter.com/%(screen_name)s" title="%(name)s">%(screen_name)s</a>""" % mentiondata,
        })
    for tagdata in tweetdata['entities'].get('hashtags', ()):
        mutations.append({
            'indices': tagdata['indices'],
            'html': u"""<a href="http://twitter.com/search?q=%%23%(text)s">#%(text)s</a>""" % tagdata,
        })

    # Mutate the tweet from the end, so the replacements don't invalidate the remaining indices.
    for mutation in sorted(mutations, key=lambda x: x['indices'][0], reverse=True):
        indices = mutation['indices']
        tweet = tweet[:indices[0]] + mutation['html'] + tweet[indices[1]:]

    return tweet


url_re = re.compile(r"""
    (?: (?<= \A )
      | (?<= [\s.:;?\-\]<\(] ) )

    https?://
    [-\w;/?:@&=+$.!~*'()%,#]+
    [\w/]

    (?= \Z | [\s\.,!:;?\-\[\]>\)] )
""", re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)

mention_re = re.compile(r"""
    (?: (?<= \A ) | (?<= \s ) )
    @ (\w+)
""", re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)

tag_re = re.compile(r"""
    (?: (?<= \A ) | (?<= \s ) )  # BOT or whitespace
    \# (\w\S*\w)      # tag
    (?<! 's )         # but don't end with 's
""", re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)

def synthesize_entities(tweetdata):
    if 'entities' in tweetdata:
        return

    tweettext = tweetdata['text']
    ents = {
        'urls': [],
        'hashtags': [],
        'user_mentions': [],
    }

    for match in url_re.finditer(tweettext):
        ents['urls'].append({
            'expanded_url': None,
            'url': match.group(0),
            'indices': [match.start(), match.end()],
        })

    for match in mention_re.finditer(tweettext):
        ents['user_mentions'].append({
            'id': None,  # we would have to get this if we ever used it
            'name': match.group(1),  # TODO: get this from the API? ugh
            'screen_name': match.group(1),
            'indices': [match.start(), match.end()],
        })

    for match in tag_re.finditer(tweettext):
        ents['hashtags'].append({
            'text': match.group(1),
            'indices': [match.start(), match.end()],
        })

    tweetdata['entities'] = ents


def object_from_twitpic_url(url):
    mo = re.match(r'http://twitpic\.com/(\w+)', url)
    twitpic_id = mo.group(1)

    try:
        return Object.objects.get(service='twitpic.com', foreign_id=twitpic_id)
    except Object.DoesNotExist:
        pass

    h = httplib2.Http()
    resp, content = h.request('http://api.twitpic.com/2/media/show.json?id=%s' % twitpic_id)

    try:
        picdata = json.loads(content)
    except ValueError:
        # Couldn't get twitpic infos... probably because we're banned.
        return None
    if picdata.get('errors'):
        # Hmm, well, guess that didn't work.
        return None

    userdata = picdata['user']
    # ugh, why did they rename these
    userdata['id'] = userdata['twitter_id']
    userdata['screen_name'] = userdata['username']
    userdata['profile_image_url'] = userdata['avatar_url']

    pic = Media(
        image_url='http://twitpic.com/show/large/%s' % twitpic_id,
        width=int(picdata['width']),
        height=int(picdata['height']),
    )
    pic.save()
    obj = Object(
        service='twitpic.com',
        foreign_id=twitpic_id,
        render_mode='image',
        title=picdata['message'],
        image=pic,
        author=account_for_twitter_user(userdata),
        time=datetime.strptime(picdata['timestamp'], '%Y-%m-%d %H:%M:%S'),
        permalink_url=url,
    )
    obj.save()
{'fraction_non_alphanumeric': 0.16829013741520266, 'fraction_numerical': 0.0031309793007479564, 'mean_word_length': 2.464597770412775, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1746188', 'n_tokens_mistral': 3845, 'n_tokens_neox': 3724, 'n_words': 724}