<commit_before>import email
import datetime
import re

from django.conf import settings
import twitter

from parliament.core.models import Politician, PoliticianInfo
from parliament.activity import utils as activity

import logging
logger = logging.getLogger(__name__)

def save_tweets():
    twitter_to_pol = dict([(int(i.value), i.politician) for i in PoliticianInfo.objects.filter(schema='twitter_id').select_related('politician')])
    screen_names = set(PoliticianInfo.objects.filter(schema='twitter').values_list('value', flat=True))
    twit = twitter.Twitter(domain='api.twitter.com/1')
    statuses = twit.openparlca.lists.mps.statuses(per_page=200)
    statuses.reverse()
    for status in statuses:
        try:
            pol = twitter_to_pol[status['user']['id']]
        except KeyError:
            logger.error("Can't find twitter ID %s (name %s)" 
                % (status['user']['id'], status['user']['screen_name']))
            continue
        if status['user']['screen_name'] not in screen_names:
            # Changed screen name
            pol.set_info('twitter', status['user']['screen_name'])
        date = datetime.date.fromtimestamp(
            email.utils.mktime_tz(
                email.utils.parsedate_tz(status['created_at'])
            )
        ) # fuck you, time formats
        guid = 'twit_%s' % status['id']<commit_msg>Twitter API v1.1 migration<commit_after>import email
import datetime
import re

from django.conf import settings
import twitter

from parliament.core.models import Politician, PoliticianInfo
from parliament.activity import utils as activity

import logging
logger = logging.getLogger(__name__)

def save_tweets():
    twitter_to_pol = dict([(int(i.value), i.politician) for i in PoliticianInfo.objects.filter(schema='twitter_id').select_related('politician')])
    screen_names = set(PoliticianInfo.objects.filter(schema='twitter').values_list('value', flat=True))
    twit = twitter.Twitter(
        auth=twitter.OAuth(**settings.TWITTER_OAUTH), domain='api.twitter.com/1.1')

    statuses = twit.lists.statuses(slug='mps', owner_screen_name='openparlca', include_rts=False, count=200)
    statuses.reverse()
    for status in statuses:
        try:
            pol = twitter_to_pol[status['user']['id']]
        except KeyError:
            logger.error("Can't find twitter ID %s (name %s)" 
                % (status['user']['id'], status['user']['screen_name']))
            continue
        if status['user']['screen_name'] not in screen_names:
            # Changed screen name
            pol.set_info('twitter', status['user']['screen_name'])
        date = datetime.date.fromtimestamp(
            email.utils.mktime_tz(
                email.utils.parsedate_tz(status['created_at'])
            )
        ) # fuck you, time formats
        guid = 'twit_%s' % status['id']
{'fraction_non_alphanumeric': 0.12789880534082923, 'fraction_numerical': 0.003865073787772312, 'mean_word_length': 3.3136363636363635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100208', 'n_tokens_mistral': 844, 'n_tokens_neox': 818, 'n_words': 175}