<commit_before>import json
import time
import urllib2

from django.core.management.base import BaseCommand, CommandError
from django.db.utils import IntegrityError
from django.conf import settings

from bioguide.models import *


class Command(BaseCommand):

    def handle(self, *args, **options):
        bioguide_ids = Legislator.objects.order_by('bioguide_id').values_list('bioguide_id', flat=True).distinct()
        for bioguide_id in bioguide_ids:
            # if LegislatorRole.objects.filter(bioguide_id=bioguide_id).count() > 0:
            #     continue

            url = 'http://api.nytimes.com/svc/politics/v3/us/legislative/congress/members/%s.json?api-key=%s' % (bioguide_id, settings.NYT_API_KEY)
            try:
                data = json.loads(urllib2.urlopen(url).read())
            except urllib2.HTTPError:
                print
                print 'URL NOT FOUND FOR: %s' % bioguide_id
                print
                continue
            result = data['results'][0]
            roles = result['roles']
            for role in roles:
                print role
                try:
                    LegislatorRole.objects.get_or_create(
                            bioguide_id=bioguide_id,
                            first=result['first_name'],
                            middle=result['middle_name'],
                            last=result['last_name'],
                            congress=role['congress'],
                            chamber=role['chamber'],
                            party=role['party'],
                            title=role['title'],
                            state=role['state'],
                            district=role['district'],
                            begin_date=role['start_date'],
                            end_date=role['end_date'])
                except IntegrityError:<commit_msg>take congress flag for role importer<commit_after>import json
import time
import urllib2

from django.core.management.base import BaseCommand, CommandError
from django.db.utils import IntegrityError
from django.conf import settings

from bioguide.models import *


class Command(BaseCommand):

    def handle(self, *args, **options):
        try:
            congress = args[0]
        except IndexError:
            congress = None
        bioguide_ids = Legislator.objects.order_by('bioguide_id').values_list('bioguide_id', flat=True).distinct()
        for bioguide_id in bioguide_ids:
            # if LegislatorRole.objects.filter(bioguide_id=bioguide_id).count() > 0:
            #     continue

            url = 'http://api.nytimes.com/svc/politics/v3/us/legislative/congress/members/%s.json?api-key=%s' % (bioguide_id, settings.NYT_API_KEY)
            try:
                data = json.loads(urllib2.urlopen(url).read())
            except urllib2.HTTPError:
                print
                print 'URL NOT FOUND FOR: %s' % bioguide_id
                print
                continue
            result = data['results'][0]
            roles = result['roles']
            for role in roles:
                if congress and int(role['congress']) != int(congress):
                    continue
                print role
                try:
                    LegislatorRole.objects.get_or_create(
                            bioguide_id=bioguide_id,
                            first=result['first_name'],
                            middle=result['middle_name'],
                            last=result['last_name'],
                            congress=role['congress'],
                            chamber=role['chamber'],
                            party=role['party'],
                            title=role['title'],
                            state=role['state'],
                            district=role['district'],
                            begin_date=role['start_date'],
                            end_date=role['end_date'])
                except IntegrityError:
{'fraction_non_alphanumeric': 0.09709962168978563, 'fraction_numerical': 0.003278688524590164, 'mean_word_length': 1.516497461928934, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '975794', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1058, 'n_words': 185}