<commit_before>#### encoding: cp1255 ####

import urllib2
import re
import logging

logger = logging.getLogger("open-knesset.parse_committee_members")

from mks.models import Member, Knesset
from committees.models import Committee
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    args = ''
    help = 'Parses commitee members from the Knesset website'

    def parse_committee_members(self):
        retval = {}

        current_knesset = Knesset.objects.current_knesset().number
        url = 'http://knesset.gov.il/committees/heb/CommitteeHistoryByKnesset.asp?knesset={}'.format(current_knesset)

        data = urllib2.urlopen(url).read()

        member_re = re.compile("/mk/heb/mk\.asp\?mk_individual_id_t=(\d+)")
        com_re = re.compile('/committees/heb/CommitteeHistoryByCommittee.asp\?com')
        com_name_re = re.compile('<b>([^<]+)</b>')

        # by committees:
        data = com_re.split(data)[1:]

        for comdata in data:

            com_name = com_name_re.findall(comdata)[0].decode('cp1255')
            # print com_name

            sections = comdata.split('<BR>')
            chairpersons = members = replacements = []
            for section in sections:
                if '\xe7\xe1\xf8\xe9 \xe4\xe5\xe5\xf2\xe3\xe4' in section:
                    members = member_re.findall(section)
                elif '\xe9\xe5\xf9\xe1/\xfa \xf8\xe0\xf9 \xe4\xe5\xe5\xf2\xe3\xe4' in section:
                    chairpersons = member_re.findall(section)
                elif '\xee\xee\xec\xe0\xe9 \xee\xf7\xe5\xed \xe1\xe5\xe5\xf2\xe3\xe4' in section:
                    replacements = member_re.findall(section)

            chairpersons = set(chairpersons)
            members = set(members)
            replacements = set(replacements)
            members = members - chairpersons - replacements

            # print chairmen, members, replacements
            retval[com_name] = (list(chairpersons), list(members), list(replacements))

        return retval

    def convert_to_mks(self, ll):
        ll = [int(x) for x in ll]
        ll2 = []
        for x in ll:
            try:
                ll2.append(Member.objects.get(id=x))
            except Member.DoesNotExist:
                logger.warn("ERROR: couldn't find member for id: %s" % x)
                # raise
        return ll

    def update_committee_members_db(self, data):
        for name, details in data.iteritems():
            chairpersons, members, replacements = details<commit_msg>modern python<commit_after>#### encoding: cp1255 ####

import urllib2
import re
import logging

logger = logging.getLogger("open-knesset.parse_committee_members")

from mks.models import Member, Knesset
from committees.models import Committee
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    args = ''
    help = 'Parses commitee members from the Knesset website'

    def parse_committee_members(self):
        retval = {}

        current_knesset = Knesset.objects.current_knesset().number
        url = 'http://knesset.gov.il/committees/heb/CommitteeHistoryByKnesset.asp?knesset={}'.format(current_knesset)

        data = urllib2.urlopen(url).read()

        member_re = re.compile("/mk/heb/mk\.asp\?mk_individual_id_t=(\d+)")
        com_re = re.compile('/committees/heb/CommitteeHistoryByCommittee.asp\?com')
        com_name_re = re.compile('<b>([^<]+)</b>')

        # by committees:
        data = com_re.split(data)[1:]

        for comdata in data:

            com_name = com_name_re.findall(comdata)[0].decode('cp1255')
            # print com_name

            sections = comdata.split('<BR>')
            chairpersons = members = replacements = []
            for section in sections:
                if '\xe7\xe1\xf8\xe9 \xe4\xe5\xe5\xf2\xe3\xe4' in section:
                    members = member_re.findall(section)
                elif '\xe9\xe5\xf9\xe1/\xfa \xf8\xe0\xf9 \xe4\xe5\xe5\xf2\xe3\xe4' in section:
                    chairpersons = member_re.findall(section)
                elif '\xee\xee\xec\xe0\xe9 \xee\xf7\xe5\xed \xe1\xe5\xe5\xf2\xe3\xe4' in section:
                    replacements = member_re.findall(section)

            chairpersons = set(chairpersons)
            members = set(members)
            replacements = set(replacements)
            members = members - chairpersons - replacements

            # print chairmen, members, replacements
            retval[com_name] = (list(chairpersons), list(members), list(replacements))

        return retval

    def convert_to_mks(self, ll):
        ll = [int(x) for x in ll]
        ll2 = []
        for x in ll:
            try:
                ll2.append(Member.objects.get(id=x))
            except Member.DoesNotExist:
                logger.warn("ERROR: couldn't find member for id: %s" % x)
                # raise
        return ll

    def update_committee_members_db(self, data):
        for name, details in data.items():
            chairpersons, members, replacements = details
{'fraction_non_alphanumeric': 0.11626984126984127, 'fraction_numerical': 0.01865079365079365, 'mean_word_length': 2.243886743886744, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6601461', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1574, 'n_words': 310}