<commit_before># -*- coding:Utf-8 -*-

#  Dieren Theater - lachambre.be to json sausage machine
#  Copyright (C) 2012  <NAME> <<EMAIL>>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re
import logging
logger = logging.getLogger('')

from lachambre.models import (Party, CommissionMembership, Commission,
                              Question, Analysis, Deputy)

from utils import (retry_on_access_error, LACHAMBRE_PREFIX, get_or_create,
                   table2dic, lame_url, read_or_dl_with_nl, read_or_dl)


def clean_models():
    logger.debug("cleaning deputies models")
    map(lambda x: x.objects.all().delete(), (Deputy, Party, Analysis))


def deputies_list(reset=False):
    soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies", reset)

    for dep in soup.table('tr'):
        items = dep('td')
        url = items[0].a['href']
        lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]

        deputy = Deputy.objects.filter(lachambre_id=lachambre_id)
        full_name = re.sub(" +", " ", items[0].a.text.strip())

        if not deputy:
            logger.info("[NEW] deputy: %s" % full_name)
        deputy = deputy[0] if deputy else Deputy(lachambre_id=lachambre_id)

        deputy.full_name = full_name
        deputy.url = url
        deputy.save()


@retry_on_access_error
def check_for_new_deputies(reset=False):
    deputies_list(reset)


def scrape():
    for index, deputy in enumerate(list(Deputy.objects.all())):
        logger.debug("%s %s" % (index, deputy.full_name))<commit_msg>[enh] reput deputies emails and websites and parties<commit_after># -*- coding:Utf-8 -*-

#  Dieren Theater - lachambre.be to json sausage machine
#  Copyright (C) 2012  <NAME> <<EMAIL>>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re
import logging
logger = logging.getLogger('')

from lachambre.models import (Party, CommissionMembership, Commission,
                              Question, Analysis, Deputy)

from utils import (retry_on_access_error, LACHAMBRE_PREFIX, get_or_create,
                   table2dic, lame_url, read_or_dl_with_nl, read_or_dl)


def clean_models():
    logger.debug("cleaning deputies models")
    map(lambda x: x.objects.all().delete(), (Deputy, Party, Analysis))


def deputies_list(reset=False):
    soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies", reset)

    for dep in soup.table('tr'):
        items = dep('td')
        url = items[0].a['href']
        lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]

        deputy = Deputy.objects.filter(lachambre_id=lachambre_id)
        full_name = re.sub(" +", " ", items[0].a.text.strip())

        if not deputy:
            logger.info("[NEW] deputy: %s" % full_name)
        deputy = deputy[0] if deputy else Deputy(lachambre_id=lachambre_id)

        if items[1].a.text.strip():
            deputy.party = get_or_create(Party, name=items[1].a.text.strip(), url=dict(items[1].a.attrs)['href'])

        email = items[2].a.text
        website = items[3].a['href'] if items[3].a else None

        if email not in deputy.emails:
            deputy.emails.append(email)
        if website not in deputy.websites:
            deputy.websites.append(website)

        deputy.full_name = full_name
        deputy.url = url
        deputy.save()


@retry_on_access_error
def check_for_new_deputies(reset=False):
    deputies_list(reset)


def scrape():
    for index, deputy in enumerate(list(Deputy.objects.all())):
        logger.debug("%s %s" % (index, deputy.full_name))
{'fraction_non_alphanumeric': 0.116751269035533, 'fraction_numerical': 0.00649746192893401, 'mean_word_length': 3.3631532329495126, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1978208', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1554, 'n_words': 440}