<commit_before># -*- coding:Utf-8 -*-
import re
from os.path import exists
from urllib import urlopen, quote
from BeautifulSoup import BeautifulSoup

from deputies.models import Deputy, Party, CommissionMembership, Document, Question, Analysis, Commission

LACHAMBRE_PREFIX="http://www.lachambre.be/kvvcr/"

def hammer_time(function):
    "decorator to retry to download a page because La Chambre website sucks"
    def wrap(*args, **kwargs):
        reset = False
        for i in xrange(4):
            try:
                return function(*args, reset=reset, **kwargs)
            except (IndexError, AttributeError, TypeError):
                reset = True
        print "WARNING, function keeps failling", function, args, kwargs
    return wrap

def lame_url(url):
    # convert super lame urls of lachambre.be into something uzable
    return quote(url.encode("iso-8859-1"), safe="%/:=&?~#+!$,;'@()*[]")

def get_or_create(klass, _id=None, **kwargs):
    if _id is None:
        object = klass.objects.filter(**kwargs)
    else:
        object = klass.objects.filter(**{_id : kwargs[_id]})
    if object:
        return object[0]
    else:
        print "add new", klass.__name__, kwargs
        return klass.objects.create(**kwargs)

def read_or_dl(url, name, reset=False):
    print "parsing", url
    if not reset and exists('dump/%s' % name):
        text = open('dump/%s' % name).read()
    else:
        text = urlopen(url).read()
        open('dump/%s' % name, "w").write(text)
    soup = BeautifulSoup(text)
    if soup.title.text == "404 Not Found":
        raise IndexError
    return soup

def clean():
    print "cleaning db"
    map(lambda x: x.objects.all().delete(), (Deputy, Party, CommissionMembership, Document, Question, Analysis, Commission))

def deputies_list():
    soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies")

    for dep in soup('table')[4]('tr'):
        items = dep('td')
        full_name = re.sub('  +', ' ', items[0].a.text)
        url = items[0].a['href']
        party = get_or_create(Party, name=items[1].a.text, url=dict(items[1].a.attrs)['href'])
        email = items[2].a.text
        website = items[3].a['href'] if items[3].a else None
        # yes, one deputies key contains a O instead of an 0, I'm not joking
        lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]
        Deputy.objects.create(full_name=full_name,
                              party=party,
                              url=url,
                              websites=[website] if website else [],
                              lachambre_id=lachambre_id,
                              emails=[email])
        print 'adding new deputy', lachambre_id, full_name, party, email, website if website else ''

def each_deputies():
    for index, deputy in enumerate(Deputy.objects.all()):
        print index, deputy.full_name
        parse_deputy(deputy)

@hammer_time
def parse_deputy(deputy, reset=False):
    soup = read_or_dl(LACHAMBRE_PREFIX + deputy.url, deputy.full_name, reset)
    deputy.language = soup.i.parent.text.split(":")[1]
    deputy.cv = re.sub('  +', ' ', soup('table')[5].p.text)
    if deputy.cv.encode("Utf-8").startswith("Députée"):
        deputy.sex = "F"
        print "F"
    elif deputy.cv.encode("Utf-8").startswith("Député"):
        deputy.sex = "M"
        print "M"
    else:
        deputy.sex = None

    # here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+
    # look at the bottom of each deputies' page
    membership = soup.find('td', rowspan="1")
    item = membership.h4
    role = None
    while item.nextSibling:
        if hasattr(item, 'tag'):
            if item.name == 'h5':
                role = item.text[6:-1]
            elif item.name == 'div':
                print "linking deputy to commission", item.a.text
                commission = get_or_create(Commission, lachambre_id=int(re.search("com=(\d+)", item.a["href"]).groups()[0]))
                deputy.commissions.append(CommissionMembership.objects.create(commission=commission, name=item.a.text, role=role, url=item.a['href']))
        item = item.nextSibling

    deputy_documents(soup, deputy)
    deputy.save()

@hammer_time
def get_deputy_documents(url, deputy, role, type=None, reset=False):
    print "working on %s %sdocuments" % (role, type + " " if type else '') #, LACHAMBRE_PREFIX + lame_url(urls[index])
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s %s' % (deputy.full_name, type if type else '', role), reset)
    setattr(deputy, "documents_%s%s_url" % (role, type + "_" if type else ''), url)
    setattr(deputy, "documents_%s%s_list" % (role, type + "_" if type else ''), [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        print "add", type if type else '', role, i.tr('td')[1].text
        getattr(deputy, "documents_%s%s_list" % (role, type + "_" if type else '')).append(get_or_create(Document, _id="lachambre_id",
                                                                                                         lachambre_id=re.search("dossierID=(\d+)", i.a["href"]).groups()[0],
                                                                                                         url=i.a['href'],
                                                                                                         title=i.table('td')[1].text,
                                                                                                         status=i.table('td')[5].text,
                                                                                                         eurovoc_main_descriptor=i.table('td')[7].text,
                                                                                                         eurovoc_descriptors=map(lambda x: x.strip(), i.table('td')[9].text.split('|')),
                                                                                                         type=type))

@hammer_time
def get_deputy_questions(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "questions_%s_url" % type, url)
    setattr(deputy, "questions_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        print "add", type, i.tr('td')[1].text.strip()
        if type == "written":
            deputy.questions_written_list.append(get_or_create(Question,
                                                               _id="lachambre_id",
                                                               title=i.table('td')[5].text,
                                                               departement=i.table('td')[3].text,
                                                               lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                                               deposition_date=re.sub("[^0-9/]", "", i.table('td')[7].contents[0]),
                                                               eurovoc_descriptors=map(lambda x: x.strip(), i.table('td')[9].text.split('|')) if len(i.table('td')) >= 10 and i.table('td')[8].text == u'Descripteurs Eurovoc :' else [],
                                                               keywords=map(lambda x: x.strip(), i.table('td')[-1].text.split('|')) if len(i.table('td')) >= 12 else [],
                                                               url=i.a['href'],
                                                               type=type))
            print deputy.questions_written_list
        else:
            getattr(deputy, "questions_%s_list" % type).append(get_or_create(Question,
                                                                             _id="lachambre_id",
                                                                             title=i.table('td')[1].text,
                                                                             reunion_type=i.table('td')[9].text,
                                                                             reunion_date=i.table('td')[7].text,
                                                                             session_id=i.table('td')[5].text,
                                                                             pdf_url=i.table('td')[11].a["href"],
                                                                             lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                                                             eurovoc_descriptors=map(lambda x: x.strip(), i.table('td')[13].text.split('|')) if len(i.table('td')) >= 14 and i.table('td')[12].text == u'Descripteurs Eurovoc :' else [],
                                                                             keywords=map(lambda x: x.strip(), i.table('td')[-1].text.split('|')) if len(i.table('td')) == 16 else [],
                                                                             url=i.a['href'],
                                                                             type=type))

@hammer_time
def get_deputy_analysis(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "analysis_%s_url" % type, url)
    setattr(deputy, "analysis_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        print "add", type, i.tr('td')[1].text.strip()
        getattr(deputy, "analysis_%s_list" % type).append(get_or_create(Analysis,
                                                                        _id="lachambre_id",
                                                                        lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                                                        url=i.a['href'],
                                                                        type=type))
<commit_msg>[mod] remove debug print<commit_after># -*- coding:Utf-8 -*-
import re
from os.path import exists
from urllib import urlopen, quote
from BeautifulSoup import BeautifulSoup

from deputies.models import Deputy, Party, CommissionMembership, Document, Question, Analysis, Commission

LACHAMBRE_PREFIX="http://www.lachambre.be/kvvcr/"

def hammer_time(function):
    "decorator to retry to download a page because La Chambre website sucks"
    def wrap(*args, **kwargs):
        reset = False
        for i in xrange(4):
            try:
                return function(*args, reset=reset, **kwargs)
            except (IndexError, AttributeError, TypeError):
                reset = True
        print "WARNING, function keeps failling", function, args, kwargs
    return wrap

def lame_url(url):
    # convert super lame urls of lachambre.be into something uzable
    return quote(url.encode("iso-8859-1"), safe="%/:=&?~#+!$,;'@()*[]")

def get_or_create(klass, _id=None, **kwargs):
    if _id is None:
        object = klass.objects.filter(**kwargs)
    else:
        object = klass.objects.filter(**{_id : kwargs[_id]})
    if object:
        return object[0]
    else:
        print "add new", klass.__name__, kwargs
        return klass.objects.create(**kwargs)

def read_or_dl(url, name, reset=False):
    print "parsing", url
    if not reset and exists('dump/%s' % name):
        text = open('dump/%s' % name).read()
    else:
        text = urlopen(url).read()
        open('dump/%s' % name, "w").write(text)
    soup = BeautifulSoup(text)
    if soup.title.text == "404 Not Found":
        raise IndexError
    return soup

def clean():
    print "cleaning db"
    map(lambda x: x.objects.all().delete(), (Deputy, Party, CommissionMembership, Document, Question, Analysis, Commission))

def deputies_list():
    soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies")

    for dep in soup('table')[4]('tr'):
        items = dep('td')
        full_name = re.sub('  +', ' ', items[0].a.text)
        url = items[0].a['href']
        party = get_or_create(Party, name=items[1].a.text, url=dict(items[1].a.attrs)['href'])
        email = items[2].a.text
        website = items[3].a['href'] if items[3].a else None
        # yes, one deputies key contains a O instead of an 0, I'm not joking
        lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]
        Deputy.objects.create(full_name=full_name,
                              party=party,
                              url=url,
                              websites=[website] if website else [],
                              lachambre_id=lachambre_id,
                              emails=[email])
        print 'adding new deputy', lachambre_id, full_name, party, email, website if website else ''

def each_deputies():
    for index, deputy in enumerate(Deputy.objects.all()):
        print index, deputy.full_name
        parse_deputy(deputy)

@hammer_time
def parse_deputy(deputy, reset=False):
    soup = read_or_dl(LACHAMBRE_PREFIX + deputy.url, deputy.full_name, reset)
    deputy.language = soup.i.parent.text.split(":")[1]
    deputy.cv = re.sub('  +', ' ', soup('table')[5].p.text)
    if deputy.cv.encode("Utf-8").startswith("Députée"):
        deputy.sex = "F"
        print "F"
    elif deputy.cv.encode("Utf-8").startswith("Député"):
        deputy.sex = "M"
        print "M"
    else:
        deputy.sex = None

    # here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+
    # look at the bottom of each deputies' page
    membership = soup.find('td', rowspan="1")
    item = membership.h4
    role = None
    while item.nextSibling:
        if hasattr(item, 'tag'):
            if item.name == 'h5':
                role = item.text[6:-1]
            elif item.name == 'div':
                print "linking deputy to commission", item.a.text
                commission = get_or_create(Commission, lachambre_id=int(re.search("com=(\d+)", item.a["href"]).groups()[0]))
                deputy.commissions.append(CommissionMembership.objects.create(commission=commission, name=item.a.text, role=role, url=item.a['href']))
        item = item.nextSibling

    deputy_documents(soup, deputy)
    deputy.save()

@hammer_time
def get_deputy_documents(url, deputy, role, type=None, reset=False):
    print "working on %s %sdocuments" % (role, type + " " if type else '') #, LACHAMBRE_PREFIX + lame_url(urls[index])
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s %s' % (deputy.full_name, type if type else '', role), reset)
    setattr(deputy, "documents_%s%s_url" % (role, type + "_" if type else ''), url)
    setattr(deputy, "documents_%s%s_list" % (role, type + "_" if type else ''), [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        print "add", type if type else '', role, i.tr('td')[1].text
        getattr(deputy, "documents_%s%s_list" % (role, type + "_" if type else '')).append(get_or_create(Document, _id="lachambre_id",
                                                                                                         lachambre_id=re.search("dossierID=(\d+)", i.a["href"]).groups()[0],
                                                                                                         url=i.a['href'],
                                                                                                         title=i.table('td')[1].text,
                                                                                                         status=i.table('td')[5].text,
                                                                                                         eurovoc_main_descriptor=i.table('td')[7].text,
                                                                                                         eurovoc_descriptors=map(lambda x: x.strip(), i.table('td')[9].text.split('|')),
                                                                                                         type=type))

@hammer_time
def get_deputy_questions(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "questions_%s_url" % type, url)
    setattr(deputy, "questions_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        print "add", type, i.tr('td')[1].text.strip()
        if type == "written":
            deputy.questions_written_list.append(get_or_create(Question,
                                                               _id="lachambre_id",
                                                               title=i.table('td')[5].text,
                                                               departement=i.table('td')[3].text,
                                                               lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                                               deposition_date=re.sub("[^0-9/]", "", i.table('td')[7].contents[0]),
                                                               eurovoc_descriptors=map(lambda x: x.strip(), i.table('td')[9].text.split('|')) if len(i.table('td')) >= 10 and i.table('td')[8].text == u'Descripteurs Eurovoc :' else [],
                                                               keywords=map(lambda x: x.strip(), i.table('td')[-1].text.split('|')) if len(i.table('td')) >= 12 else [],
                                                               url=i.a['href'],
                                                               type=type))
        else:
            getattr(deputy, "questions_%s_list" % type).append(get_or_create(Question,
                                                                             _id="lachambre_id",
                                                                             title=i.table('td')[1].text,
                                                                             reunion_type=i.table('td')[9].text,
                                                                             reunion_date=i.table('td')[7].text,
                                                                             session_id=i.table('td')[5].text,
                                                                             pdf_url=i.table('td')[11].a["href"],
                                                                             lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                                                             eurovoc_descriptors=map(lambda x: x.strip(), i.table('td')[13].text.split('|')) if len(i.table('td')) >= 14 and i.table('td')[12].text == u'Descripteurs Eurovoc :' else [],
                                                                             keywords=map(lambda x: x.strip(), i.table('td')[-1].text.split('|')) if len(i.table('td')) == 16 else [],
                                                                             url=i.a['href'],
                                                                             type=type))

@hammer_time
def get_deputy_analysis(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "analysis_%s_url" % type, url)
    setattr(deputy, "analysis_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        print "add", type, i.tr('td')[1].text.strip()
        getattr(deputy, "analysis_%s_list" % type).append(get_or_create(Analysis,
                                                                        _id="lachambre_id",
                                                                        lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                                                        url=i.a['href'],
                                                                        type=type))

{'fraction_non_alphanumeric': 0.1395613644239814, 'fraction_numerical': 0.00831724342789247, 'mean_word_length': 1.4387299287697695, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 198, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '67562', 'n_tokens_mistral': 5621, 'n_tokens_neox': 5474, 'n_words': 1087}