<commit_before>from pyquery import PyQuery as pq
from mptracker.scraper.common import Scraper, pqitems, get_cdep_id


class ProposalScraper(Scraper):

    proposals_url = ('http://www.cdep.ro/pls/proiecte/upl_pck.lista'
                     '?cam=2&anp=2013')

    def fetch_proposals(self):
        page = self.fetch_url(self.proposals_url)
        the_table = pqitems(page, 'table table table table table')[-1]
        rows = iter(pqitems(the_table, 'tr'))
        assert next(rows).text() == "Numar Titlu Stadiu"
        for row in rows:
            tr_1 = pqitems(row, 'td')[1]
            url = pq('a', tr_1).attr('href')
            assert url.startswith('http://www.cdep.ro/pls/proiecte'
                                  '/upl_pck.proiect?cam=2&idp=')
            yield self.fetch_proposal_details(url)

    def fetch_proposal_details(self, url):
        return url
<commit_msg>scrape proposal details<commit_after>import re
from pyquery import PyQuery as pq
from mptracker.scraper.common import Scraper, pqitems, get_cdep_id
from mptracker.common import fix_local_chars


class ProposalScraper(Scraper):

    proposals_url = ('http://www.cdep.ro/pls/proiecte/upl_pck.lista'
                     '?cam=2&anp=2013')

    def fix_name(self, name):
        return fix_local_chars(re.sub(r'[\s\-]+', ' ', name))

    def fetch_proposals(self):
        page = self.fetch_url(self.proposals_url)
        the_table = pqitems(page, 'table table table table table')[-1]
        rows = iter(pqitems(the_table, 'tr'))
        assert next(rows).text() == "Numar Titlu Stadiu"
        for row in rows:
            tr_1 = pqitems(row, 'td')[1]
            url = pq('a', tr_1).attr('href')
            assert url.startswith('http://www.cdep.ro/pls/proiecte'
                                  '/upl_pck.proiect?cam=2&idp=')
            yield self.fetch_proposal_details(url)

    def fetch_proposal_details(self, url):
        page = self.fetch_url(url)
        out = {
            'headline': pq('.headline', page).text(),
            'url': url,
        }

        [hook_td] = pqitems(page, ':contains("Nr. înregistrare")')
        metadata_table = pq(hook_td.parents('table')[-1])
        for row in pqitems(metadata_table.children('tr')):
            cols = row.children()
            label = cols.eq(0).text().strip()
            val_td = cols.eq(1) if len(cols) > 1 else None

            if label == "- Camera Deputatilor:":
                out['cdep_serial'] = val_td.text()

            elif label == "Tip initiativa:":
                out['proposal_type'] = val_td.text()

            elif label == "Initiator:":
                sponsors = []
                for a in val_td('a').items():
                    cdep_id = get_cdep_id(a.attr('href'), fail='none')
                    if cdep_id is not None:
                        sponsors.append({
                            'cdep_id': cdep_id,
                            'name': self.fix_name(a.text()),
                        })
                out['sponsors'] = sponsors

        return out

{'fraction_non_alphanumeric': 0.13291762389235312, 'fraction_numerical': 0.007876599934361668, 'mean_word_length': 1.9736585365853658, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2621771', 'n_tokens_mistral': 1035, 'n_tokens_neox': 1001, 'n_words': 190}