<commit_before>#!/usr/local/bin/python
# -*- coding: utf-8 -*-

"""
Document wrappers for LegCo Agendas
"""
from collections import OrderedDict
import logging
import lxml
from lxml import etree
from lxml.html.clean import clean_html, Cleaner
import re
from lxml.html import HTMLParser
import itertools
from raw.utils import to_string, to_unicode, grouper


logger = logging.getLogger('legcowatch')
QUESTION_PATTERN_E = ur'^\*?([0-9]+)\..*?Hon\s(.*?)\sto ask:'
QUESTION_PATTERN_C = ur'^\*?([0-9]+)\.\s*(.*?)議員問:'
LEGISLATION_E = u'Subsidiary Legislation'
LEGISLATION_C = u'附屬法例'
OTHER_PAPERS_E = u'Other Paper'
OTHER_PAPERS_C = u'其他文件'


class CouncilAgenda(object):
    """
    Object representing the Council Agenda document.  This class
    parses the document source and makes all of the individual elements easily accessible
    """
    SECTION_MAP = OrderedDict(
        (
            ('tabled_papers', [u'Tabling of Paper', u'提交文件']),
            ('members_bills', [u"Members' Bill", u"Member's Bill", u'議員法案']),
            ('members_motions', [u"Members' Motion", u"Member's Motion", u'議員議案']),
            ('questions', [u'Question', u'質詢']),
            ('bills', [u'Bill', u'法案']),
            ('motions', [u'Motion', u'議案']),
        )
    )

    def __init__(self, uid, source, *args, **kwargs):
        self.uid = uid
        if uid[-1] == 'e':
            self.english = True
        else:
            self.english = False
        # Raw html string
        self.source = source
        self.tree = None
        self.tabled_papers = None
        self.questions = None
        self.motions = None
        self.bills = None
        self.members_bills = None
        self.members_motions = None
        self.other = None
        self._headers = []
        self._load()
        self._clean()
        self._parse()

    def __repr__(self):
        return u'<CouncilAgenda: {}>'.format(self.uid)

    def _load(self):
        """
        Load the ElementTree from the source
        """
        # Convert directional quotation marks to regular quotes
        double_quotes = ur'[\u201c\u201d]'
        self.source = re.sub(double_quotes, u'"', self.source)
        single_quotes = ur'[\u2019\u2018]'
        self.source = re.sub(single_quotes, u"'", self.source)
        # Convert colons
        self.source = self.source.replace(u'\uff1a', u':')
        # Remove line breaks and tabs
        self.source = self.source.replace(u'\n', u'')
        self.source = self.source.replace(u'\t', u'')
        # There are also some "zero width joiners" in random places
        # in the text.  Doesn't seem to cause any harm, though, so leave for now
        # these are the codeS: &#8205, &#160 (nbsp), \xa0 (nbsp), \u200d
        # Also previously had some non breaking spaces in unicode \u00a0, but this
        # may have been fixed by changing the parser below

        # Use the lxml cleaner
        cleaner = Cleaner()
        parser = HTMLParser(encoding='utf-8')
        # Finally, load the cleaned string to an ElementTree
        self.tree = cleaner.clean_html(lxml.html.fromstring(to_string(self.source), parser=parser))
        # self.tree = lxml.html.fromstring(to_string(self.source))

    def _clean(self):
        """
        Removes some of extraneous tags to make parsing easier
        """
        etree.strip_tags(self.tree, 'strong')
        for xx in self.tree.find_class('pydocx-tab'):
            xx.drop_tag()

    def _parse(self):
        """
        Parse the source document and populate this object's properties
        """
        # The A is for special question sections, such as the agenda on June 18, 2014
        pattern = ur'^[IVA]+\.'
        current_section = None
        # Iterate over the elements in self.tree.iter()
        # We only want paragraphs and table, since these appear to be the main top level elements
        # ie. we don't want to go fully down the tree
        # This is buggy for converted DOC files, since usually the tables have nested p elements
        for elem in self.tree.iter('table', 'p'):
            # When we encounter a header element, figure out what section it is a header for
            text = elem.text_content().strip()
            if text == u'':
                continue
            if text and re.search(pattern, text):
                section_name = self._identify_section(text)
                if section_name is not None:
                    logger.info(u'Identified header {} as {}'.format(text, section_name))
                    current_section = section_name
                else:
                    logger.warn(u"Could not identify section from header {}".format(text))
                    current_section = "other"
                # If this is the first time in this section, initailize the array for storing stuff
                if getattr(self, current_section) is None:
                    setattr(self, current_section, [])
                self._headers.append((current_section, elem))
            else:
                # Add all the elements we encounter to the list for the current section until
                # we encounter another header element, or the end of the document
                if current_section is not None:
                    arr = getattr(self, current_section)
                    arr.append(elem)

        # Once all of the sections are split up, parse each of them separately
        for section in CouncilAgenda.SECTION_MAP.keys():
            if getattr(self, section) is not None:
                getattr(self, "_parse_{}".format(section))()
        # We won't parse others, since we don't know what those are

    def _parse_tabled_papers(self):
        """
        Parse elements for tabled papers
        """
        if self.tabled_papers is None:
            return
        # Filter out paragraphs, which are actually children of the table elements
        self.tabled_papers = [xx for xx in self.tabled_papers if xx.tag == u'table']
        logger.info(u'Parsing tabled papers from {} elements'.format(len(self.tabled_papers)))
        parsed_papers = []
        for tbl in self.tabled_papers:
            # We only care about tables, specifically:
            # 1) a table with "Subsidiary Legislation / Instruments as its title, and
            # 2) a table with "Other Paper" as the title
            # One difficulty is that not every table is headered, such as
            # in the case of the June 5 2013 agenda, since there was only one paper
            first_row_text = tbl[0].text_content().strip()
            match_text = LEGISLATION_E if self.english else LEGISLATION_C
            match_text2 = OTHER_PAPERS_E if self.english else OTHER_PAPERS_C
            if match_text in first_row_text:
                # Subsidiary legislation
                # Papers occur in single rows
                logger.debug(u'Found subsidiary legislation table')
                parsed_papers.append(self._parse_tabled_legislation(tbl[1:]))
            elif match_text2 in first_row_text:
                # Other papers table
                # rows occur in pairs, with the first row being the title
                # and the second row being the presenter
                logger.debug(u'Found other papers table')
                parsed_papers.append(self._parse_other_papers(tbl[1:]))
            else:
                # No title, try to infer the table
                # For some Chinese agendas, it seems like the title is not included
                # or it could be in a different element
                # Check the second row to see if there are parenthesis
                # If there are, then it's likely Other papers
                # Or, can check the last column to see if there is a legislation number
                paper_number = ur'\d+/\d+'
                last_col = tbl[0][-1].text_content().strip()
                match = re.search(paper_number, last_col)
                if match:
                    logger.debug(u'Inferred subsidiary legislation table')
                    parsed_papers.append(self._parse_tabled_legislation(tbl))
                else:
                    logger.debug(u'Inferred other papers table')
                    parsed_papers.append(self._parse_other_papers(tbl))<commit_msg>Fix bug in older tabled paper parsing<commit_after>#!/usr/local/bin/python
# -*- coding: utf-8 -*-

"""
Document wrappers for LegCo Agendas
"""
from collections import OrderedDict
import logging
import lxml
from lxml import etree
from lxml.html.clean import clean_html, Cleaner
import re
from lxml.html import HTMLParser
import itertools
from raw.utils import to_string, to_unicode, grouper


logger = logging.getLogger('legcowatch')
QUESTION_PATTERN_E = ur'^\*?([0-9]+)\..*?Hon\s(.*?)\sto ask:'
QUESTION_PATTERN_C = ur'^\*?([0-9]+)\.\s*(.*?)議員問:'
LEGISLATION_E = u'Subsidiary Legislation'
LEGISLATION_C = u'附屬法例'
OTHER_PAPERS_E = u'Other Paper'
OTHER_PAPERS_C = u'其他文件'


class CouncilAgenda(object):
    """
    Object representing the Council Agenda document.  This class
    parses the document source and makes all of the individual elements easily accessible
    """
    SECTION_MAP = OrderedDict(
        (
            ('tabled_papers', [u'Tabling of Paper', u'提交文件']),
            ('members_bills', [u"Members' Bill", u"Member's Bill", u'議員法案']),
            ('members_motions', [u"Members' Motion", u"Member's Motion", u'議員議案']),
            ('questions', [u'Question', u'質詢']),
            ('bills', [u'Bill', u'法案']),
            ('motions', [u'Motion', u'議案']),
        )
    )

    def __init__(self, uid, source, *args, **kwargs):
        self.uid = uid
        if uid[-1] == 'e':
            self.english = True
        else:
            self.english = False
        # Raw html string
        self.source = source
        self.tree = None
        self.tabled_papers = None
        self.questions = None
        self.motions = None
        self.bills = None
        self.members_bills = None
        self.members_motions = None
        self.other = None
        self._headers = []
        self._load()
        self._clean()
        self._parse()

    def __repr__(self):
        return u'<CouncilAgenda: {}>'.format(self.uid)

    def _load(self):
        """
        Load the ElementTree from the source
        """
        # Convert directional quotation marks to regular quotes
        double_quotes = ur'[\u201c\u201d]'
        self.source = re.sub(double_quotes, u'"', self.source)
        single_quotes = ur'[\u2019\u2018]'
        self.source = re.sub(single_quotes, u"'", self.source)
        # Convert colons
        self.source = self.source.replace(u'\uff1a', u':')
        # Remove line breaks and tabs
        self.source = self.source.replace(u'\n', u'')
        self.source = self.source.replace(u'\t', u'')
        # There are also some "zero width joiners" in random places
        # in the text.  Doesn't seem to cause any harm, though, so leave for now
        # these are the codeS: &#8205, &#160 (nbsp), \xa0 (nbsp), \u200d
        # Also previously had some non breaking spaces in unicode \u00a0, but this
        # may have been fixed by changing the parser below

        # Use the lxml cleaner
        cleaner = Cleaner()
        parser = HTMLParser(encoding='utf-8')
        # Finally, load the cleaned string to an ElementTree
        self.tree = cleaner.clean_html(lxml.html.fromstring(to_string(self.source), parser=parser))
        # self.tree = lxml.html.fromstring(to_string(self.source))

    def _clean(self):
        """
        Removes some of extraneous tags to make parsing easier
        """
        etree.strip_tags(self.tree, 'strong')
        for xx in self.tree.find_class('pydocx-tab'):
            xx.drop_tag()

    def _parse(self):
        """
        Parse the source document and populate this object's properties
        """
        # The A is for special question sections, such as the agenda on June 18, 2014
        pattern = ur'^[IVA]+\.'
        current_section = None
        # Iterate over the elements in self.tree.iter()
        # We only want paragraphs and table, since these appear to be the main top level elements
        # ie. we don't want to go fully down the tree
        # This is buggy for converted DOC files (older agendas),
        # since usually the tables have nested p elements
        for elem in self.tree.iter('table', 'p'):
            # When we encounter a header element, figure out what section it is a header for
            text = elem.text_content().strip()
            if text == u'':
                continue
            if text and re.search(pattern, text):
                section_name = self._identify_section(text)
                if section_name is not None:
                    logger.info(u'Identified header {} as {}'.format(text, section_name))
                    current_section = section_name
                else:
                    logger.warn(u"Could not identify section from header {}".format(text))
                    current_section = "other"
                # If this is the first time in this section, initailize the array for storing stuff
                if getattr(self, current_section) is None:
                    setattr(self, current_section, [])
                self._headers.append((current_section, elem))
            else:
                # Add all the elements we encounter to the list for the current section until
                # we encounter another header element, or the end of the document
                if current_section is not None:
                    arr = getattr(self, current_section)
                    arr.append(elem)

        # Once all of the sections are split up, parse each of them separately
        for section in CouncilAgenda.SECTION_MAP.keys():
            if getattr(self, section) is not None:
                getattr(self, "_parse_{}".format(section))()
        # We won't parse others, since we don't know what those are

    def _parse_tabled_papers(self):
        """
        Parse elements for tabled papers
        """
        if self.tabled_papers is None:
            return
        # Filter out paragraphs, which are actually children of the table elements
        self.tabled_papers = [xx for xx in self.tabled_papers if xx.tag == u'table']
        logger.info(u'Parsing tabled papers from {} elements'.format(len(self.tabled_papers)))
        parsed_papers = []
        for tbl in self.tabled_papers:
            # We only care about tables, specifically:
            # 1) a table with "Subsidiary Legislation / Instruments as its title, and
            # 2) a table with "Other Paper" as the title
            # One difficulty is that not every table is headered, such as
            # in the case of the June 5 2013 agenda, since there was only one paper
            first_row_text = tbl[0].text_content().strip()
            match_text = LEGISLATION_E if self.english else LEGISLATION_C
            match_text2 = OTHER_PAPERS_E if self.english else OTHER_PAPERS_C
            if match_text in first_row_text:
                # Subsidiary legislation
                # Papers occur in single rows
                logger.debug(u'Found subsidiary legislation table')
                # In older files, sometimes the table has all the elements after it in its iterator,
                # so to avoid processing to much, just process the direct tr children
                parsed_papers.append(self._parse_tabled_legislation(tbl.xpath('./tr')[1:]))
            elif match_text2 in first_row_text:
                # Other papers table
                # rows occur in pairs, with the first row being the title
                # and the second row being the presenter
                logger.debug(u'Found other papers table')
                parsed_papers.append(self._parse_other_papers(tbl.xpath('./tr')[1:]))
            else:
                # No title, try to infer the table
                # For some Chinese agendas, it seems like the title is not included
                # or it could be in a different element
                # Check the second row to see if there are parenthesis
                # If there are, then it's likely Other papers
                # Or, can check the last column to see if there is a legislation number
                paper_number = ur'\d+/\d+'
                last_col = tbl[0][-1].text_content().strip()
                match = re.search(paper_number, last_col)
                if match:
                    logger.debug(u'Inferred subsidiary legislation table')
                    parsed_papers.append(self._parse_tabled_legislation(tbl.xpath('./tr')))
                else:
                    logger.debug(u'Inferred other papers table')
                    parsed_papers.append(self._parse_other_papers(tbl.xpath('./tr')))
{'fraction_non_alphanumeric': 0.09379277509968457, 'fraction_numerical': 0.006665476403023269, 'mean_word_length': 2.0759655866739886, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6501426', 'n_tokens_mistral': 4887, 'n_tokens_neox': 4586, 'n_words': 1536}