<commit_before>from .base import LegistarScraper
from lxml.etree import tostring
from collections import deque

class LegistarBillScraper(LegistarScraper):
    def legislation(self, search_text='', created_after=None, 
                    created_before=None) :

        # If legislation is added to the the legistar system while we
        # are scraping, it will shift the list of legislation down and
        # we might revisit the same legislation. So, we keep track of
        # the last few pieces of legislation we've visited in order to
        # make sure we are not revisiting
        scraped_leg = deque([], maxlen=10)

        for page in self.searchLegislation(search_text, created_after,
                                           created_before) :
            for legislation_summary in self.parseSearchResults(page) :
                if not legislation_summary['url'] in scraped_leg :
                    yield legislation_summary
                    scraped_leg.append(legislation_summary['url'])

    def searchLegislation(self, search_text='', created_after=None,
                          created_before=None):
        """
        Submit a search query on the legislation search page, and return a list
        of summary results.
        """

        page = self.lxmlize(self.LEGISLATION_URL)

        page = self._advancedSearch(page)

        payload = {}

        # Enter the search parameters TODO: Each of the possible form
        # fields should be represented as keyword arguments to this
        # function. The default query string should be for the the
        # default 'Legislative text' field.
        payload['ctl00$ContentPlaceHolder1$txtText'] = search_text

        if created_after and created_before :
            payload.update(dateWithin(created_after, created_before))

        elif created_before :
            payload.update(dateBound(created_before))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '<'

        elif created_after :
            payload.update(dateBound(created_after))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '>'


        # Return up to one million search results
        payload['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'
        payload['ctl00_ContentPlaceHolder1_lstYearsAdvanced_ClientState'] = '{"value":"All"}'
        payload['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'


        payload.update(self.sessionSecrets(page))

        return self.pages(self.LEGISLATION_URL, payload)

    def parseSearchResults(self, page) :
        """Take a page of search results and return a sequence of data
        of tuples about the legislation, of the form

        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        table = page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
        for legislation, headers, row in self.parseDataTable(table):
            # Do legislation search-specific stuff
            # ------------------------------------
            # First column should be the ID of the record.
            id_key = headers[0]
            try:
                legislation_id = legislation[id_key]['label']
            except TypeError:
                continue
            legislation_url = legislation[id_key]['url'].split(self.BASE_URL)[-1]
            legislation[id_key] = legislation_id
            legislation['url'] = self.BASE_URL + legislation_url.split('&Options')[0]

            yield legislation

    def _advancedSearch(self, page) :
        search_switcher = page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0]

        if 'simple search' in search_switcher.value.lower() :
            return page
        else :
            payload = self.sessionSecrets(page)
            payload[search_switcher.name] = search_switcher.value

            page = self.lxmlize(self.LEGISLATION_URL, payload)

            if 'simple search' not in page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].value.lower() :
                raise ValueError('Not on the advanced search page')

            return page

    def details(self, detail_url, div_id) :
        detail_page = self.lxmlize(detail_url)
        
        detail_div = detail_page.xpath(".//div[@id='%s']" % div_id)[0]

        return self.parseDetails(detail_div)

    def legDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageDetails'
        return self.details(detail_url, div_id)

    def actionDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageTop1'
        return self.details(detail_url, div_id)

    def history(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        try :
            history_table = detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridLegislation_ctl00']")[0]
        except IndexError :
            print(detail_url)
            raise

        history = [row[0] for row in self.parseDataTable(history_table)] 

        history = sorted(history, key = self._actionSortKey)

        for action in history :
            yield action

                    
    def _actionSortKey(self, action) :
        action_date = self.toDate(action['Date'])

        try :
            action_url = action['Action\xa0Details']['url']
        except TypeError :
            action_url = '0'

        return (action_date, action_url)

    def text(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        text_div = detail_page.xpath("//div[@id='ctl00_ContentPlaceHolder1_divText']")

        if len(text_div) :
            return tostring(text_div[0], pretty_print=True).decode()<commit_msg>fall back to default sort order of history if exception<commit_after>from .base import LegistarScraper
from lxml.etree import tostring
from collections import deque

class LegistarBillScraper(LegistarScraper):
    def legislation(self, search_text='', created_after=None, 
                    created_before=None) :

        # If legislation is added to the the legistar system while we
        # are scraping, it will shift the list of legislation down and
        # we might revisit the same legislation. So, we keep track of
        # the last few pieces of legislation we've visited in order to
        # make sure we are not revisiting
        scraped_leg = deque([], maxlen=10)

        for page in self.searchLegislation(search_text, created_after,
                                           created_before) :
            for legislation_summary in self.parseSearchResults(page) :
                if not legislation_summary['url'] in scraped_leg :
                    yield legislation_summary
                    scraped_leg.append(legislation_summary['url'])

    def searchLegislation(self, search_text='', created_after=None,
                          created_before=None):
        """
        Submit a search query on the legislation search page, and return a list
        of summary results.
        """

        page = self.lxmlize(self.LEGISLATION_URL)

        page = self._advancedSearch(page)

        payload = {}

        # Enter the search parameters TODO: Each of the possible form
        # fields should be represented as keyword arguments to this
        # function. The default query string should be for the the
        # default 'Legislative text' field.
        payload['ctl00$ContentPlaceHolder1$txtText'] = search_text

        if created_after and created_before :
            payload.update(dateWithin(created_after, created_before))

        elif created_before :
            payload.update(dateBound(created_before))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '<'

        elif created_after :
            payload.update(dateBound(created_after))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '>'


        # Return up to one million search results
        payload['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'
        payload['ctl00_ContentPlaceHolder1_lstYearsAdvanced_ClientState'] = '{"value":"All"}'
        payload['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'


        payload.update(self.sessionSecrets(page))

        return self.pages(self.LEGISLATION_URL, payload)

    def parseSearchResults(self, page) :
        """Take a page of search results and return a sequence of data
        of tuples about the legislation, of the form

        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        table = page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
        for legislation, headers, row in self.parseDataTable(table):
            # Do legislation search-specific stuff
            # ------------------------------------
            # First column should be the ID of the record.
            id_key = headers[0]
            try:
                legislation_id = legislation[id_key]['label']
            except TypeError:
                continue
            legislation_url = legislation[id_key]['url'].split(self.BASE_URL)[-1]
            legislation[id_key] = legislation_id
            legislation['url'] = self.BASE_URL + legislation_url.split('&Options')[0]

            yield legislation

    def _advancedSearch(self, page) :
        search_switcher = page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0]

        if 'simple search' in search_switcher.value.lower() :
            return page
        else :
            payload = self.sessionSecrets(page)
            payload[search_switcher.name] = search_switcher.value

            page = self.lxmlize(self.LEGISLATION_URL, payload)

            if 'simple search' not in page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].value.lower() :
                raise ValueError('Not on the advanced search page')

            return page

    def details(self, detail_url, div_id) :
        detail_page = self.lxmlize(detail_url)
        
        detail_div = detail_page.xpath(".//div[@id='%s']" % div_id)[0]

        return self.parseDetails(detail_div)

    def legDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageDetails'
        return self.details(detail_url, div_id)

    def actionDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageTop1'
        return self.details(detail_url, div_id)

    def history(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        try :
            history_table = detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridLegislation_ctl00']")[0]
        except IndexError :
            print(detail_url)
            raise

        history = [row[0] for row in self.parseDataTable(history_table)] 

        try :
            history = sorted(history, key = self._actionSortKey)
        except (TypeError, ValueError) :
            pass

        for action in history :
            yield action

                    
    def _actionSortKey(self, action) :
        action_date = self.toDate(action['Date'])
        action_url = action['Action\xa0Details']['url']

        return (action_date, action_url)

    def text(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        text_div = detail_page.xpath("//div[@id='ctl00_ContentPlaceHolder1_divText']")

        if len(text_div) :
            return tostring(text_div[0], pretty_print=True).decode()
{'fraction_non_alphanumeric': 0.0973122461325728, 'fraction_numerical': 0.011148561057816956, 'mean_word_length': 2.4543283582089552, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1154177', 'n_tokens_mistral': 3245, 'n_tokens_neox': 3104, 'n_words': 756}