<commit_before>import mechanize
from bs4 import BeautifulSoup


class RegulatedEntityScraper(object):
    url = "http://www15.tceq.texas.gov/crpub/index.cfm?fuseaction=regent.RNSearch"

    def __init__(self, rn_number):
        self.rn_number = rn_number
        br = mechanize.Browser()
        br.set_handle_robots(False)
        br.open(self.url)
        br.select_form(nr=0)
        br.form['re_ref_num_txt'] = self.rn_number
        response = br.submit()
        self.regulated_entity_url = response.geturl()
        self.html = response.read()


    def get_values(self):
        output = {}
        soup = BeautifulSoup(self.html)
        return {
            'regulated_entity_rn_number': self.rn_number,
            'url': self.regulated_entity_url,
            'name': self.get_name(soup),
            'primary_business': self.get_primary_business(soup),
            'street_address': self.get_street_address(soup),
            'county': self.get_county(soup),
            'nearest_city': self.get_nearest_city(soup),
            'nearest_zipcode': self.get_nearest_zipcode(soup),
            'physical_location': self.get_physical_location(soup)
        }

    def get_permits(self):
        output = []
        soup = BeautifulSoup(self.html)

        for tr in self._get_permit_trs(soup)[1:-1]:
            output.append(
                {
                    'program': self._get_program(tr),
                    'id_type': self._get_id_type(tr),
                    'id_number': self._get_id_number(tr),
                    'id_status': self._get_id_status(tr),
                    'url': self._get_url(tr)
                }
            )
        return output

    def has_page(self):
        return self.regulated_entity_url.find('notfound') < 0

    def get_name(self, soup):
        return soup.find(id='reinfo').find_all('p')[1].contents[-1].strip()

    def get_primary_business(self, soup):
        return soup.find(id='reinfo').find_all('p')[2].contents[-1].strip()

    def get_street_address(self, soup):
        return soup.find(id='street_addr').find_all('p')[0].contents[-1].strip()

    def get_county(self, soup):
        return soup.find(id='geo_loc').find_all('p')[0].contents[-1].strip()

    def get_nearest_city(self, soup):
        return soup.find(id='geo_loc').find_all('p')[1].contents[-1].strip()

    def get_nearest_zipcode(self, soup):
        return soup.find(id='geo_loc').find_all('p')[3].contents[-1].strip()

    def get_physical_location(self, soup):
        return soup.find(id='geo_loc').find_all('p')[4].contents[-1].strip()
<commit_msg>Fix regulated entities without a name<commit_after>import mechanize
from bs4 import BeautifulSoup


class RegulatedEntityScraper(object):
    url = "http://www15.tceq.texas.gov/crpub/index.cfm?fuseaction=regent.RNSearch"

    def __init__(self, rn_number):
        self.rn_number = rn_number
        br = mechanize.Browser()
        br.set_handle_robots(False)
        br.open(self.url)
        br.select_form(nr=0)
        br.form['re_ref_num_txt'] = self.rn_number
        response = br.submit()
        self.regulated_entity_url = response.geturl()
        self.html = response.read()


    def get_values(self):
        output = {}
        soup = BeautifulSoup(self.html)
        return {
            'regulated_entity_rn_number': self.rn_number,
            'url': self.regulated_entity_url,
            'name': self.get_name(soup),
            'primary_business': self.get_primary_business(soup),
            'street_address': self.get_street_address(soup),
            'county': self.get_county(soup),
            'nearest_city': self.get_nearest_city(soup),
            'nearest_zipcode': self.get_nearest_zipcode(soup),
            'physical_location': self.get_physical_location(soup)
        }

    def get_permits(self):
        output = []
        soup = BeautifulSoup(self.html)

        for tr in self._get_permit_trs(soup)[1:-1]:
            output.append(
                {
                    'program': self._get_program(tr),
                    'id_type': self._get_id_type(tr),
                    'id_number': self._get_id_number(tr),
                    'id_status': self._get_id_status(tr),
                    'url': self._get_url(tr)
                }
            )
        return output

    def has_page(self):
        return self.regulated_entity_url.find('notfound') < 0

    def get_name(self, soup):
        if soup.find(id='reinfo').find_all('p')[1].find('a'):
            return soup.find(id='reinfo').find_all('p')[1].contents[2].strip()
        else:
            return soup.find(id='reinfo').find_all('p')[1].contents[-1].strip()

    def get_primary_business(self, soup):
        return soup.find(id='reinfo').find_all('p')[2].contents[-1].strip()

    def get_street_address(self, soup):
        return soup.find(id='street_addr').find_all('p')[0].contents[-1].strip()

    def get_county(self, soup):
        return soup.find(id='geo_loc').find_all('p')[0].contents[-1].strip()

    def get_nearest_city(self, soup):
        return soup.find(id='geo_loc').find_all('p')[1].contents[-1].strip()

    def get_nearest_zipcode(self, soup):
        return soup.find(id='geo_loc').find_all('p')[3].contents[-1].strip()

    def get_physical_location(self, soup):
        return soup.find(id='geo_loc').find_all('p')[4].contents[-1].strip()

{'fraction_non_alphanumeric': 0.14445479962721341, 'fraction_numerical': 0.008387698042870456, 'mean_word_length': 2.5163826998689385, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4621354', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1743, 'n_words': 238}