<commit_before># -*- coding: utf-8 -*-

import re
import math
import urlparse

from scrapy.spider import BaseSpider
from scrapy.http import Request, FormRequest
from scrapy.selector import HtmlXPathSelector
from scrapy import log
from bs4 import BeautifulSoup

from registry.items import Corporation, Person, CorporationDocument, StatementDocument, RegistryStatement, PersonCorpRelation, RegistryExtract
from registry import pdfparse

class CorporationSpider(BaseSpider):
    name = "corps"
    page_by_page = True # Scrape page-by-page -- VERY slow (~19+ days)
                        # Guess db ids, MUCH faster for individual IDs,
                        # But the IDs aren't contiguous enough for this
                        # to be useful; there are ~500K corporations
                        # but ~10M IDs. Currently overruns memory.
    #allowed_domains = ["enreg.reestri.gov.ge"]
    base_url = "https://enreg.reestri.gov.ge/main.php"
    start_urls = [base_url]
    #MAX_CONSEC_MISSES = 100

    # Override so that we can set our cookie easily.
    def start_requests(self):
        #log.msg("in start_requests")
        my_url = self.base_url+"?c=app&m=search_form"
        yield Request(my_url, callback=self.parse_corpclasses)

    # Grabs the different corporation classes from the search
    # form dropdown menu.
    def parse_corpclasses(self, response):
        #log.msg("in parse_corpclasses")
        soup = BeautifulSoup(response.body, "html5lib", from_encoding="utf-8")
        form = soup.find(id="s_search_persons_form")
        if self.page_by_page == True:
            for opt in form.find_all("option"):
                # 0 is nothing. Skip individuals for dev.
                if(opt['value'] == '0' or opt['value'] == '1'):
                    continue
                #log.msg("Found corp class: {}".format(opt['value']))
            
                request = Request(self.base_url, 
                              #formdata=form_data,
                              callback=self.setup_cookies,
                              dont_filter=True,
                              meta={'cookiejar': opt['value'],
                                    'corp_class': opt['value']})
                yield request
        else: # Guess ID numbers instead
            form_data ={'c': 'search',
                    'm': 'find_legal_persons',
                    's_legal_person_idnumber':'',
                    's_legal_person_name':'',
                    's_legal_person_form': '1'}
        
            yield FormRequest(self.base_url,
                            formdata=form_data,
                            callback=self.guess_biggest_id)

    # This works but overflows memory because there are about
    # 10M id numbers but only about 500k real corporations.
    # Used when page_by_page set to True, so don't do that right now.
    def guess_biggest_id(self, response):
        """ We assume that corporation listings are listed
        approximately in the order that they were created.
        Therefore the most recently updated corporation is
        the one first on the list. This is usually going to
        be on the individual entrepreneurs page, it has the
        most activity. So scrape that, get the first id, and
        then count up to that; that'll give us a rough idea
        of how many there are."""
        soup = BeautifulSoup(response.body, "html5lib", from_encoding="utf-8")
        buttons = soup.find_all("img",src="https://enreg.reestri.gov.ge/images/info.png")
        results = []
        
        for b in buttons:
            dbid = b.parent['onclick'].split("(")[-1].rstrip(")")
            results.append(int(dbid))
        
        biggest = sorted(results)[-1]+1
        log.msg("Guessing biggest is {}".format(biggest)) 
        for dbid in range(0,biggest):
            corp_url = self.base_url+"?c=app&m=show_legal_person&legal_code_id={}".format(dbid)
            request = Request(url=corp_url,callback=self.parse_corpdetails)
            request.meta['id_code_reestri_db'] = dbid
            request.meta['cookiejar'] = '1' # Don't need a separate jar
            yield request

    # This site does some incredibly stupid things with cookies
    # so we need to use a separate cookie jar for each type of
    # corporation that we will scrape.
    def setup_cookies(self, response):
        form_data ={'c': 'search',
                    'm': 'find_legal_persons',
                    's_legal_person_idnumber':'',
                    's_legal_person_name':''}
        
        form_data['s_legal_person_form'] = response.meta['cookiejar']

        request = FormRequest(self.base_url,
                            formdata=form_data,
                            callback=self.parse_corpresults,
                            meta={'cookiejar': response.meta['cookiejar']})
        if 'renew' in response.meta:
            request.meta['renew'] = response.meta['renew']
            request.meta['page'] = response.meta['page']
        yield request
    
    # Finds out how many pages of results there are and launches
    # requests for them.
    def parse_corpresults(self, response):
        # The total number of records is listed at the bottom of the table
        # Divide by 5 records per page to get the total number of pages
        # we need to scrape.
        RESULTS_PER_PAGE = 5
        form_data ={'c': 'search',
                    'm': 'find_legal_persons',
                   }
        # The site does some dumb things with session variables, sometimes
        # they need to get renewed.
        if 'renew' in response.meta:
            form_data['p'] = response.meta['page']
            request = FormRequest(self.base_url,
                              formdata=form_data,
                              callback=self.parse_corptable,
                              meta={'cookiejar': response.meta['cookiejar'],
                                    'page': response.meta['page'],
                                    'type': response.meta['cookiejar'],
                                    })
            yield request
        # Otherwise, this is our first time viewing this result type, and
        # we start from the beginning.
        soup = BeautifulSoup(response.body, "html5lib", from_encoding="utf-8")
        cells = soup.find_all("td")
        td = None

        # The number of results is in a <td> that contains the text
        # სულ.
        # After that, there is a <strong> tag that has the actual number
        # in it, and then some other stuff. So we search through all
        # the td tags until we find the one with matching text,
        # and then grab the number in its <strong> tag.
        regx = re.compile(u"^\s+სულ\s+$")
        for cell in cells:
            if (regx.match(cell.contents[0])):
                td = cell #Found the right cell!
                break;

        total_results = float(td.find("strong").string)
        total_pages = int(math.ceil(total_results/RESULTS_PER_PAGE))
        #log.msg("Total pages: {}".format(str(total_pages)))<commit_msg>Forgot to add dont_filter=True to cookie renewal requests.<commit_after># -*- coding: utf-8 -*-

import re
import math
import urlparse

from scrapy.spider import BaseSpider
from scrapy.http import Request, FormRequest
from scrapy.selector import HtmlXPathSelector
from scrapy import log
from bs4 import BeautifulSoup

from registry.items import Corporation, Person, CorporationDocument, StatementDocument, RegistryStatement, PersonCorpRelation, RegistryExtract
from registry import pdfparse

class CorporationSpider(BaseSpider):
    name = "corps"
    page_by_page = True # Scrape page-by-page -- VERY slow (~19+ days)
                        # Guess db ids, MUCH faster for individual IDs,
                        # But the IDs aren't contiguous enough for this
                        # to be useful; there are ~500K corporations
                        # but ~10M IDs. Currently overruns memory.
    #allowed_domains = ["enreg.reestri.gov.ge"]
    base_url = "https://enreg.reestri.gov.ge/main.php"
    start_urls = [base_url]
    #MAX_CONSEC_MISSES = 100

    # Override so that we can set our cookie easily.
    def start_requests(self):
        #log.msg("in start_requests")
        my_url = self.base_url+"?c=app&m=search_form"
        yield Request(my_url, callback=self.parse_corpclasses)

    # Grabs the different corporation classes from the search
    # form dropdown menu.
    def parse_corpclasses(self, response):
        #log.msg("in parse_corpclasses")
        soup = BeautifulSoup(response.body, "html5lib", from_encoding="utf-8")
        form = soup.find(id="s_search_persons_form")
        if self.page_by_page == True:
            for opt in form.find_all("option"):
                # 0 is nothing. Skip individuals for dev.
                if(opt['value'] == '0' or opt['value'] == '1'):
                    continue
                #log.msg("Found corp class: {}".format(opt['value']))
            
                request = Request(self.base_url, 
                              #formdata=form_data,
                              callback=self.setup_cookies,
                              dont_filter=True,
                              meta={'cookiejar': opt['value'],
                                    'corp_class': opt['value']})
                yield request
        else: # Guess ID numbers instead
            form_data ={'c': 'search',
                    'm': 'find_legal_persons',
                    's_legal_person_idnumber':'',
                    's_legal_person_name':'',
                    's_legal_person_form': '1'}
        
            yield FormRequest(self.base_url,
                            formdata=form_data,
                            callback=self.guess_biggest_id)

    # This works but overflows memory because there are about
    # 10M id numbers but only about 500k real corporations.
    # Used when page_by_page set to True, so don't do that right now.
    def guess_biggest_id(self, response):
        """ We assume that corporation listings are listed
        approximately in the order that they were created.
        Therefore the most recently updated corporation is
        the one first on the list. This is usually going to
        be on the individual entrepreneurs page, it has the
        most activity. So scrape that, get the first id, and
        then count up to that; that'll give us a rough idea
        of how many there are."""
        soup = BeautifulSoup(response.body, "html5lib", from_encoding="utf-8")
        buttons = soup.find_all("img",src="https://enreg.reestri.gov.ge/images/info.png")
        results = []
        
        for b in buttons:
            dbid = b.parent['onclick'].split("(")[-1].rstrip(")")
            results.append(int(dbid))
        
        biggest = sorted(results)[-1]+1
        log.msg("Guessing biggest is {}".format(biggest)) 
        for dbid in range(0,biggest):
            corp_url = self.base_url+"?c=app&m=show_legal_person&legal_code_id={}".format(dbid)
            request = Request(url=corp_url,callback=self.parse_corpdetails)
            request.meta['id_code_reestri_db'] = dbid
            request.meta['cookiejar'] = '1' # Don't need a separate jar
            yield request

    # This site does some incredibly stupid things with cookies
    # so we need to use a separate cookie jar for each type of
    # corporation that we will scrape.
    def setup_cookies(self, response):
        form_data ={'c': 'search',
                    'm': 'find_legal_persons',
                    's_legal_person_idnumber':'',
                    's_legal_person_name':''}
        
        form_data['s_legal_person_form'] = response.meta['cookiejar']

        if 'renew' in response.meta:
            request = FormRequest(self.base_url,
                            dont_filter=True,
                            formdata=form_data,
                            callback=self.parse_corpresults,
                            meta={'cookiejar': response.meta['cookiejar'],
                                  'renew': response.meta['renew'],
                                  'page': response.meta['page'],})
        else:
            request = FormRequest(self.base_url,
                            formdata=form_data,
                            callback=self.parse_corpresults,
                            meta={'cookiejar': response.meta['cookiejar']})

        yield request
    
    # Finds out how many pages of results there are and launches
    # requests for them.
    def parse_corpresults(self, response):
        # The total number of records is listed at the bottom of the table
        # Divide by 5 records per page to get the total number of pages
        # we need to scrape.
        RESULTS_PER_PAGE = 5
        form_data ={'c': 'search',
                    'm': 'find_legal_persons',
                   }
        # The site does some dumb things with session variables, sometimes
        # they need to get renewed.
        if 'renew' in response.meta:
            form_data['p'] = response.meta['page']
            request = FormRequest(self.base_url,
                              dont_filter=True,
                              formdata=form_data,
                              callback=self.parse_corptable,
                              meta={'cookiejar': response.meta['cookiejar'],
                                    'page': response.meta['page'],
                                    'type': response.meta['cookiejar'],
                                    })
            yield request
        # Otherwise, this is our first time viewing this result type, and
        # we start from the beginning.
        soup = BeautifulSoup(response.body, "html5lib", from_encoding="utf-8")
        cells = soup.find_all("td")
        td = None

        # The number of results is in a <td> that contains the text
        # სულ.
        # After that, there is a <strong> tag that has the actual number
        # in it, and then some other stuff. So we search through all
        # the td tags until we find the one with matching text,
        # and then grab the number in its <strong> tag.
        regx = re.compile(u"^\s+სულ\s+$")
        for cell in cells:
            if (regx.match(cell.contents[0])):
                td = cell #Found the right cell!
                break;

        total_results = float(td.find("strong").string)
        total_pages = int(math.ceil(total_results/RESULTS_PER_PAGE))
        #log.msg("Total pages: {}".format(str(total_pages)))
{'fraction_non_alphanumeric': 0.09432667901488646, 'fraction_numerical': 0.004802085477121493, 'mean_word_length': 1.8747781502662197, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4212761', 'n_tokens_mistral': 4022, 'n_tokens_neox': 3821, 'n_words': 1157}