<commit_before># -*- coding: utf-8 -*-
import scrapy
from datetime import datetime
from ..items import OpengazettesItem
import romanify


class GazettesSpider(scrapy.Spider):
    name = "gazettes"
    allowed_domains = ["kenyalaw.org"]

    def start_requests(self):
        # Get the year to be crawled from the arguments
        # The year is passed like this: scrapy crawl gazettes -a year=2017
        # Default to current year if year not passed in
        try:
            year = self.year
        except AttributeError:
            year = datetime.now().strftime('%Y')

        url = 'http://kenyalaw.org/kenya_gazette/gazette/year/%s' % \
            (year)
        yield scrapy.Request(url, callback=self.parse)

    def parse(self, response):
        # Get all rows in the "Weekly Issues" div
        weekly_rows = response.xpath('//*[@id="content"]/div[1]/table/tr')
        # Get all rows in the "Special Issues" div
        special_rows = response.xpath('//*[@id="content"]/div[2]/table/tr')

        no_of_weekly_issues = len(weekly_rows)

        rows = weekly_rows + special_rows
        row_counter = 0
        previous_volume_number = False

        for row in rows:
            # Immediately increment row_counter
            row_counter += 1
            gazette_meta = OpengazettesItem()

            # If we have already gone through all weekly, this is special
            # Otherwise, it is still a weekly issue
            if row_counter > no_of_weekly_issues:
                gazette_meta['special_issue'] = True
            else:
                gazette_meta['special_issue'] = False

            gazette_meta['gazette_link'] = row.xpath(
                'td/a/@href').extract_first()

            if gazette_meta['gazette_link']:
                # Add volume and issue number to metadata from URL
                # Here, we replace l with I to handle human input error
                # BEWARE: This might cause weird behaviour in future
                if not previous_volume_number:
                    gazette_meta['gazette_volume'] = romanify.roman2arabic(
                        row.xpath('td/a/@href')
                        .re(r'(Vol*.*No)|(\/ol*.*No)')[0]
                        .replace('Vol', '').replace('VoI', '')
                        .replace('\/ol', '').replace('.', '').replace('-', '')
                        .replace(' ', '').replace('l', 'I').replace('No', ''))

                    previous_volume_number = gazette_meta['gazette_volume']
                else:
                    gazette_meta['gazette_volume'] = previous_volume_number

                gazette_meta['gazette_number'] = row.xpath('td/a/@href')\
                    .re(r'No*.*')[0].replace('No', '').replace('.', '')\
                    .replace(' ', '')

                # Add publication date to metadata from table data
                gazette_meta['publication_date'] = datetime.strptime(
                    row.xpath('td/text()')[1].extract(), '%d %B,%Y')

                request = scrapy.Request(gazette_meta['gazette_link'],<commit_msg>Fix gazette_volume showing unwanted URL segments<commit_after># -*- coding: utf-8 -*-
import scrapy
from datetime import datetime
from ..items import OpengazettesItem
import romanify


class GazettesSpider(scrapy.Spider):
    name = "gazettes"
    allowed_domains = ["kenyalaw.org"]

    def start_requests(self):
        # Get the year to be crawled from the arguments
        # The year is passed like this: scrapy crawl gazettes -a year=2017
        # Default to current year if year not passed in
        try:
            year = self.year
        except AttributeError:
            year = datetime.now().strftime('%Y')

        url = 'http://kenyalaw.org/kenya_gazette/gazette/year/%s' % \
            (year)
        yield scrapy.Request(url, callback=self.parse)

    def parse(self, response):
        # Get all rows in the "Weekly Issues" div
        weekly_rows = response.xpath('//*[@id="content"]/div[1]/table/tr')
        # Get all rows in the "Special Issues" div
        special_rows = response.xpath('//*[@id="content"]/div[2]/table/tr')

        no_of_weekly_issues = len(weekly_rows)

        rows = weekly_rows + special_rows
        row_counter = 0
        previous_volume_number = False

        for row in rows:
            # Immediately increment row_counter
            row_counter += 1
            gazette_meta = OpengazettesItem()

            # If we have already gone through all weekly, this is special
            # Otherwise, it is still a weekly issue
            if row_counter > no_of_weekly_issues:
                gazette_meta['special_issue'] = True
            else:
                gazette_meta['special_issue'] = False

            gazette_meta['gazette_link'] = row.xpath(
                'td/a/@href').extract_first()

            if gazette_meta['gazette_link']:
                # Add volume and issue number to metadata from URL
                # Here, we replace l with I to handle human input error
                # BEWARE: This might cause weird behaviour in future
                if not previous_volume_number:
                    gazette_meta['gazette_volume'] = romanify.roman2arabic(
                        row.xpath('td/a/@href')
                        .re(r'Vol*.*No')[0]
                        .replace('Vol', '').replace('.', '').replace('-', '')
                        .replace(' ', '').replace('l', 'I').replace('No', ''))

                    previous_volume_number = gazette_meta['gazette_volume']
                else:
                    gazette_meta['gazette_volume'] = previous_volume_number

                gazette_meta['gazette_number'] = row.xpath('td/a/@href')\
                    .re(r'No *.*')[0].replace('No', '').replace('.', '')\
                    .replace(' ', '')

                # Add publication date to metadata from table data
                gazette_meta['publication_date'] = datetime.strptime(
                    row.xpath('td/text()')[1].extract(), '%d %B,%Y')

                request = scrapy.Request(gazette_meta['gazette_link'],
{'fraction_non_alphanumeric': 0.12085385878489327, 'fraction_numerical': 0.004269293924466338, 'mean_word_length': 1.8990956687291767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '224395', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1684, 'n_words': 417}