<commit_before># -*- coding: utf-8 -*-
import scrapy
from datetime import datetime
from ..items import OpengazettesItem
import romanify


class GazettesSpider(scrapy.Spider):
    name = "gazettes"
    allowed_domains = ["kenyalaw.org"]
    start_urls = ['http://kenyalaw.org/kenya_gazette/gazette/year/2007']

    def parse(self, response):
        # Extract all gazette links
        # Add publication date to metadata from table data
        rows = response.css('#content tr')
        for row in rows:
            gazette_meta = OpengazettesItem()
            gazette_meta['gazette_link'] = row.xpath(
                'td/a/@href').extract_first()
            if gazette_meta['gazette_link']:
                # Add volume and issue number to metadata from URL<commit_msg>Pass year to crawl to crawler at runtime<commit_after># -*- coding: utf-8 -*-
import scrapy
from datetime import datetime
from ..items import OpengazettesItem
import romanify


class GazettesSpider(scrapy.Spider):
    name = "gazettes"
    allowed_domains = ["kenyalaw.org"]

    def start_requests(self):
        # Get the year to be crawled from the arguments
        # The year is passed like this: scrapy crawl gazettes -a year=2017
        year = self.year
        url = 'http://kenyalaw.org/kenya_gazette/gazette/year/%s' % \
            (year)
        yield scrapy.Request(url, callback=self.parse)

    def parse(self, response):
        # Extract all gazette links
        # Add publication date to metadata from table data
        rows = response.css('#content tr')
        for row in rows:
            gazette_meta = OpengazettesItem()
            gazette_meta['gazette_link'] = row.xpath(
                'td/a/@href').extract_first()
            if gazette_meta['gazette_link']:
                # Add volume and issue number to metadata from URL
{'fraction_non_alphanumeric': 0.09666848716548335, 'fraction_numerical': 0.005461496450027308, 'mean_word_length': 2.5163147792706333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5619766', 'n_tokens_mistral': 550, 'n_tokens_neox': 534, 'n_words': 156}