<commit_before>#-*- coding: utf-8 -*-
import re
from scrapy               import Spider, Request
from dateutil             import parser, relativedelta
from artbot_scraper.items import EventItem

class AGNSpider(Spider):
    name            = 'AGN'
    allowed_domains = ['artgallery.nsw.gov.au']
    start_urls      = ['http://www.artgallery.nsw.gov.au/exhibitions/current/']

    def parse(self, response):
        for href in response.xpath('//div[contains(@class, "currentExhibition")]//a[1]/@href'):
            url = response.urljoin(href.extract())

            yield Request(url, callback=self.parse_current_exhibition)

    def parse_current_exhibition(self, response):
        item = EventItem()
        item['url']         = response.url
        item['venue']       = 'AGN'
        item['title']       = response.xpath('.//h2//text()').extract_first().strip()
        item['description'] = ''.join(response.xpath('.//div[contains(@class, "lumpy-main")]//text()').extract()).strip()
        item['image']       = 'http://www.artgallery.nsw.gov.au' + response.xpath('.//div[contains(@id, "content")]//img/@src').extract_first()

        season  = response.xpath('.//div[contains(@class, "exhib-details")]//h3/text()').extract_first().strip()
        match   = re.match(u'(?P<start>\d+\s+\w+)[\s\-\–]*(?P<end>\d+\s+\w+)', season, re.UNICODE)

        if (match):
            start = parser.parse(match.group('start'))
            end   = parser.parse(match.group('end'))

            if (end < start):
                end = end + relativedelta.relativedelta(years=+1)
<commit_msg>Updated season regex to support new format.<commit_after>#-*- coding: utf-8 -*-
import re
from scrapy               import Spider, Request
from dateutil             import parser, relativedelta
from artbot_scraper.items import EventItem

class AGNSpider(Spider):
    name            = 'AGN'
    allowed_domains = ['artgallery.nsw.gov.au']
    start_urls      = ['http://www.artgallery.nsw.gov.au/exhibitions/current/']

    def parse(self, response):
        for href in response.xpath('//div[contains(@class, "currentExhibition")]//a[1]/@href'):
            url = response.urljoin(href.extract())

            yield Request(url, callback=self.parse_current_exhibition)

    def parse_current_exhibition(self, response):
        item = EventItem()
        item['url']         = response.url
        item['venue']       = 'AGN'
        item['title']       = response.xpath('.//h2//text()').extract_first().strip()
        item['description'] = ''.join(response.xpath('.//div[contains(@class, "lumpy-main")]//text()').extract()).strip()
        item['image']       = 'http://www.artgallery.nsw.gov.au' + response.xpath('.//div[contains(@id, "content")]//img/@src').extract_first()

        season  = ' '.join(response.xpath('.//div[contains(@class, "exhib-details")]//h3/text()').extract()).strip()
        match   = re.search(u'(?P<start>\d+\s+\w+[\s+\d]*)[\s\-\–]*(?P<end>\d+\s+\w+\s+\d+)$', season, re.UNICODE)

        if (match):
            start = parser.parse(match.group('start'))
            end   = parser.parse(match.group('end'))

            if (end < start):
                end = end + relativedelta.relativedelta(years=+1)

{'fraction_non_alphanumeric': 0.1912585244885307, 'fraction_numerical': 0.0030998140111593306, 'mean_word_length': 2.617713004484305, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3087408', 'n_tokens_mistral': 1047, 'n_tokens_neox': 1011, 'n_words': 160}