<commit_before># -*- coding: utf-8 -*-
import re
from scrapy               import Spider
from dateutil             import parser
from artbot_scraper.items import EventItem
from pytz                 import timezone


class MCASpider(Spider):
    name            = 'MCA'
    allowed_domains = ['mca.com.au']
    start_urls      = ['http://www.mca.com.au/whatson/next-month/']

    def parse(self, response):
        for event in response.xpath('//div[contains(@class, "featured_item")]'):
            item                = EventItem()
            item['url']         = 'http://www.mca.com.au' + event.xpath('.//a/@href').extract_first()
            item['venue']       = self.name
            item['title']       = event.xpath('.//h2/text()').extract_first().strip()
            item['description'] = ''.join(event.xpath('.//div[@class="col-md-4"]/p[3]//text()').extract())
            item['image']       = response.urljoin(event.xpath('.//div[contains(@class, "featured-image")]/@style').re_first('(?<=\().*(?=\))'))

            season = event.xpath('.//b[contains(@class, "occurrence_date")]/text()').extract_first().strip().replace(u'\u2013\xa0', u'')
            match  = re.match('(?P<start>\d+\s+\w+)[\s\-]*(?P<end>\d+\s+\w+)', season)

            if (match):
                tz            = timezone('Australia/Sydney')
                item['start'] = tz.localize(parser.parse(match.group('start')))
                item['end']   = tz.localize(parser.parse(match.group('end')))

            yield item
<commit_msg>Excluded MCA permanent collection.<commit_after># -*- coding: utf-8 -*-
import re
from scrapy               import Spider
from dateutil             import parser
from artbot_scraper.items import EventItem
from pytz                 import timezone


class MCASpider(Spider):
    name            = 'MCA'
    allowed_domains = ['mca.com.au']
    start_urls      = ['http://www.mca.com.au/whatson/next-month/']

    def parse(self, response):
        for event in response.xpath('//div[contains(@class, "featured_item")]'):
            url = event.xpath('.//a/@href').extract_first()
            if 'mca-collection' in url:
                continue

            item                = EventItem()
            item['url']         = response.urljoin(url)
            item['venue']       = self.name
            item['title']       = event.xpath('.//h2/text()').extract_first().strip()
            item['description'] = ''.join(event.xpath('.//div[@class="col-md-4"]/p[3]//text()').extract())
            item['image']       = response.urljoin(event.xpath('.//div[contains(@class, "featured-image")]/@style').re_first('(?<=\().*(?=\))'))

            season = event.xpath('.//b[contains(@class, "occurrence_date")]/text()').extract_first().strip().replace(u'\u2013\xa0', u'')
            match  = re.match('(?P<start>\d+\s+\w+)[\s\-]*(?P<end>\d+\s+\w+)', season)

            if (match):
                tz            = timezone('Australia/Sydney')
                item['start'] = tz.localize(parser.parse(match.group('start')))
                item['end']   = tz.localize(parser.parse(match.group('end')))

            yield item

{'fraction_non_alphanumeric': 0.1996820349761526, 'fraction_numerical': 0.005723370429252782, 'mean_word_length': 2.0934119960668633, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '348814', 'n_tokens_mistral': 1009, 'n_tokens_neox': 959, 'n_words': 145}