<commit_before># -*- coding: utf-8 -*-
import collections
import functools
import os
import re

import scrapy
from scrapy.http import FormRequest, Request, Response

SEMESTER = '20161'
TIMES = ['0730', '0820', '0910', '1010', '1100', '1330', '1420', '1510', '1620',
         '1710', '1830', '1920', '2020', '2110']


class CagrSpider(scrapy.Spider):
    name = 'cagr'
    allowed_domains = ['sistemas.ufsc.br']
    login_url = 'http://sistemas.ufsc.br/login'
    table_url = ('https://cagr.sistemas.ufsc.br/modules/aluno/'
                 'cadastroTurmas/index.xhtml')

    # current campus and subject being scraped
    campus, subject = tuple(), dict()
    campi_names, index = collections.OrderedDict(), None
    time_regex = re.compile(
        '(?P<day>\\d).(?P<time>\\d{4}).(?P<qty>\\d+)\\W*(?P<room>\\S*\\b)')

    def start_requests(self):
        def populate(res):
            if 'collecta.sistemas.ufsc.br' in res.url:
                yield FormRequest.from_response(res, 'j_id20',
                                                 callback=populate)
            else:
                for campus in res.css('[id="formBusca:selectCampus"] option'):
                    id_ = campus.css('::attr(value)').extract_first().strip()
                    name = campus.css('::text').extract_first().strip()[5:]
                    self.campi_names[id_] = name

                self.campus, self.index = self.campi_names.popitem(), res
                yield self.make_requests_from_index(page=1)<commit_msg>Don't use generators where possible<commit_after># -*- coding: utf-8 -*-
import collections
import functools
import os
import re

import scrapy
from scrapy.http import FormRequest, Request, Response

SEMESTER = '20161'
TIMES = ['0730', '0820', '0910', '1010', '1100', '1330', '1420', '1510', '1620',
         '1710', '1830', '1920', '2020', '2110']


class CagrSpider(scrapy.Spider):
    name = 'cagr'
    allowed_domains = ['sistemas.ufsc.br']
    login_url = 'http://sistemas.ufsc.br/login'
    table_url = ('https://cagr.sistemas.ufsc.br/modules/aluno/'
                 'cadastroTurmas/index.xhtml')

    # current campus and subject being scraped
    campus, subject = tuple(), dict()
    campi_names, index = collections.OrderedDict(), None
    time_regex = re.compile(
        '(?P<day>\\d).(?P<time>\\d{4}).(?P<qty>\\d+)\\W*(?P<room>\\S*\\b)')

    def start_requests(self):
        def populate(res):
            if 'collecta.sistemas.ufsc.br' in res.url:
                return FormRequest.from_response(res, 'j_id20',
                                                 callback=populate)

            for campus in res.css('[id="formBusca:selectCampus"] option'):
                id_ = campus.css('::attr(value)').extract_first().strip()
                name = campus.css('::text').extract_first().strip()[5:]
                self.campi_names[id_] = name

            self.campus, self.index = self.campi_names.popitem(), res
            return self.make_requests_from_index(page=1)
{'fraction_non_alphanumeric': 0.164054684894965, 'fraction_numerical': 0.04468156052017339, 'mean_word_length': 2.4602076124567476, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '197081', 'n_tokens_mistral': 1088, 'n_tokens_neox': 974, 'n_words': 190}