<commit_before>import re
from urllib.parse import (
    parse_qs,
    quote,
    unquote,
    urlencode,
    urlparse,
    urlunparse,
)

from scrapy import Spider
from scrapy.http import Request
from scrapy.selector import Selector

ref = re.compile(r"<(https?:[^>]*)>")


def rebuild(components, query):
    return urlunparse(
        (*components[:4], urlencode(query, doseq=True), *components[5:])
    )


def repl(match):
    return quote(match.group(0))


class JurisSpider(Spider):
    name = "juris"
    allowed_domains = ["stf.jus.br"]

    def __init__(self, url):
        self.start_url = url

    def start_requests(self):
        components = urlparse(self.start_url)
        query = parse_qs(components[4])
        query.pop("pagina")

        def parse_start_url(res):
            match = re.search(
                r"\d+\s*/\s*(\d+)",
                res.selector.css(
                    "#divNaoImprimir > table:nth-child(1) td:nth-child(3)::text"
                ).extract_first(),
            )
            if not match:
                return

            for i in range(1, int(match.group(1)) + 1):
                yield Request(rebuild(components, {**query, "pagina": i}))

        yield Request(rebuild(components, query), callback=parse_start_url)

    def parse(self, res):
        for d in (
            Selector(text=ref.sub(repl, res.text))
            .xpath(
                u'//p[strong[text()="Doutrina"]]/following-sibling::pre/text()'
            )<commit_msg>Fix error on missing 'pagina' param<commit_after>import re
from urllib.parse import (
    parse_qs,
    quote,
    unquote,
    urlencode,
    urlparse,
    urlunparse,
)

from scrapy import Spider
from scrapy.http import Request
from scrapy.selector import Selector

ref = re.compile(r"<(https?:[^>]*)>")


def rebuild(components, query):
    return urlunparse(
        (*components[:4], urlencode(query, doseq=True), *components[5:])
    )


def repl(match):
    return quote(match.group(0))


class JurisSpider(Spider):
    name = "juris"
    allowed_domains = ["stf.jus.br"]

    def __init__(self, url):
        self.start_url = url

    def start_requests(self):
        components = urlparse(self.start_url)
        query = parse_qs(components[4])
        query.pop("pagina", None)

        def parse_start_url(res):
            match = re.search(
                r"\d+\s*/\s*(\d+)",
                res.selector.css(
                    "#divNaoImprimir > table:nth-child(1) td:nth-child(3)::text"
                ).extract_first(),
            )
            if not match:
                return

            for i in range(1, int(match.group(1)) + 1):
                yield Request(rebuild(components, {**query, "pagina": i}))

        yield Request(rebuild(components, query), callback=parse_start_url)

    def parse(self, res):
        for d in (
            Selector(text=ref.sub(repl, res.text))
            .xpath(
                u'//p[strong[text()="Doutrina"]]/following-sibling::pre/text()'
            )
{'fraction_non_alphanumeric': 0.13817097415506957, 'fraction_numerical': 0.005964214711729622, 'mean_word_length': 2.167890870933893, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852654', 'n_tokens_mistral': 1003, 'n_tokens_neox': 957, 'n_words': 196}