<commit_before># -*- coding: utf-8 -*-
import scrapy
import scrapy_proj.items

class SanatateSpider(scrapy.Spider):
    name = "sanatate"

    def start_requests(self):
        urls = [
            'http://www.ms.ro/acte-normative-in-transparenta/?vpage=1',
        ]

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):

        for item in response.css('.panel'):
            heading = item.css('div.panel-heading')
            body = item.css('div.panel-body')

            title = item.css('a.panel-title::text').extract_first()
            contact = {
                'name': body.xpath('//p[contains(text(), "Contact")]/text()').re_first(r'Contact:\s*(.*)')
            }

            yield scrapy_proj.items.ActItem(
                title = title,
                contact = contact
            )

        next_pages = response.css('.pt-cv-pagination a::attr(href)').extract()
        next_pages.reverse()
        for next_page in next_pages:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)<commit_msg>added extra documents to act items<commit_after># -*- coding: utf-8 -*-
import scrapy
import scrapy_proj.items

class SanatateSpider(scrapy.Spider):
    name = "sanatate"

    def start_requests(self):
        urls = [
            'http://www.ms.ro/acte-normative-in-transparenta/?vpage=1',
        ]

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):

        for item in response.css('.panel'):
            heading = item.css('div.panel-heading')
            body = item.css('div.panel-body')

            documents_anchors = body.xpath('.//a[contains(@href, ".pdf")]')
            documents = []

            for anchor in documents_anchors:
                href = response.urljoin(anchor.xpath('.//@href').extract_first())
                name = anchor.xpath('.//text()').extract_first()
                documents.append({
                    'type': name,
                    'url': href
                })

            title = item.css('a.panel-title::text').extract_first()
            contact = {
                'name': body.xpath('.//p[contains(text(), "Contact")]/text()').re_first(r'Contact:\s*(.*)')
            }

            anchors = body.css('a')

            for anchor in anchors:
                href = anchor.css('::attr(href)').extract()

            yield scrapy_proj.items.ActItem(
                title = title,
                contact = contact,
                documents = documents
            )

        next_pages = response.css('.pt-cv-pagination a::attr(href)').extract()
        next_pages.reverse()
        for next_page in next_pages:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)
{'fraction_non_alphanumeric': 0.1423647018269562, 'fraction_numerical': 0.0013788348845225785, 'mean_word_length': 1.8478900883218843, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4319766', 'n_tokens_mistral': 868, 'n_tokens_neox': 825, 'n_words': 147}