<commit_before>from urllib.parse import urljoin

import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider
from feeds.utils import generate_feed_header


class MomoxFashionComSpider(FeedsSpider):
    name = "momoxfashion.com"

    _base_url = f"https://www.{name}"
    _scrape_pages = 3

    def start_requests(self):
        links = self.settings.get("FEEDS_SPIDER_MOMOXFASHION_COM_LINKS")
        if links:
            links = links.split()
        else:
            links = ["de/herren?sortiertnach=neueste"]

        for link in links:
            yield scrapy.Request(
                urljoin(self._base_url, link), meta={"dont_cache": True, "path": link}
            )

    def feed_headers(self):
        return []

    def parse(self, response):
        if len(response.css(".thumbnail")) == 0:
            self.logger.info("No items found.")
            return

        for item in response.css(".thumbnail"):
            il = FeedEntryItemLoader(selector=item, base_url=self._base_url)
            il.add_css("title", ".item_brand_text ::text")
            il.add_css("title", ".item-title ::text")
            il.add_css("title", ".current-price ::text")
            il.add_value(
                "link",
                response.urljoin(item.css(".item-link::attr(href)").extract_first()),
            )
            image_url = item.css(".item-image::attr(data-bg)").re_first(
                r"url\(([^)]+)\)"
            )
            il.add_value("content_html", f'<img src="{image_url}">')
            il.add_css("content_html", ".item-des-container")
            il.add_value("path", response.meta["path"])
            yield il.load_item()
<commit_msg>momoxfashion.com: strip query params from link<commit_after>import re
from urllib.parse import urljoin

import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider
from feeds.utils import generate_feed_header


class MomoxFashionComSpider(FeedsSpider):
    name = "momoxfashion.com"

    _base_url = f"https://www.{name}"
    _scrape_pages = 3

    def start_requests(self):
        links = self.settings.get("FEEDS_SPIDER_MOMOXFASHION_COM_LINKS")
        if links:
            links = links.split()
        else:
            links = ["de/herren?sortiertnach=neueste"]

        for link in links:
            yield scrapy.Request(
                urljoin(self._base_url, link), meta={"dont_cache": True, "path": link}
            )

    def feed_headers(self):
        return []

    def parse(self, response):
        if len(response.css(".thumbnail")) == 0:
            self.logger.info("No items found.")
            return

        for item in response.css(".thumbnail"):
            il = FeedEntryItemLoader(selector=item, base_url=self._base_url)
            il.add_css("title", ".item_brand_text ::text")
            il.add_css("title", ".item-title ::text")
            il.add_css("title", ".current-price ::text")
            link = response.urljoin(
                re.sub(r"\?.*", "", item.css(".item-link::attr(href)").extract_first())
            )
            il.add_value("link", link)
            image_url = item.css(".item-image::attr(data-bg)").re_first(
                r"url\(([^)]+)\)"
            )
            il.add_value("content_html", f'<img src="{image_url}">')
            il.add_css("content_html", ".item-des-container")
            il.add_value("path", response.meta["path"])
            yield il.load_item()

{'fraction_non_alphanumeric': 0.1395015754798052, 'fraction_numerical': 0.0011458034947006588, 'mean_word_length': 2.272727272727273, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7465946', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1064, 'n_words': 196}