<commit_before>import json
from urllib.parse import parse_qs, urlparse

import scrapy
from scrapy.http import FormRequest

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsXMLFeedSpider


class UebermedienDeSpider(FeedsXMLFeedSpider):
    name = "uebermedien.de"
    allowed_domains = ["uebermedien.de", "steadyhq.com"]
    start_urls = ["https://uebermedien.de/feed/"]
    namespaces = [("dc", "http://purl.org/dc/elements/1.1/")]

    _title = "uebermedien.de"
    _subtitle = "Medien besser kritisieren."
    _steady_token = None

    def start_requests(self):
        username = self.spider_settings.get("username")
        password = <PASSWORD>("password")
        if username and password:
            yield scrapy.Request(
                "https://steadyhq.com/oauth/authorize?"
                + "client_id=0c29f006-1a98-48f1-8a63-2c0652c59f28&"
                + "redirect_uri=https://uebermedien.de&scope=read&"
                + "response_type=code&refresh_only=false",
                meta={"dont_cache": True},
                callback=self._steady_login,
            )
        else:
            self.logger.info("Login failed: No username or password given")
            # We can still try to scrape the free articles.
            yield from super().start_requests()

    def _steady_login(self, response):
        username = self.spider_settings.get("username")
        password = self.spider_settings.get("password")
        yield FormRequest.from_response(
            response,
            formdata={"user[email]": username, "user[password]": password},
            callback=self._request_steady_token,
            dont_filter=True,
            meta={"dont_cache": True, "handle_httpstatus_list": [301]},
        )

    def _request_steady_token(self, response):
        try:
            code = parse_qs(urlparse(response.url).query)["code"][0]
        except KeyError:
            self.logger.error("Login failed: Wrong username and password")
            return

        body = {
            "client_id": "0c29f006-1a98-48f1-8a63-2c0652c59f28",
            "grant_type": "authorization_code",
            "code": code,
            "redirect_uri": "https://uebermedien.de",
        }
        yield scrapy.Request(
            "https://steadyhq.com/api/v1/oauth/token",
            method="POST",
            body=json.dumps(body),
            headers={"Accept": "application/json", "Content-Type": "application/json"},
            meta={"dont_cache": True},
            callback=self._set_steady_token,
        )

    def _set_steady_token(self, response):
        self._steady_token = json.loads(response.text)["access_token"]
        return super().start_requests()

    def parse_node(self, response, node):
        il = FeedEntryItemLoader(<commit_msg>uebermedien.de: Cache steady login.<commit_after>import json
from urllib.parse import parse_qs, urlparse

import scrapy
from scrapy.http import FormRequest

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsXMLFeedSpider


class UebermedienDeSpider(FeedsXMLFeedSpider):
    name = "uebermedien.de"
    allowed_domains = ["uebermedien.de", "steadyhq.com"]
    start_urls = ["https://uebermedien.de/feed/"]
    namespaces = [("dc", "http://purl.org/dc/elements/1.1/")]

    _title = "uebermedien.de"
    _subtitle = "Medien besser kritisieren."
    _steady_token = None

    def start_requests(self):
        username = self.spider_settings.get("username")
        password = self.spider_settings.get("password")
        if username and password:
            yield scrapy.Request(
                "https://steadyhq.com/en/oauth/authorize?"
                + "client_id=0c29f006-1a98-48f1-8a63-2c0652c59f28&"
                + "redirect_uri=https://uebermedien.de&scope=read&"
                + "response_type=code&refresh_only=false",
                callback=self._steady_login,
            )
        else:
            self.logger.info("Login failed: No username or password given")
            # We can still try to scrape the free articles.
            yield from super().start_requests()

    def _steady_login(self, response):
        username = self.spider_settings.get("username")
        password = self.spider_settings.get("password")
        yield FormRequest.from_response(
            response,
            formdata={"user[email]": username, "user[password]": password},
            callback=self._request_steady_token,
            dont_filter=True,
            meta={"handle_httpstatus_list": [301]},
        )

    def _request_steady_token(self, response):
        try:
            code = parse_qs(urlparse(response.url).query)["code"][0]
        except KeyError:
            self.logger.error("Login failed: Wrong username and password")
            return

        body = {
            "client_id": "0c29f006-1a98-48f1-8a63-2c0652c59f28",
            "grant_type": "authorization_code",
            "code": code,
            "redirect_uri": "https://uebermedien.de",
        }
        yield scrapy.Request(
            "https://steadyhq.com/api/v1/oauth/token",
            method="POST",
            body=json.dumps(body),
            headers={"Accept": "application/json", "Content-Type": "application/json"},
            callback=self._set_steady_token,
        )

    def _set_steady_token(self, response):
        self._steady_token = json.loads(response.text)["access_token"]
        return super().start_requests()

    def parse_node(self, response, node):
        il = FeedEntryItemLoader(
{'fraction_non_alphanumeric': 0.11906909615731553, 'fraction_numerical': 0.01984484935955259, 'mean_word_length': 2.4542056074766356, 'pattern_counts': {'":': 21, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4799909', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1686, 'n_words': 294}