<commit_before>import abc
from typing import List

from mediawords.db import DatabaseHandler
from mediawords.dbi.downloads.store import store_content
from mediawords.util.log import create_logger
from mediawords.util.perl import decode_object_from_bytes_if_needed

from crawler_fetcher.exceptions import McCrawlerFetcherSoftError
from crawler_fetcher.handler import AbstractDownloadHandler
from crawler_fetcher.handlers.default.store_mixin import DefaultStoreMixin

log = create_logger(__name__)


class AbstractDownloadFeedHandler(DefaultStoreMixin, AbstractDownloadHandler, metaclass=abc.ABCMeta):
    """
    Handler for 'feed' downloads.

    The feed handler parses the feed and looks for the URLs of any new stories. A story is considered new if the URL or
    GUID is not already in the database for the given media source and if the story title is unique for the media source
    for the calendar week. If the story is new, a story is added to the stories table and a download with a type of
    'pending' is added to the downloads table.
    """

    @abc.abstractmethod
    def add_stories_from_feed(self, db: DatabaseHandler, download: dict, content: str) -> List[int]:
        """
        Return a list of new, to-be-fetcher story IDs that were added from the feed.

        For example, if 'syndicated' feed had three new stories, implementation would add them to "stories" table and
        return a list of story IDs that are to be fetched later.

        If helper returns an empty arrayref, '(redundant feed)' will be written instead of feed contents.
        """
        raise NotImplemented("Abstract method")

    @abc.abstractmethod
    def return_stories_to_be_extracted_from_feed(self, db: DatabaseHandler, download: dict, content: str) -> List[int]:
        """
        Return a list of stories that have to be extracted from this feed.

        For example, 'web_page' feed creates a single story for itself so it has to be extracted right away.
        """
        raise NotImplemented("Abstract method")

    def store_download(self, db: DatabaseHandler, download: dict, content: str) -> List[int]:
        download = decode_object_from_bytes_if_needed(download)
        content = decode_object_from_bytes_if_needed(content)

        downloads_id = download['downloads_id']

        log.info(f"Processing feed download {downloads_id}...")

        try:
            added_story_ids = self.add_stories_from_feed(db=db, download=download, content=content)
            story_ids_to_extract = self.add_stories_from_feed(db=db, download=download, content=content)

        except Exception as ex:

            error_message = f"Error processing feed for download {downloads_id}: {ex}"
            log.error(error_message)
<commit_msg>Add 'web_page' stories to the extraction queue<commit_after>import abc
from typing import List

from mediawords.db import DatabaseHandler
from mediawords.dbi.downloads.store import store_content
from mediawords.util.log import create_logger
from mediawords.util.perl import decode_object_from_bytes_if_needed

from crawler_fetcher.exceptions import McCrawlerFetcherSoftError
from crawler_fetcher.handler import AbstractDownloadHandler
from crawler_fetcher.handlers.default.store_mixin import DefaultStoreMixin

log = create_logger(__name__)


class AbstractDownloadFeedHandler(DefaultStoreMixin, AbstractDownloadHandler, metaclass=abc.ABCMeta):
    """
    Handler for 'feed' downloads.

    The feed handler parses the feed and looks for the URLs of any new stories. A story is considered new if the URL or
    GUID is not already in the database for the given media source and if the story title is unique for the media source
    for the calendar week. If the story is new, a story is added to the stories table and a download with a type of
    'pending' is added to the downloads table.
    """

    @abc.abstractmethod
    def add_stories_from_feed(self, db: DatabaseHandler, download: dict, content: str) -> List[int]:
        """
        Return a list of new, to-be-fetcher story IDs that were added from the feed.

        For example, if 'syndicated' feed had three new stories, implementation would add them to "stories" table and
        return a list of story IDs that are to be fetched later.

        If helper returns an empty arrayref, '(redundant feed)' will be written instead of feed contents.
        """
        raise NotImplemented("Abstract method")

    @abc.abstractmethod
    def return_stories_to_be_extracted_from_feed(self, db: DatabaseHandler, download: dict, content: str) -> List[int]:
        """
        Return a list of stories that have to be extracted from this feed.

        For example, 'web_page' feed creates a single story for itself so it has to be extracted right away.
        """
        raise NotImplemented("Abstract method")

    def store_download(self, db: DatabaseHandler, download: dict, content: str) -> List[int]:
        download = decode_object_from_bytes_if_needed(download)
        content = decode_object_from_bytes_if_needed(content)

        downloads_id = download['downloads_id']

        log.info(f"Processing feed download {downloads_id}...")

        try:
            added_story_ids = self.add_stories_from_feed(db=db, download=download, content=content)
            story_ids_to_extract = self.return_stories_to_be_extracted_from_feed(
                db=db, download=download, content=content,
            )

        except Exception as ex:

            error_message = f"Error processing feed for download {downloads_id}: {ex}"
            log.error(error_message)

{'fraction_non_alphanumeric': 0.06418737707849097, 'fraction_numerical': 0.0, 'mean_word_length': 3.6117065127782357, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6078852', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1482, 'n_words': 563}