<commit_before>from dataclasses import dataclass
import re
import time
from typing import Union, List, Dict, Optional

from furl import furl

from mediawords.db import DatabaseHandler
from mediawords.util.log import create_logger
from mediawords.util.perl import decode_object_from_bytes_if_needed
from mediawords.util.url import is_http_url, fix_common_url_mistakes, canonical_url
from mediawords.util.web.user_agent import UserAgent

from facebook_fetch_story_stats.config import FacebookConfig
from facebook_fetch_story_stats.exceptions import (
    McFacebookException,
    McFacebookInvalidParametersException,
    McFacebookInvalidConfigurationException,
    McFacebookInvalidURLException,
    McFacebookUnexpectedAPIResponseException,
    McFacebookErrorAPIResponseException,
    McFacebookSoftFailureException,
)

log = create_logger(__name__)

__FACEBOOK_API_HTTP_TIMEOUT = 60
"""Facebook API HTTP timeout."""

__FACEBOOK_GRAPH_API_RETRY_COUNT = 22
"""Number of retries to do on retryable Facebook Graph API errors (such as rate limiting issues or API downtime)."""

__FACEBOOK_GRAPH_API_RETRYABLE_ERROR_CODES = {

    # API Unknown -- Possibly a temporary issue due to downtime. Wait and retry the operation. If it occurs again, check
    # that you are requesting an existing API.

    # API Service -- Temporary issue due to downtime - retry the operation after waiting.
    2,

    # API Too Many Calls -- Temporary issue due to throttling. Wait and retry the operation, or examine your API request
    # volume.
    4,

    # API User Too Many Calls -- Temporary issue due to throttling. Wait and retry the operation, or examine your API
    # request volume.
    17,

    # Application limit reached -- Temporary issue due to downtime or throttling. Wait and retry the operation, or
    # examine your API request volume.
    341,

}
"""
Facebook Graph API's error codes of retryable (potentially temporary) errors on which we should retry.

https://developers.facebook.com/docs/graph-api/using-graph-api/error-handling#errorcodes
"""

__URL_PATTERNS_WHICH_WONT_WORK = [

    # Google Search
    re.compile(r'^https?://.*?\.google\..{2,7}/(search|webhp).+?', flags=re.IGNORECASE),

    # Google Trends
    re.compile(r'^https?://.*?\.google\..{2,7}/trends/explore.*?', flags=re.IGNORECASE),

    # Facebook posts/stories
    # For reasons unknown, one can't fetch stats for FB posts/stories via the FB graph API, even though they're
    # viewable in a web browser. A previous attempt to get them via a third-party scraper package failed due to
    # cookie requirements—see this commit for the attempted implementation:
    # https://github.com/mediacloud/backend/pull/811/commits/74926ed7e353d8c0a20fefbae3f30a59bf9de77c
    re.compile(r'^https?://.*?\.facebook\..*/posts/.*?|^https?://.*?\.facebook\..*/story.*?', flags=re.IGNORECASE)
]
"""URL patterns for which we're sure we won't get correct results (so we won't even try)."""


def _api_request(node: str, params: Dict[str, Union[str, List[str]]], config: FacebookConfig) -> Union[dict, list]:
    """
    Make Facebook API request.

    Return successful or failed API response if we were able to make a request. Throw McFacebookException subclass if
    something went wrong.

    :param node: Facebook API node to call.
    :param params: Dictionary of parameters to pass to the API; values might be either strings of lists of strings if
                   multiple values with the same key have to be passed.
    :param config: Facebook configuration object.
    :return: API response.
    """
    node = decode_object_from_bytes_if_needed(node)
    params = decode_object_from_bytes_if_needed(params)

    if node is None:
        raise McFacebookInvalidParametersException("Node is undefined (node might be an empty string).")
<commit_msg>break regex into two lines<commit_after>from dataclasses import dataclass
import re
import time
from typing import Union, List, Dict, Optional

from furl import furl

from mediawords.db import DatabaseHandler
from mediawords.util.log import create_logger
from mediawords.util.perl import decode_object_from_bytes_if_needed
from mediawords.util.url import is_http_url, fix_common_url_mistakes, canonical_url
from mediawords.util.web.user_agent import UserAgent

from facebook_fetch_story_stats.config import FacebookConfig
from facebook_fetch_story_stats.exceptions import (
    McFacebookException,
    McFacebookInvalidParametersException,
    McFacebookInvalidConfigurationException,
    McFacebookInvalidURLException,
    McFacebookUnexpectedAPIResponseException,
    McFacebookErrorAPIResponseException,
    McFacebookSoftFailureException,
)

log = create_logger(__name__)

__FACEBOOK_API_HTTP_TIMEOUT = 60
"""Facebook API HTTP timeout."""

__FACEBOOK_GRAPH_API_RETRY_COUNT = 22
"""Number of retries to do on retryable Facebook Graph API errors (such as rate limiting issues or API downtime)."""

__FACEBOOK_GRAPH_API_RETRYABLE_ERROR_CODES = {

    # API Unknown -- Possibly a temporary issue due to downtime. Wait and retry the operation. If it occurs again, check
    # that you are requesting an existing API.

    # API Service -- Temporary issue due to downtime - retry the operation after waiting.
    2,

    # API Too Many Calls -- Temporary issue due to throttling. Wait and retry the operation, or examine your API request
    # volume.
    4,

    # API User Too Many Calls -- Temporary issue due to throttling. Wait and retry the operation, or examine your API
    # request volume.
    17,

    # Application limit reached -- Temporary issue due to downtime or throttling. Wait and retry the operation, or
    # examine your API request volume.
    341,

}
"""
Facebook Graph API's error codes of retryable (potentially temporary) errors on which we should retry.

https://developers.facebook.com/docs/graph-api/using-graph-api/error-handling#errorcodes
"""

__URL_PATTERNS_WHICH_WONT_WORK = [

    # Google Search
    re.compile(r'^https?://.*?\.google\..{2,7}/(search|webhp).+?', flags=re.IGNORECASE),

    # Google Trends
    re.compile(r'^https?://.*?\.google\..{2,7}/trends/explore.*?', flags=re.IGNORECASE),

    # Facebook posts/stories
    # For reasons unknown, one can't fetch stats for FB posts/stories via the FB graph API, even though they're
    # viewable in a web browser. A previous attempt to get them via a third-party scraper package failed due to
    # cookie requirements—see this commit for the attempted implementation:
    # https://github.com/mediacloud/backend/pull/811/commits/74926ed7e353d8c0a20fefbae3f30a59bf9de77c
    re.compile(r'^https?://.*?\.facebook\..*/posts/.*?', flags=re.IGNORECASE),
    re.compile(r'^https?://.*?\.facebook\..*/story.*?', flags=re.IGNORECASE)
]
"""URL patterns for which we're sure we won't get correct results (so we won't even try)."""


def _api_request(node: str, params: Dict[str, Union[str, List[str]]], config: FacebookConfig) -> Union[dict, list]:
    """
    Make Facebook API request.

    Return successful or failed API response if we were able to make a request. Throw McFacebookException subclass if
    something went wrong.

    :param node: Facebook API node to call.
    :param params: Dictionary of parameters to pass to the API; values might be either strings of lists of strings if
                   multiple values with the same key have to be passed.
    :param config: Facebook configuration object.
    :return: API response.
    """
    node = decode_object_from_bytes_if_needed(node)
    params = decode_object_from_bytes_if_needed(params)

    if node is None:
        raise McFacebookInvalidParametersException("Node is undefined (node might be an empty string).")

{'fraction_non_alphanumeric': 0.08480749219562955, 'fraction_numerical': 0.010145681581685745, 'mean_word_length': 4.041967213114754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2408235', 'n_tokens_mistral': 2309, 'n_tokens_neox': 2157, 'n_words': 764}