<commit_before>"""Creating and finding media within topics."""

import re
import time
import typing

from mediawords.db import DatabaseHandler
from mediawords.db.locks import get_session_lock
from mediawords.util.log import create_logger
import mediawords.util.url

log = create_logger(__name__)

# url and name length limits necessary to fit within postgres field
MAX_URL_LENGTH = 1024
MAX_NAME_LENGTH = 124

# try appending this to urls to generate a unique url for get_unique_medium_url()
URL_SPIDERED_SUFFIX = '#spider'

# names for spidered tag and tag set
SPIDERED_TAG_TAG = 'spidered'
SPIDERED_TAG_SET = 'spidered'

# retry query for new unique name to avoid race condition
_GUESS_MEDIUM_RETRIES = 5


class McTopicMediaException(Exception):
    """Exception arising from this package."""

    pass


class McTopicMediaUniqueException(McTopicMediaException):
    """Exception raised when guess_medium is unable to find a unique name or url for a new media source."""

    pass


def _normalize_url(url: str) -> str:
    """Cap max length of url and run through mediawords.util.url.normalize_url_lossy."""
    nu = mediawords.util.url.normalize_url_lossy(url)
    if nu is None:
        nu = url

    return nu[0:MAX_URL_LENGTH]


def generate_medium_url_and_name_from_url(story_url: str) -> tuple:
    """Derive the url and a media source name from a story url.

    This function just returns the pathless normalized url as the medium_url and the host nane as the medium name.

    Arguments:
    url - story url

    Returns:
    tuple in the form (medium_url, medium_name)

    """
    normalized_url = _normalize_url(story_url)

    matches = re.search(r'(http.?://([^/]+))', normalized_url, flags=re.I)
    if matches is None:
        log.warning("Unable to find host name in url: normalized_url (%s)" % story_url)
        return (story_url, story_url)

    (medium_url, medium_name) = (matches.group(1).lower(), matches.group(2).lower())

    if not medium_url.endswith('/'):
        medium_url += "/"

    return (medium_url, medium_name)


def _normalized_urls_out_of_date(db: DatabaseHandler) -> bool:
    """Return True iff there is at least one medium with a null normalized_url."""

    null_medium = db.query("select * from media where normalized_url is null limit 1").hash()

    return null_medium is not None


def _update_media_normalized_urls(db: DatabaseHandler) -> None:
    """Keep normalized_url field in media table up to date.

    Set the normalized_url field of any row in media for which it is null.  Take care to lock the process
    so that only one process is doing this work at a time.
    """
    if not _normalized_urls_out_of_date(db):
        return

    # put a lock on this because the process of generating all media urls will take a couple hours, and we don't
    # want all workers to do the work
    locked = False
    while not locked:
        db.begin()
        # poll instead of block so that we can releae the transaction and see whether someone else has already
        # updated all of the media
        locked = get_session_lock(db, 'MediaWords::TM::Media::media_normalized_urls', 1, wait=False)
        if not locked:
            db.commit()
            log.info("sleeping for media_normalized_urls lock...")
            time.sleep(1)

    if not _normalized_urls_out_of_date(db):
        db.commit()
        return

    log.warning("updating media_normalized_urls ...")

    media = db.query("select * from media where normalized_url is null").hashes()<commit_msg>optimize locking for mediawords.tm.media.lookup_medium<commit_after>"""Creating and finding media within topics."""

import re
import time
import typing

from mediawords.db import DatabaseHandler
from mediawords.db.locks import get_session_lock
from mediawords.util.log import create_logger
import mediawords.util.url

log = create_logger(__name__)

# url and name length limits necessary to fit within postgres field
MAX_URL_LENGTH = 1024
MAX_NAME_LENGTH = 124

# try appending this to urls to generate a unique url for get_unique_medium_url()
URL_SPIDERED_SUFFIX = '#spider'

# names for spidered tag and tag set
SPIDERED_TAG_TAG = 'spidered'
SPIDERED_TAG_SET = 'spidered'

# retry query for new unique name to avoid race condition
_GUESS_MEDIUM_RETRIES = 5


class McTopicMediaException(Exception):
    """Exception arising from this package."""

    pass


class McTopicMediaUniqueException(McTopicMediaException):
    """Exception raised when guess_medium is unable to find a unique name or url for a new media source."""

    pass


def _normalize_url(url: str) -> str:
    """Cap max length of url and run through mediawords.util.url.normalize_url_lossy."""
    nu = mediawords.util.url.normalize_url_lossy(url)
    if nu is None:
        nu = url

    return nu[0:MAX_URL_LENGTH]


def generate_medium_url_and_name_from_url(story_url: str) -> tuple:
    """Derive the url and a media source name from a story url.

    This function just returns the pathless normalized url as the medium_url and the host nane as the medium name.

    Arguments:
    url - story url

    Returns:
    tuple in the form (medium_url, medium_name)

    """
    normalized_url = _normalize_url(story_url)

    matches = re.search(r'(http.?://([^/]+))', normalized_url, flags=re.I)
    if matches is None:
        log.warning("Unable to find host name in url: normalized_url (%s)" % story_url)
        return (story_url, story_url)

    (medium_url, medium_name) = (matches.group(1).lower(), matches.group(2).lower())

    if not medium_url.endswith('/'):
        medium_url += "/"

    return (medium_url, medium_name)


def _normalized_urls_out_of_date(db: DatabaseHandler) -> bool:
    """Return True iff there is at least one medium with a null normalized_url."""

    null_medium = db.query("select * from media where normalized_url is null limit 1").hash()

    return null_medium is not None


def _update_media_normalized_urls(db: DatabaseHandler) -> None:
    """Keep normalized_url field in media table up to date.

    Set the normalized_url field of any row in media for which it is null.  Take care to lock the process
    so that only one process is doing this work at a time.
    """
    # put a lock on this because the process of generating all media urls will take a couple hours, and we don't
    # want all workers to do the work
    locked = False
    while not locked:
        if not _normalized_urls_out_of_date(db):
            return

        db.begin()

        # poll instead of block so that we can releae the transaction and see whether someone else has already
        # updated all of the media
        locked = get_session_lock(db, 'MediaWords::TM::Media::media_normalized_urls', 1, wait=False)

        if not locked:
            db.commit()
            log.info("sleeping for media_normalized_urls lock...")
            time.sleep(1)

    log.warning("updating media_normalized_urls ...")

    media = db.query("select * from media where normalized_url is null").hashes()
{'fraction_non_alphanumeric': 0.08143183114660582, 'fraction_numerical': 0.003993154592127781, 'mean_word_length': 3.484015345268542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3608239', 'n_tokens_mistral': 2115, 'n_tokens_neox': 2033, 'n_words': 755}