<commit_before>import copy
import functools
import re
from typing import Dict, Iterator, List, Optional, Set, Tuple, Union

from loguru import logger

from flexget.utils.serialization import Serializer

logger = logger.bind(name='utils.qualities')


@functools.total_ordering
class QualityComponent:
    """"""

    def __init__(
        self,
        type: str,
        value: int,
        name: str,
        regexp: Optional[str] = None,
        modifier: Optional[int] = None,
        defaults: Optional[List['QualityComponent']] = None,
    ) -> None:
        """
        :param type: Type of quality component. (resolution, source, codec, color_range or audio)
        :param value: Value used to sort this component with others of like type.
        :param name: Canonical name for this quality component.
        :param regexp: Regexps used to match this component.
        :param modifier: An integer that affects sorting above all other components.
        :param defaults: An iterable defining defaults for other quality components if this component matches.
        """

        if type not in ['resolution', 'source', 'codec', 'color_range', 'audio']:
            raise ValueError('%s is not a valid quality component type.' % type)
        self.type = type
        self.value = value
        self.name = name
        self.modifier = modifier
        self.defaults = defaults or []

        # compile regexp
        if regexp is None:
            regexp = re.escape(name)
        self.regexp = re.compile(r'(?<![^\W_])(' + regexp + r')(?![^\W_])', re.IGNORECASE)

    def matches(self, text: str) -> Tuple[bool, str]:
        """Test if quality matches to text.

        :param string text: data te be tested against
        :returns: tuple (matches, remaining text without quality data)
        """

        match = self.regexp.search(text)
        if not match:
            return False, ""
        else:
            # remove matching part from the text
            text = text[: match.start()] + text[match.end() :]
        return True, text

    def __hash__(self) -> int:
        return hash(self.type + str(self.value))

    def __bool__(self) -> bool:
        return bool(self.value)

    def __eq__(self, other) -> bool:
        if isinstance(other, str):
            other = _registry.get(other)
        if not isinstance(other, QualityComponent):
            raise TypeError('Cannot compare %r and %r' % (self, other))
        if other.type == self.type:
            return self.value == other.value
        else:
            raise TypeError('Cannot compare %s and %s' % (self.type, other.type))

    def __lt__(self, other) -> bool:
        if isinstance(other, str):
            other = _registry.get(other)
        if not isinstance(other, QualityComponent):
            raise TypeError('Cannot compare %r and %r' % (self, other))
        if other.type == self.type:
            return self.value < other.value
        else:
            raise TypeError('Cannot compare %s and %s' % (self.type, other.type))

    def __add__(self, other):
        if not isinstance(other, int):
            raise TypeError()
        l = globals().get('_' + self.type + 's')
        index = l.index(self) + other
        if index >= len(l):
            index = -1
        return l[index]

    def __sub__(self, other):
        if not isinstance(other, int):
            raise TypeError()
        l = globals().get('_' + self.type + 's')
        index = l.index(self) - other
        if index < 0:
            index = 0
        return l[index]

    def __repr__(self) -> str:
        return f'<{self.type.title()}(name={self.name},value={self.value})>'

    def __str__(self) -> str:
        return self.name

    def __deepcopy__(self, memo=None):
        # No mutable attributes, return a regular copy
        return copy.copy(self)


_resolutions = [
    QualityComponent('resolution', 10, '360p'),
    QualityComponent('resolution', 20, '368p', '368p?'),
    QualityComponent('resolution', 30, '480p', '480p?'),
    QualityComponent('resolution', 35, '540p', '540p?'),
    QualityComponent('resolution', 40, '576p', '576p?'),
    QualityComponent('resolution', 45, 'hr'),
    QualityComponent('resolution', 50, '720i'),
    QualityComponent('resolution', 60, '720p', '(1280x)?720(p|hd)?x?([56]0)?'),
    QualityComponent('resolution', 70, '1080i'),
    QualityComponent('resolution', 80, '1080p', '(1920x)?1080p?x?([56]0)?'),
    QualityComponent('resolution', 90, '2160p', '((3840x)?2160p?x?([56]0)?)|4k'),
]
_sources = [
    QualityComponent('source', 10, 'workprint', modifier=-8),
    QualityComponent('source', 20, 'cam', '(?:hd)?cam', modifier=-7),
    QualityComponent('source', 30, 'ts', '(?:hd)?ts|telesync', modifier=-6),
    QualityComponent('source', 40, 'tc', 'tc|telecine', modifier=-5),
    QualityComponent('source', 50, 'r5', 'r[2-8c]', modifier=-4),
    QualityComponent('source', 60, 'hdrip', r'hd[\W_]?rip', modifier=-3),
    QualityComponent('source', 70, 'ppvrip', r'ppv[\W_]?rip', modifier=-2),
    QualityComponent('source', 80, 'preair', modifier=-1),
    QualityComponent('source', 90, 'tvrip', r'tv[\W_]?rip'),
    QualityComponent('source', 100, 'dsr', r'dsr|ds[\W_]?rip'),
    QualityComponent('source', 110, 'sdtv', r'(?:[sp]dtv|dvb)(?:[\W_]?rip)?'),
    QualityComponent('source', 120, 'dvdscr', r'(?:(?:dvd|web)[\W_]?)?scr(?:eener)?', modifier=0),
    QualityComponent('source', 130, 'bdscr', 'bdscr(?:eener)?'),
    QualityComponent('source', 140, 'webrip', r'web[\W_]?rip'),
    QualityComponent('source', 150, 'hdtv', r'a?hdtv(?:[\W_]?rip)?'),
    QualityComponent('source', 160, 'webdl', r'web(?:[\W_]?(dl|hd))?'),
    QualityComponent('source', 170, 'dvdrip', r'dvd(?:[\W_]?rip)?'),
    QualityComponent('source', 175, 'remux'),
    QualityComponent('source', 180, 'bluray', r'(?:b[dr][\W_]?rip|blu[\W_]?ray(?:[\W_]?rip)?)'),
]
_codecs = [
    QualityComponent('codec', 10, 'divx'),
    QualityComponent('codec', 20, 'xvid'),
    QualityComponent('codec', 30, 'h264', '[hx].?264'),
    QualityComponent('codec', 35, 'vp9'),
    QualityComponent('codec', 40, 'h265', '[hx].?265|hevc'),
]

_color_ranges = [
    QualityComponent('color_range', 10, '8bit', r'8[^\w]?bits?|hi8p?'),
    QualityComponent('color_range', 20, '10bit', r'10[^\w]?bits?|hi10p?'),
    QualityComponent('color_range', 40, 'hdrplus', r'hdr(10)?[^\w]?(\+|p|plus)'),
    QualityComponent('color_range', 30, 'hdr', r'hdr([^\w]?10)?'),
    QualityComponent('color_range', 50, 'dolbyvision', r'(dolby[^\w]?vision|dv|dovi)'),
]

channels = r'(?:(?:[^\w+]?[1-7][\W_]?(?:0|1|ch)))'
_audios = [
    QualityComponent('audio', 10, 'mp3'),
    # TODO: No idea what order these should go in or if we need different regexps
    QualityComponent('audio', 20, 'aac', 'aac%s?' % channels),
    QualityComponent('audio', 30, 'dd5.1', 'dd%s' % channels),
    QualityComponent('audio', 40, 'ac3', 'ac3%s?' % channels),
    QualityComponent('audio', 45, 'dd+5.1', 'dd[p+]%s' % channels),
    QualityComponent('audio', 50, 'flac', 'flac%s?' % channels),
    # The DTSs are a bit backwards, but the more specific one needs to be parsed first
    QualityComponent('audio', 70, 'dtshd', r'dts[\W_]?hd(?:[\W_]?ma)?%s?' % channels),
    QualityComponent('audio', 60, 'dts'),
    QualityComponent('audio', 80, 'truehd', 'truehd%s?' % channels),
]

_UNKNOWNS = {
    'resolution': QualityComponent('resolution', 0, 'unknown'),
    'source': QualityComponent('source', 0, 'unknown'),
    'codec': QualityComponent('codec', 0, 'unknown'),
    'color_range': QualityComponent('color_range', 0, 'unknown'),
    'audio': QualityComponent('audio', 0, 'unknown'),
}

# For wiki generating help
'''for type in (_resolutions, _sources, _codecs, _color_ranges, _audios):
    print '{{{#!td style="vertical-align: top"'
    for item in reversed(type):
        print '- ' + item.name
    print '}}}'
'''

_registry: Dict[str, QualityComponent] = {}
for items in (_resolutions, _sources, _codecs, _color_ranges, _audios):
    for item in items:
        _registry[item.name] = item


def all_components() -> Iterator[QualityComponent]:
    return iter(_registry.values())


@functools.total_ordering<commit_msg>Remove (really) old qualities wiki generation code.<commit_after>import copy
import functools
import re
from typing import Dict, Iterator, List, Optional, Set, Tuple, Union

from loguru import logger

from flexget.utils.serialization import Serializer

logger = logger.bind(name='utils.qualities')


@functools.total_ordering
class QualityComponent:
    """"""

    def __init__(
        self,
        type: str,
        value: int,
        name: str,
        regexp: Optional[str] = None,
        modifier: Optional[int] = None,
        defaults: Optional[List['QualityComponent']] = None,
    ) -> None:
        """
        :param type: Type of quality component. (resolution, source, codec, color_range or audio)
        :param value: Value used to sort this component with others of like type.
        :param name: Canonical name for this quality component.
        :param regexp: Regexps used to match this component.
        :param modifier: An integer that affects sorting above all other components.
        :param defaults: An iterable defining defaults for other quality components if this component matches.
        """

        if type not in ['resolution', 'source', 'codec', 'color_range', 'audio']:
            raise ValueError('%s is not a valid quality component type.' % type)
        self.type = type
        self.value = value
        self.name = name
        self.modifier = modifier
        self.defaults = defaults or []

        # compile regexp
        if regexp is None:
            regexp = re.escape(name)
        self.regexp = re.compile(r'(?<![^\W_])(' + regexp + r')(?![^\W_])', re.IGNORECASE)

    def matches(self, text: str) -> Tuple[bool, str]:
        """Test if quality matches to text.

        :param string text: data te be tested against
        :returns: tuple (matches, remaining text without quality data)
        """

        match = self.regexp.search(text)
        if not match:
            return False, ""
        else:
            # remove matching part from the text
            text = text[: match.start()] + text[match.end() :]
        return True, text

    def __hash__(self) -> int:
        return hash(self.type + str(self.value))

    def __bool__(self) -> bool:
        return bool(self.value)

    def __eq__(self, other) -> bool:
        if isinstance(other, str):
            other = _registry.get(other)
        if not isinstance(other, QualityComponent):
            raise TypeError('Cannot compare %r and %r' % (self, other))
        if other.type == self.type:
            return self.value == other.value
        else:
            raise TypeError('Cannot compare %s and %s' % (self.type, other.type))

    def __lt__(self, other) -> bool:
        if isinstance(other, str):
            other = _registry.get(other)
        if not isinstance(other, QualityComponent):
            raise TypeError('Cannot compare %r and %r' % (self, other))
        if other.type == self.type:
            return self.value < other.value
        else:
            raise TypeError('Cannot compare %s and %s' % (self.type, other.type))

    def __add__(self, other):
        if not isinstance(other, int):
            raise TypeError()
        l = globals().get('_' + self.type + 's')
        index = l.index(self) + other
        if index >= len(l):
            index = -1
        return l[index]

    def __sub__(self, other):
        if not isinstance(other, int):
            raise TypeError()
        l = globals().get('_' + self.type + 's')
        index = l.index(self) - other
        if index < 0:
            index = 0
        return l[index]

    def __repr__(self) -> str:
        return f'<{self.type.title()}(name={self.name},value={self.value})>'

    def __str__(self) -> str:
        return self.name

    def __deepcopy__(self, memo=None):
        # No mutable attributes, return a regular copy
        return copy.copy(self)


_resolutions = [
    QualityComponent('resolution', 10, '360p'),
    QualityComponent('resolution', 20, '368p', '368p?'),
    QualityComponent('resolution', 30, '480p', '480p?'),
    QualityComponent('resolution', 35, '540p', '540p?'),
    QualityComponent('resolution', 40, '576p', '576p?'),
    QualityComponent('resolution', 45, 'hr'),
    QualityComponent('resolution', 50, '720i'),
    QualityComponent('resolution', 60, '720p', '(1280x)?720(p|hd)?x?([56]0)?'),
    QualityComponent('resolution', 70, '1080i'),
    QualityComponent('resolution', 80, '1080p', '(1920x)?1080p?x?([56]0)?'),
    QualityComponent('resolution', 90, '2160p', '((3840x)?2160p?x?([56]0)?)|4k'),
]
_sources = [
    QualityComponent('source', 10, 'workprint', modifier=-8),
    QualityComponent('source', 20, 'cam', '(?:hd)?cam', modifier=-7),
    QualityComponent('source', 30, 'ts', '(?:hd)?ts|telesync', modifier=-6),
    QualityComponent('source', 40, 'tc', 'tc|telecine', modifier=-5),
    QualityComponent('source', 50, 'r5', 'r[2-8c]', modifier=-4),
    QualityComponent('source', 60, 'hdrip', r'hd[\W_]?rip', modifier=-3),
    QualityComponent('source', 70, 'ppvrip', r'ppv[\W_]?rip', modifier=-2),
    QualityComponent('source', 80, 'preair', modifier=-1),
    QualityComponent('source', 90, 'tvrip', r'tv[\W_]?rip'),
    QualityComponent('source', 100, 'dsr', r'dsr|ds[\W_]?rip'),
    QualityComponent('source', 110, 'sdtv', r'(?:[sp]dtv|dvb)(?:[\W_]?rip)?'),
    QualityComponent('source', 120, 'dvdscr', r'(?:(?:dvd|web)[\W_]?)?scr(?:eener)?', modifier=0),
    QualityComponent('source', 130, 'bdscr', 'bdscr(?:eener)?'),
    QualityComponent('source', 140, 'webrip', r'web[\W_]?rip'),
    QualityComponent('source', 150, 'hdtv', r'a?hdtv(?:[\W_]?rip)?'),
    QualityComponent('source', 160, 'webdl', r'web(?:[\W_]?(dl|hd))?'),
    QualityComponent('source', 170, 'dvdrip', r'dvd(?:[\W_]?rip)?'),
    QualityComponent('source', 175, 'remux'),
    QualityComponent('source', 180, 'bluray', r'(?:b[dr][\W_]?rip|blu[\W_]?ray(?:[\W_]?rip)?)'),
]
_codecs = [
    QualityComponent('codec', 10, 'divx'),
    QualityComponent('codec', 20, 'xvid'),
    QualityComponent('codec', 30, 'h264', '[hx].?264'),
    QualityComponent('codec', 35, 'vp9'),
    QualityComponent('codec', 40, 'h265', '[hx].?265|hevc'),
]

_color_ranges = [
    QualityComponent('color_range', 10, '8bit', r'8[^\w]?bits?|hi8p?'),
    QualityComponent('color_range', 20, '10bit', r'10[^\w]?bits?|hi10p?'),
    QualityComponent('color_range', 40, 'hdrplus', r'hdr(10)?[^\w]?(\+|p|plus)'),
    QualityComponent('color_range', 30, 'hdr', r'hdr([^\w]?10)?'),
    QualityComponent('color_range', 50, 'dolbyvision', r'(dolby[^\w]?vision|dv|dovi)'),
]

channels = r'(?:(?:[^\w+]?[1-7][\W_]?(?:0|1|ch)))'
_audios = [
    QualityComponent('audio', 10, 'mp3'),
    # TODO: No idea what order these should go in or if we need different regexps
    QualityComponent('audio', 20, 'aac', 'aac%s?' % channels),
    QualityComponent('audio', 30, 'dd5.1', 'dd%s' % channels),
    QualityComponent('audio', 40, 'ac3', 'ac3%s?' % channels),
    QualityComponent('audio', 45, 'dd+5.1', 'dd[p+]%s' % channels),
    QualityComponent('audio', 50, 'flac', 'flac%s?' % channels),
    # The DTSs are a bit backwards, but the more specific one needs to be parsed first
    QualityComponent('audio', 70, 'dtshd', r'dts[\W_]?hd(?:[\W_]?ma)?%s?' % channels),
    QualityComponent('audio', 60, 'dts'),
    QualityComponent('audio', 80, 'truehd', 'truehd%s?' % channels),
]

_UNKNOWNS = {
    'resolution': QualityComponent('resolution', 0, 'unknown'),
    'source': QualityComponent('source', 0, 'unknown'),
    'codec': QualityComponent('codec', 0, 'unknown'),
    'color_range': QualityComponent('color_range', 0, 'unknown'),
    'audio': QualityComponent('audio', 0, 'unknown'),
}

_registry: Dict[str, QualityComponent] = {}
for items in (_resolutions, _sources, _codecs, _color_ranges, _audios):
    for item in items:
        _registry[item.name] = item


def all_components() -> Iterator[QualityComponent]:
    return iter(_registry.values())


@functools.total_ordering
{'fraction_non_alphanumeric': 0.17955852346503431, 'fraction_numerical': 0.03005008347245409, 'mean_word_length': 3.386764307024681, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2820302', 'n_tokens_mistral': 5598, 'n_tokens_neox': 5142, 'n_words': 1306}