<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals, absolute_import

import unicodedata as ud

from anitopy.element import ElementCategory, Elements


class KeywordOption:
    def __init__(self, identifiable=True, searchable=True, valid=True):
        self.identifiable = identifiable
        self.searchable = searchable
        self.valid = valid


class Keyword:
    def __init__(self, category, options):
        self.category = category
        self.options = options


class KeywordManager:
    def __init__(self):
        options_default = KeywordOption()
        options_invalid = KeywordOption(valid=False)
        options_unidentifiable = KeywordOption(identifiable=False)
        options_unidentifiable_invalid = KeywordOption(identifiable=False,
                                                       valid=False)
        options_unidentifiable_unsearchable = KeywordOption(identifiable=False,
                                                            searchable=False)

        self._file_extensions = {}
        self._keys = {}

        self.add(ElementCategory.ANIME_SEASON_PREFIX, options_unidentifiable, [
            'SAISON', 'SEASON'])

        self.add(ElementCategory.ANIME_TYPE, options_unidentifiable, [
            'GEKIJOUBAN', 'MOVIE',
            'OAD', 'OAV', 'ONA', 'OVA',
            'SPECIAL', 'SPECIALS',
            'TV'])
        self.add(ElementCategory.ANIME_TYPE,
                 options_unidentifiable_unsearchable,
                 ['SP'])  # e.g. "<NAME> SP Professional"
        self.add(ElementCategory.ANIME_TYPE, options_unidentifiable_invalid, [
            'ED', 'ENDING', 'NCED',
            'NCOP', 'OP', 'OPENING',
            'PREVIEW', 'PV'])

        self.add(ElementCategory.AUDIO_TERM, options_default, [
            # Audio channels
            '2.0CH', '2CH', '5.1', '5.1CH', 'DTS', 'DTS-ES', 'DTS5.1',
            'TRUEHD5.1',
            # Audio codec
            'AAC', 'AACX2', 'AACX3', 'AACX4', 'AC3', 'EAC3', 'E-AC-3',
            'FLAC', 'FLACX2', 'FLACX3', 'FLACX4', 'LOSSLESS', 'MP3', 'OGG',
            'VORBIS',
            # Audio language
            'DUALAUDIO', 'DUAL AUDIO'])

        self.add(ElementCategory.DEVICE_COMPATIBILITY, options_default, [
            'IPAD3', 'IPHONE5', 'IPOD', 'PS3', 'XBOX', 'XBOX360'])
        self.add(ElementCategory.DEVICE_COMPATIBILITY, options_unidentifiable,
                 ['ANDROID'])

        self.add(ElementCategory.EPISODE_PREFIX, options_default, [
            'EP', 'EP.', 'EPS', 'EPS.', 'EPISODE', 'EPISODE.', 'EPISODES',
            'CAPITULO', 'EPISODIO', 'FOLGE'])
        self.add(ElementCategory.EPISODE_PREFIX, options_invalid, [
            'E', '\x7B2C'])  # single-letter episode keywords are not valid

        self.add(ElementCategory.FILE_EXTENSION, options_default, [
            '3GP', 'AVI', 'DIVX', 'FLV', 'M2TS', 'MKV', 'MOV', 'MP4', 'MPG',
            'OGM', 'RM', 'RMVB', 'WEBM', 'WMV'])
        self.add(ElementCategory.FILE_EXTENSION, options_invalid, [
            'AAC', 'AIFF', 'FLAC', 'M4A', 'MP3', 'MKA', 'OGG', 'WAV', 'WMA',
            '7Z', 'RAR', 'ZIP',
            'ASS', 'SRT'])

        self.add(ElementCategory.LANGUAGE, options_default, [
            'ENG', 'ENGLISH', 'ESPANOL', 'JAP', 'PT-BR', 'SPANISH', 'VOSTFR'])
        self.add(ElementCategory.LANGUAGE, options_unidentifiable, [
            'ESP', 'ITA'])  # e.g. "Tokyo ESP", "Bokura ga Ita"

        self.add(ElementCategory.OTHER, options_default, [
            'REMASTER', 'REMASTERED', 'UNCENSORED', 'UNCUT',<commit_msg>Feat: add keyword for TS files<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals, absolute_import

import unicodedata as ud

from anitopy.element import ElementCategory, Elements


class KeywordOption:
    def __init__(self, identifiable=True, searchable=True, valid=True):
        self.identifiable = identifiable
        self.searchable = searchable
        self.valid = valid


class Keyword:
    def __init__(self, category, options):
        self.category = category
        self.options = options


class KeywordManager:
    def __init__(self):
        options_default = KeywordOption()
        options_invalid = KeywordOption(valid=False)
        options_unidentifiable = KeywordOption(identifiable=False)
        options_unidentifiable_invalid = KeywordOption(identifiable=False,
                                                       valid=False)
        options_unidentifiable_unsearchable = KeywordOption(identifiable=False,
                                                            searchable=False)

        self._file_extensions = {}
        self._keys = {}

        self.add(ElementCategory.ANIME_SEASON_PREFIX, options_unidentifiable, [
            'SAISON', 'SEASON'])

        self.add(ElementCategory.ANIME_TYPE, options_unidentifiable, [
            'GEKIJOUBAN', 'MOVIE',
            'OAD', 'OAV', 'ONA', 'OVA',
            'SPECIAL', 'SPECIALS',
            'TV'])
        self.add(ElementCategory.ANIME_TYPE,
                 options_unidentifiable_unsearchable,
                 ['SP'])  # e.g. "<NAME> SP Professional"
        self.add(ElementCategory.ANIME_TYPE, options_unidentifiable_invalid, [
            'ED', 'ENDING', 'NCED',
            'NCOP', 'OP', 'OPENING',
            'PREVIEW', 'PV'])

        self.add(ElementCategory.AUDIO_TERM, options_default, [
            # Audio channels
            '2.0CH', '2CH', '5.1', '5.1CH', 'DTS', 'DTS-ES', 'DTS5.1',
            'TRUEHD5.1',
            # Audio codec
            'AAC', 'AACX2', 'AACX3', 'AACX4', 'AC3', 'EAC3', 'E-AC-3',
            'FLAC', 'FLACX2', 'FLACX3', 'FLACX4', 'LOSSLESS', 'MP3', 'OGG',
            'VORBIS',
            # Audio language
            'DUALAUDIO', 'DUAL AUDIO'])

        self.add(ElementCategory.DEVICE_COMPATIBILITY, options_default, [
            'IPAD3', 'IPHONE5', 'IPOD', 'PS3', 'XBOX', 'XBOX360'])
        self.add(ElementCategory.DEVICE_COMPATIBILITY, options_unidentifiable,
                 ['ANDROID'])

        self.add(ElementCategory.EPISODE_PREFIX, options_default, [
            'EP', 'EP.', 'EPS', 'EPS.', 'EPISODE', 'EPISODE.', 'EPISODES',
            'CAPITULO', 'EPISODIO', 'FOLGE'])
        self.add(ElementCategory.EPISODE_PREFIX, options_invalid, [
            'E', '\x7B2C'])  # single-letter episode keywords are not valid

        self.add(ElementCategory.FILE_EXTENSION, options_default, [
            '3GP', 'AVI', 'DIVX', 'FLV', 'M2TS', 'MKV', 'MOV', 'MP4', 'MPG',
            'OGM', 'RM', 'RMVB', 'TS', 'WEBM', 'WMV'])
        self.add(ElementCategory.FILE_EXTENSION, options_invalid, [
            'AAC', 'AIFF', 'FLAC', 'M4A', 'MP3', 'MKA', 'OGG', 'WAV', 'WMA',
            '7Z', 'RAR', 'ZIP',
            'ASS', 'SRT'])

        self.add(ElementCategory.LANGUAGE, options_default, [
            'ENG', 'ENGLISH', 'ESPANOL', 'JAP', 'PT-BR', 'SPANISH', 'VOSTFR'])
        self.add(ElementCategory.LANGUAGE, options_unidentifiable, [
            'ESP', 'ITA'])  # e.g. "Tokyo ESP", "Bokura ga Ita"

        self.add(ElementCategory.OTHER, options_default, [
            'REMASTER', 'REMASTERED', 'UNCENSORED', 'UNCUT',
{'fraction_non_alphanumeric': 0.14594670015348124, 'fraction_numerical': 0.010046044370029301, 'mean_word_length': 2.366838891498356, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '463815', 'n_tokens_mistral': 2364, 'n_tokens_neox': 2288, 'n_words': 433}