<commit_before># -*- coding: utf-8 -*-
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 <NAME> <<EMAIL>>
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import threading
from itertools import groupby
from classes import DownloadTask, ListTask, StopTask, LanguageError, PluginError, BadStateError
import Queue
import logging
import mimetypes
import os
import plugins
import traceback


# be nice
try:
    from logging import NullHandler
except ImportError:

    class NullHandler(logging.Handler):
        def emit(self, record):
            pass
logger = logging.getLogger('subliminal')
logger.addHandler(NullHandler())

# const
FORMATS = ['video/x-msvideo', 'video/quicktime', 'video/x-matroska', 'video/mp4']
LANGUAGES = ['aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az', 'ba', 'be', 'bg', 'bh', 'bi',
    'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce', 'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee',
    'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr', 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv',
    'ha', 'he', 'hi', 'ho', 'hr', 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is', 'it', 'iu',
    'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn', 'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb',
    'lg', 'li', 'ln', 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mo', 'mr', 'ms', 'mt', 'my', 'na',
    'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', 'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps',
    'pt', 'qu', 'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk', 'sl', 'sm', 'sn', 'so', 'sq',
    'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw',
    'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', 'yo', 'za', 'zh', 'zu']  # ISO 639-1
PLUGINS = ['Addic7ed', 'BierDopje', 'OpenSubtitles', 'SubsWiki', 'Subtitulos', 'TheSubDB']
API_PLUGINS = filter(lambda p: getattr(plugins, p).api_based, PLUGINS)
IDLE = 0
RUNNING = 1
PAUSED = 2


class Subliminal(object):
    """Main Subliminal class"""

    def __init__(self, cache_dir=False, workers=4, multi=False, force=False, max_depth=3, files_mode=-1):
        # set default values
        self.multi = multi
        self.force = force
        self.max_depth = max_depth
        self.cache_dir = None
        self.taskQueue = Queue.PriorityQueue()
        self.resultQueue = Queue.Queue()
        self._languages = None
        self._plugins = API_PLUGINS
        self.workers = workers
        self.files_mode = files_mode
        self.state = IDLE
        # handle cache directory preferences
        try:
            if cache_dir:  # custom configuration file
                self.cache_dir = cache_dir
                if not os.path.isdir(self.cache_dir):  # custom file doesn't exist, create it
                    os.makedirs(self.cache_dir)
                    logger.debug(u'Creating cache directory: %s' % self.cache_dir)
        except:
            self.cache_dir = None
            logger.error(u'Failed to use the cache directory, continue without it')

    def get_languages(self):
        """Getter for languages"""
        return self._languages

    def set_languages(self, languages):
        """Setter for languages"""
        logger.debug(u'Setting languages to %r' % languages)
        for l in languages:
            if l not in LANGUAGES:
                raise LanguageError(l)
        self._languages = languages

    def get_plugins(self):
        """Getter for plugins"""
        return self._plugins

    def set_plugins(self, plugins):
        """Setter for plugins"""
        logger.debug(u'Setting plugins to %r' % plugins)
        for p in plugins:
            if p not in PLUGINS:
                raise PluginError(p)
        self._plugins = plugins

    # getters/setters for the property _languages and _plugins
    languages = property(get_languages, set_languages)
    plugins = property(get_plugins, set_plugins)

    def listSubtitles(self, entries):
        """
        Search subtitles within the plugins and return all found subtitles in a list of Subtitle object.
        No need to worry about workers.

        Attributes:
            entries -- filepath or folderpath of video file or a list of that"""
        if self.state != IDLE:
            raise BadStateError(self.state)
        self.startWorkers()
        # valid argument
        if isinstance(entries, basestring):
            entries = [entries]
        # find files and languages
        search_results = []
        for e in entries:
            search_results.extend(self._recursiveSearch(e))
        # find subtitles
        task_count = 0
        for (filepath, languages) in search_results:
            logger.debug(u'Listing subtitles for %s with languages %r in plugins %r' % (filepath, languages, self._plugins))
            for plugin in self._plugins:
                self.taskQueue.put((5, ListTask(filepath, languages, plugin, self.getConfigDict())))
                task_count += 1
        subtitles = []
        for _ in range(task_count):
            subtitles.extend(self.resultQueue.get(timeout=4))
        self.stopWorkers()
        return subtitles

    def downloadSubtitles(self, entries):
        """
        Download subtitles using the plugins preferences and languages. Also use internal algorithm to find
        the best match inside a plugin.
        No need to worry about workers.

        Attributes:
            entries -- filepath or folderpath of video file or a list of that"""
        if self.state != IDLE:
            raise BadStateError(self.state)
        self.startWorkers()
        subtitles = self.listSubtitles(entries)
        task_count = 0
        for (_, subsBySource) in groupby(sorted(subtitles, key=lambda x: x.source), lambda x: x.source):
            if not self.multi:
                self.taskQueue.put((5, DownloadTask(sorted(list(subsBySource), cmp=self._cmpSubtitles))))
                task_count += 1
                continue
            for (__, subsBySourceByLanguage) in groupby(sorted(subsBySource, key=lambda x: x.language), lambda x: x.language):
                self.taskQueue.put((5, DownloadTask(sorted(list(subsBySourceByLanguage), cmp=self._cmpSubtitles))))
                task_count += 1
        paths = []
        for _ in range(task_count):
            paths.append(self.resultQueue.get(timeout=10))
        self.stopWorkers()
        return paths

    def _cmpSubtitles(self, x, y):
        """Compares 2 subtitles elements x and y using source, languages and plugin"""
        sources = sorted([x.source, y.source])
        if x.source != y.source and sources.index(x.source) < sources(y.source):
            return - 1
        if x.source != y.source and sources.index(x.source) > sources(y.source):
            return 1
        if self._languages and self._languages.index(x.language) < self._languages.index(y.language):
            return - 1
        if self._languages and self._languages.index(x.language) > self._languages.index(y.language):
            return 1
        if self._plugins.index(x.plugin) < self._plugins.index(y.plugin):<commit_msg>Add auto argument to manage workers<commit_after># -*- coding: utf-8 -*-
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 <NAME> <<EMAIL>>
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import threading
from itertools import groupby
from classes import DownloadTask, ListTask, StopTask, LanguageError, PluginError, BadStateError
import Queue
import logging
import mimetypes
import os
import plugins
import traceback


# be nice
try:
    from logging import NullHandler
except ImportError:

    class NullHandler(logging.Handler):
        def emit(self, record):
            pass
logger = logging.getLogger('subliminal')
logger.addHandler(NullHandler())

# const
FORMATS = ['video/x-msvideo', 'video/quicktime', 'video/x-matroska', 'video/mp4']
LANGUAGES = ['aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az', 'ba', 'be', 'bg', 'bh', 'bi',
    'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce', 'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee',
    'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr', 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv',
    'ha', 'he', 'hi', 'ho', 'hr', 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is', 'it', 'iu',
    'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn', 'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb',
    'lg', 'li', 'ln', 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mo', 'mr', 'ms', 'mt', 'my', 'na',
    'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', 'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps',
    'pt', 'qu', 'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk', 'sl', 'sm', 'sn', 'so', 'sq',
    'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw',
    'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', 'yo', 'za', 'zh', 'zu']  # ISO 639-1
PLUGINS = ['Addic7ed', 'BierDopje', 'OpenSubtitles', 'SubsWiki', 'Subtitulos', 'TheSubDB']
API_PLUGINS = filter(lambda p: getattr(plugins, p).api_based, PLUGINS)
IDLE = 0
RUNNING = 1
PAUSED = 2


class Subliminal(object):
    """Main Subliminal class"""

    def __init__(self, cache_dir=False, workers=4, multi=False, force=False, max_depth=3, files_mode=-1):
        # set default values
        self.multi = multi
        self.force = force
        self.max_depth = max_depth
        self.cache_dir = None
        self.taskQueue = Queue.PriorityQueue()
        self.resultQueue = Queue.Queue()
        self._languages = None
        self._plugins = API_PLUGINS
        self.workers = workers
        self.files_mode = files_mode
        self.state = IDLE
        # handle cache directory preferences
        try:
            if cache_dir:  # custom configuration file
                self.cache_dir = cache_dir
                if not os.path.isdir(self.cache_dir):  # custom file doesn't exist, create it
                    os.makedirs(self.cache_dir)
                    logger.debug(u'Creating cache directory: %s' % self.cache_dir)
        except:
            self.cache_dir = None
            logger.error(u'Failed to use the cache directory, continue without it')

    def get_languages(self):
        """Getter for languages"""
        return self._languages

    def set_languages(self, languages):
        """Setter for languages"""
        logger.debug(u'Setting languages to %r' % languages)
        for l in languages:
            if l not in LANGUAGES:
                raise LanguageError(l)
        self._languages = languages

    def get_plugins(self):
        """Getter for plugins"""
        return self._plugins

    def set_plugins(self, plugins):
        """Setter for plugins"""
        logger.debug(u'Setting plugins to %r' % plugins)
        for p in plugins:
            if p not in PLUGINS:
                raise PluginError(p)
        self._plugins = plugins

    # getters/setters for the property _languages and _plugins
    languages = property(get_languages, set_languages)
    plugins = property(get_plugins, set_plugins)

    def listSubtitles(self, entries, auto=True):
        """
        Search subtitles within the plugins and return all found subtitles in a list of Subtitle object.

        Attributes:
            entries -- filepath or folderpath of video file or a list of that
            auto    -- automaticaly manage workers"""
        if auto:
            if self.state != IDLE:
                raise BadStateError(self.state)
            self.startWorkers()
        # valid argument
        if isinstance(entries, basestring):
            entries = [entries]
        # find files and languages
        search_results = []
        for e in entries:
            search_results.extend(self._recursiveSearch(e))
        # find subtitles
        task_count = 0
        for (filepath, languages) in search_results:
            logger.debug(u'Listing subtitles for %s with languages %r in plugins %r' % (filepath, languages, self._plugins))
            for plugin in self._plugins:
                self.taskQueue.put((5, ListTask(filepath, languages, plugin, self.getConfigDict())))
                task_count += 1
        subtitles = []
        for _ in range(task_count):
            subtitles.extend(self.resultQueue.get(timeout=4))
        if auto:
            self.stopWorkers()
        return subtitles

    def downloadSubtitles(self, entries, auto=True):
        """
        Download subtitles using the plugins preferences and languages. Also use internal algorithm to find
        the best match inside a plugin.

        Attributes:
            entries -- filepath or folderpath of video file or a list of that
            auto    -- automaticaly manage workers"""
        if auto:
            if self.state != IDLE:
                raise BadStateError(self.state)
            self.startWorkers()
        subtitles = self.listSubtitles(entries, False)
        task_count = 0
        for (_, subsBySource) in groupby(sorted(subtitles, key=lambda x: x.source), lambda x: x.source):
            if not self.multi:
                self.taskQueue.put((5, DownloadTask(sorted(list(subsBySource), cmp=self._cmpSubtitles))))
                task_count += 1
                continue
            for (__, subsBySourceByLanguage) in groupby(sorted(subsBySource, key=lambda x: x.language), lambda x: x.language):
                self.taskQueue.put((5, DownloadTask(sorted(list(subsBySourceByLanguage), cmp=self._cmpSubtitles))))
                task_count += 1
        paths = []
        for _ in range(task_count):
            paths.append(self.resultQueue.get(timeout=10))
        if auto:
            self.stopWorkers()
        return paths

    def _cmpSubtitles(self, x, y):
        """Compares 2 subtitles elements x and y using source, languages and plugin"""
        sources = sorted([x.source, y.source])
        if x.source != y.source and sources.index(x.source) < sources(y.source):
            return - 1
        if x.source != y.source and sources.index(x.source) > sources(y.source):
            return 1
        if self._languages and self._languages.index(x.language) < self._languages.index(y.language):
            return - 1
        if self._languages and self._languages.index(x.language) > self._languages.index(y.language):
            return 1
        if self._plugins.index(x.plugin) < self._plugins.index(y.plugin):
{'fraction_non_alphanumeric': 0.14757998872957234, 'fraction_numerical': 0.004257717112265982, 'mean_word_length': 2.649074708704592, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5905373', 'n_tokens_mistral': 5038, 'n_tokens_neox': 4931, 'n_words': 1567}