<commit_before>from __future__ import absolute_import, division, unicode_literals
from past.builtins import cmp
from builtins import next
from builtins import map
from builtins import str
from builtins import range
from past.builtins import basestring
from builtins import object

import copy
import hashlib
import itertools
import logging
import threading
import random
import string
from functools import wraps

from sqlalchemy import Column, Integer, String, Unicode

from flexget import config_schema, db_schema
from flexget.entry import EntryUnicodeError
from flexget.event import event, fire_event
from flexget.logger import capture_output
from flexget.manager import Session
from flexget.plugin import plugins as all_plugins
from flexget.plugin import (
    DependencyError, get_plugins, phase_methods, plugin_schemas, PluginError, PluginWarning, task_phases)
from flexget.utils import requests
from flexget.utils.database import with_session
from flexget.utils.simple_persistence import SimpleTaskPersistence

log = logging.getLogger('task')
Base = db_schema.versioned_base('feed', 0)


class TaskConfigHash(Base):
    """Stores the config hash for tasks so that we can tell if the config has changed since last run."""

    __tablename__ = 'feed_config_hash'

    id = Column(Integer, primary_key=True)
    task = Column('name', Unicode, index=True, nullable=False)
    hash = Column('hash', String)

    def __repr__(self):
        return '<TaskConfigHash(task=%s,hash=%s)>' % (self.task, self.hash)


@with_session
def config_changed(task=None, session=None):
    """
    Forces config_modified flag to come out true on next run of `task`. Used when the db changes, and all
    entries need to be reprocessed.

    :param task: Name of the task. If `None`, will be set for all tasks.
    """
    log.debug('Marking config for %s as changed.' % (task or 'all tasks'))
    task_hash = session.query(TaskConfigHash)
    if task:
        task_hash = task_hash.filter(TaskConfigHash.task == task)
    task_hash.delete()


def use_task_logging(func):

    @wraps(func)
    def wrapper(self, *args, **kw):
        # Set the task name in the logger and capture output
        from flexget import logger
        with logger.task_logging(self.name):
            if self.output:
                with capture_output(self.output, loglevel=self.loglevel):
                    return func(self, *args, **kw)
            else:
                return func(self, *args, **kw)

    return wrapper


class EntryIterator(object):
    """An iterator over a subset of entries to emulate old task.accepted/rejected/failed/entries properties."""

    def __init__(self, entries, states):
        self.all_entries = entries
        if isinstance(states, basestring):
            states = [states]
        self.filter = lambda e: e._state in states

    def __iter__(self):
        return filter(self.filter, self.all_entries)

    def __bool__(self):
        return any(e for e in self)

    def __len__(self):
        return sum(1 for e in self)

    def __add__(self, other):
        return itertools.chain(self, other)

    def __radd__(self, other):
        return itertools.chain(other, self)

    def __getitem__(self, item):
        if not isinstance(item, int):
            raise ValueError('Index must be integer.')
        for index, entry in enumerate(self):
            if index == item:
                return entry
        else:
            raise IndexError('%d is out of bounds' % item)

    def __getslice__(self, a, b):
        return list(itertools.islice(self, a, b))

    def reverse(self):
        self.all_entries.sort(reverse=True)

    def sort(self, *args, **kwargs):
        self.all_entries.sort(*args, **kwargs)


class EntryContainer(list):
    """Container for a list of entries, also contains accepted, rejected failed iterators over them."""<commit_msg>Convert to new style slicing for EntryIterator<commit_after>from __future__ import absolute_import, division, unicode_literals
from past.builtins import cmp
from builtins import next
from builtins import map
from builtins import str
from builtins import range
from past.builtins import basestring
from builtins import object

import copy
import hashlib
import itertools
import logging
import threading
import random
import string
from functools import wraps

from sqlalchemy import Column, Integer, String, Unicode

from flexget import config_schema, db_schema
from flexget.entry import EntryUnicodeError
from flexget.event import event, fire_event
from flexget.logger import capture_output
from flexget.manager import Session
from flexget.plugin import plugins as all_plugins
from flexget.plugin import (
    DependencyError, get_plugins, phase_methods, plugin_schemas, PluginError, PluginWarning, task_phases)
from flexget.utils import requests
from flexget.utils.database import with_session
from flexget.utils.simple_persistence import SimpleTaskPersistence

log = logging.getLogger('task')
Base = db_schema.versioned_base('feed', 0)


class TaskConfigHash(Base):
    """Stores the config hash for tasks so that we can tell if the config has changed since last run."""

    __tablename__ = 'feed_config_hash'

    id = Column(Integer, primary_key=True)
    task = Column('name', Unicode, index=True, nullable=False)
    hash = Column('hash', String)

    def __repr__(self):
        return '<TaskConfigHash(task=%s,hash=%s)>' % (self.task, self.hash)


@with_session
def config_changed(task=None, session=None):
    """
    Forces config_modified flag to come out true on next run of `task`. Used when the db changes, and all
    entries need to be reprocessed.

    :param task: Name of the task. If `None`, will be set for all tasks.
    """
    log.debug('Marking config for %s as changed.' % (task or 'all tasks'))
    task_hash = session.query(TaskConfigHash)
    if task:
        task_hash = task_hash.filter(TaskConfigHash.task == task)
    task_hash.delete()


def use_task_logging(func):

    @wraps(func)
    def wrapper(self, *args, **kw):
        # Set the task name in the logger and capture output
        from flexget import logger
        with logger.task_logging(self.name):
            if self.output:
                with capture_output(self.output, loglevel=self.loglevel):
                    return func(self, *args, **kw)
            else:
                return func(self, *args, **kw)

    return wrapper


class EntryIterator(object):
    """An iterator over a subset of entries to emulate old task.accepted/rejected/failed/entries properties."""

    def __init__(self, entries, states):
        self.all_entries = entries
        if isinstance(states, basestring):
            states = [states]
        self.filter = lambda e: e._state in states

    def __iter__(self):
        return filter(self.filter, self.all_entries)

    def __bool__(self):
        return any(e for e in self)

    def __len__(self):
        return sum(1 for e in self)

    def __add__(self, other):
        return itertools.chain(self, other)

    def __radd__(self, other):
        return itertools.chain(other, self)

    def __getitem__(self, item):
        if isinstance(item, slice):
            return list(itertools.islice(self, item.start, item.stop))
        if not isinstance(item, int):
            raise ValueError('Index must be integer.')
        for index, entry in enumerate(self):
            if index == item:
                return entry
        else:
            raise IndexError('%d is out of bounds' % item)

    def reverse(self):
        self.all_entries.sort(reverse=True)

    def sort(self, *args, **kwargs):
        self.all_entries.sort(*args, **kwargs)


class EntryContainer(list):
    """Container for a list of entries, also contains accepted, rejected failed iterators over them."""
{'fraction_non_alphanumeric': 0.08608082103912765, 'fraction_numerical': 0.0005131494547787043, 'mean_word_length': 3.3095632946379214, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7412108', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2150, 'n_words': 764}