<commit_before>import logging
from datetime import datetime, timedelta
from sqlalchemy import Column, Integer, String, DateTime, PickleType
from flexget.manager import Base, Session
from flexget.feed import Entry
from flexget.plugin import register_plugin, priority, PluginWarning

log = logging.getLogger('backlog')


class BacklogEntry(Base):

    __tablename__ = 'backlog'

    id = Column(Integer, primary_key=True)
    feed = Column(String)
    title = Column(String)
    expire = Column(DateTime)
    entry = Column(PickleType(mutable=False))

    def __repr__(self):
        return '<BacklogEntry(title=%s)>' % (self.title)


class InputBacklog(object):
    """
    Keeps feed history for given amount of time.

    Example:

    backlog: 4 days

    Rarely useful for end users, mainly used by other plugins.
    """

    def validator(self):
        from flexget import validator
        root = validator.factory('regexp_match', message='Must be in format <number> <hours|minutes|days|weeks>')
        root.accept('\d+ (minute|hour|day|week)s?')
        return root

    def get_amount(self, value):
        amount, unit = value.split(' ')
        # Make sure unit name is plural.
        if not unit.endswith('s'):
            unit = unit + 's'
        log.debug('amount: %s unit: %s' % (repr(amount), repr(unit)))
        params = {unit: int(amount)}
        try:
            return timedelta(**params)
        except TypeError:
            raise PluginWarning('Invalid time format \'%s\'' % value, log)

    @priority(-255)
    def on_feed_input(self, feed, config):
        # Take a snapshot of the entries' states after the input event in case we have to store them to backlog
        for entry in feed.entries:
            entry.take_snapshot('after_input')
        injections = self.get_injections(feed)
        if config:
            # If backlog is manually enabled for this feed, learn the entries.
            self.learn_backlog(feed, config)
        # Return the entries from backlog that are not already in the feed
        return injections

    def on_feed_abort(self, feed):
        """Remember all entries for 12 hours when feed gets aborted."""
        log.debug('Remembering all entries to backlog for 12 hours because of feed abort.')
        self.learn_backlog(feed, '12 hours')

    def add_backlog(self, feed, entry, amount=''):
        """Add single entry to feed backlog"""
        snapshot = entry.snapshots.get('after_input')
        if not snapshot:
            log.warning('No input snapshot available for `%s`, using current state' % entry['title'])
            snapshot = dict(entry)
        session = Session()
        expire_time = datetime.now() + self.get_amount(amount)
        backlog_entry = session.query(BacklogEntry).filter(BacklogEntry.title == entry['title']).\
                                                filter(BacklogEntry.feed == feed.name).first()
        if backlog_entry:
            # If there is already a backlog entry for this, update the expiry time if necessary.
            if backlog_entry.expire < expire_time:
                log.debug('Updating expiry time for %s' % entry['title'])
                backlog_entry.expire = expire_time
        else:
            log.debug('Saving %s' % entry['title'])
            backlog_entry = BacklogEntry()
            backlog_entry.title = entry['title']
            backlog_entry.entry = snapshot
            backlog_entry.feed = feed.name
            backlog_entry.expire = expire_time
            session.add(backlog_entry)
        session.commit()

    def learn_backlog(self, feed, amount=''):<commit_msg>Update backlog on_feed_abort to api v2<commit_after>import logging
from datetime import datetime, timedelta
from sqlalchemy import Column, Integer, String, DateTime, PickleType
from flexget.manager import Base, Session
from flexget.feed import Entry
from flexget.plugin import register_plugin, priority, PluginWarning

log = logging.getLogger('backlog')


class BacklogEntry(Base):

    __tablename__ = 'backlog'

    id = Column(Integer, primary_key=True)
    feed = Column(String)
    title = Column(String)
    expire = Column(DateTime)
    entry = Column(PickleType(mutable=False))

    def __repr__(self):
        return '<BacklogEntry(title=%s)>' % (self.title)


class InputBacklog(object):
    """
    Keeps feed history for given amount of time.

    Example:

    backlog: 4 days

    Rarely useful for end users, mainly used by other plugins.
    """

    def validator(self):
        from flexget import validator
        root = validator.factory('regexp_match', message='Must be in format <number> <hours|minutes|days|weeks>')
        root.accept('\d+ (minute|hour|day|week)s?')
        return root

    def get_amount(self, value):
        amount, unit = value.split(' ')
        # Make sure unit name is plural.
        if not unit.endswith('s'):
            unit = unit + 's'
        log.debug('amount: %s unit: %s' % (repr(amount), repr(unit)))
        params = {unit: int(amount)}
        try:
            return timedelta(**params)
        except TypeError:
            raise PluginWarning('Invalid time format \'%s\'' % value, log)

    @priority(-255)
    def on_feed_input(self, feed, config):
        # Take a snapshot of the entries' states after the input event in case we have to store them to backlog
        for entry in feed.entries:
            entry.take_snapshot('after_input')
        injections = self.get_injections(feed)
        if config:
            # If backlog is manually enabled for this feed, learn the entries.
            self.learn_backlog(feed, config)
        # Return the entries from backlog that are not already in the feed
        return injections

    def on_feed_abort(self, feed, config):
        """Remember all entries for 12 hours when feed gets aborted."""
        log.debug('Remembering all entries to backlog for 12 hours because of feed abort.')
        self.learn_backlog(feed, '12 hours')

    def add_backlog(self, feed, entry, amount=''):
        """Add single entry to feed backlog"""
        snapshot = entry.snapshots.get('after_input')
        if not snapshot:
            log.warning('No input snapshot available for `%s`, using current state' % entry['title'])
            snapshot = dict(entry)
        session = Session()
        expire_time = datetime.now() + self.get_amount(amount)
        backlog_entry = session.query(BacklogEntry).filter(BacklogEntry.title == entry['title']).\
                                                filter(BacklogEntry.feed == feed.name).first()
        if backlog_entry:
            # If there is already a backlog entry for this, update the expiry time if necessary.
            if backlog_entry.expire < expire_time:
                log.debug('Updating expiry time for %s' % entry['title'])
                backlog_entry.expire = expire_time
        else:
            log.debug('Saving %s' % entry['title'])
            backlog_entry = BacklogEntry()
            backlog_entry.title = entry['title']
            backlog_entry.entry = snapshot
            backlog_entry.feed = feed.name
            backlog_entry.expire = expire_time
            session.add(backlog_entry)
        session.commit()

    def learn_backlog(self, feed, amount=''):
{'fraction_non_alphanumeric': 0.09993127147766323, 'fraction_numerical': 0.00288659793814433, 'mean_word_length': 2.551000488042948, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5194459', 'n_tokens_mistral': 2056, 'n_tokens_neox': 1980, 'n_words': 621}