<commit_before>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # pylint: disable=unused-import, redefined-builtin

import os
import re
import logging

from sqlalchemy import Column, Integer, Unicode

from flexget import options, plugin
from flexget.utils.tools import native_str_to_text
from flexget.db_schema import versioned_base
from flexget.entry import Entry
from flexget.event import event
from flexget.manager import Session

log = logging.getLogger('tail')
Base = versioned_base('tail', 0)


class TailPosition(Base):
    __tablename__ = 'tail'
    id = Column(Integer, primary_key=True)
    task = Column(Unicode)
    filename = Column(Unicode)
    position = Column(Integer)


class InputTail(object):

    """
    Parse any text for entries using regular expression.

    ::

      file: <file>
      entry:
        <field>: <regexp to match value>
      format:
        <field>: <python string formatting>

    Note: each entry must have at least two fields, title and url

    You may wish to specify encoding used by file so file can be properly
    decoded. List of encodings
    at http://docs.python.org/library/codecs.html#standard-encodings.

    Example::

      tail:
        file: ~/irclogs/some/log
        entry:
          title: 'TITLE: (.*) URL:'
          url: 'URL: (.*)'
        encoding: utf8
    """
    schema = {
        'type': 'object',
        'properties': {
            'file': {'type': 'string', 'format': 'file'},
            'encoding': {'type': 'string'},
            'entry': {
                'type': 'object',
                'properties': {
                    'url': {'type': 'string', 'format': 'regex'},
                    'title': {'type': 'string', 'format': 'regex'}
                },
                'required': ['url', 'title']
            },
            'format': {
                'type': 'object',
                'additionalProperties': {'type': 'string'}
            }
        },
        'required': ['file', 'entry'],
        'additionalProperties': False
    }

    def format_entry(self, entry, d):
        for k, v in d.items():
            entry[k] = v % entry

    def on_task_input(self, task, config):

        # Let details plugin know that it is ok if this task doesn't produce any entries
        task.no_entries_ok = True

        filename = os.path.expanduser(config['file'])
        encoding = config.get('encoding', None)
        with Session() as session:
            db_pos = (session.query(TailPosition).
                      filter(TailPosition.task == task.name).filter(TailPosition.filename == filename).first())
            if db_pos:
                last_pos = db_pos.position
            else:
                last_pos = 0

            with open(filename, 'r') as file:
                if task.options.tail_reset == filename or task.options.tail_reset == task.name:
                    if last_pos == 0:
                        log.info('Task %s tail position is already zero' % task.name)
                    else:<commit_msg>[fix] - tail - Open file as binary. Fixes #1269<commit_after>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # pylint: disable=unused-import, redefined-builtin

import os
import re
import logging

from sqlalchemy import Column, Integer, Unicode

from flexget import options, plugin
from flexget.utils.tools import native_str_to_text
from flexget.db_schema import versioned_base
from flexget.entry import Entry
from flexget.event import event
from flexget.manager import Session

log = logging.getLogger('tail')
Base = versioned_base('tail', 0)


class TailPosition(Base):
    __tablename__ = 'tail'
    id = Column(Integer, primary_key=True)
    task = Column(Unicode)
    filename = Column(Unicode)
    position = Column(Integer)


class InputTail(object):

    """
    Parse any text for entries using regular expression.

    ::

      file: <file>
      entry:
        <field>: <regexp to match value>
      format:
        <field>: <python string formatting>

    Note: each entry must have at least two fields, title and url

    You may wish to specify encoding used by file so file can be properly
    decoded. List of encodings
    at http://docs.python.org/library/codecs.html#standard-encodings.

    Example::

      tail:
        file: ~/irclogs/some/log
        entry:
          title: 'TITLE: (.*) URL:'
          url: 'URL: (.*)'
        encoding: utf8
    """
    schema = {
        'type': 'object',
        'properties': {
            'file': {'type': 'string', 'format': 'file'},
            'encoding': {'type': 'string'},
            'entry': {
                'type': 'object',
                'properties': {
                    'url': {'type': 'string', 'format': 'regex'},
                    'title': {'type': 'string', 'format': 'regex'}
                },
                'required': ['url', 'title']
            },
            'format': {
                'type': 'object',
                'additionalProperties': {'type': 'string'}
            }
        },
        'required': ['file', 'entry'],
        'additionalProperties': False
    }

    def format_entry(self, entry, d):
        for k, v in d.items():
            entry[k] = v % entry

    def on_task_input(self, task, config):

        # Let details plugin know that it is ok if this task doesn't produce any entries
        task.no_entries_ok = True

        filename = os.path.expanduser(config['file'])
        encoding = config.get('encoding', None)
        with Session() as session:
            db_pos = (session.query(TailPosition).
                      filter(TailPosition.task == task.name).filter(TailPosition.filename == filename).first())
            if db_pos:
                last_pos = db_pos.position
            else:
                last_pos = 0

            with open(filename, 'rb') as file:
                if task.options.tail_reset == filename or task.options.tail_reset == task.name:
                    if last_pos == 0:
                        log.info('Task %s tail position is already zero' % task.name)
                    else:
{'fraction_non_alphanumeric': 0.1171303074670571, 'fraction_numerical': 0.0019521717911176184, 'mean_word_length': 2.048091224590977, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7293913', 'n_tokens_mistral': 1744, 'n_tokens_neox': 1662, 'n_words': 508}