<commit_before>"""Plugin for text file or URL feeds via regex."""
from __future__ import unicode_literals, division, absolute_import
import re
import logging

from flexget import plugin
from flexget.entry import Entry
from flexget.event import event
from flexget.utils.cached_input import cached

log = logging.getLogger('text')


class Text(object):

    """
    Parse any text for entries using regular expression.

    Example::

      url: <url>
      entry:
        <field>: <regexp to match value>
      format:
        <field>: <python string formatting>

    Note: each entry must have atleast two fields, title and url

    Example::

      text:
        url: http://www.nbc.com/Heroes/js/novels.js
        entry:
          title: novelTitle = "(.*)"
          url: novelPrint = "(.*)"
        format:
          url: http://www.nbc.com%(url)s
    """
    schema = {
        'type': 'object',
        'properties': {
            'url': {
                'oneOf': [
                    {'type': 'string', 'format': 'url'},
                    {'type': 'string', 'format': 'file'}
                ]
            },
            'entry': {
                'type': 'object',
                'properties': {
                    'url': {'type': 'string', 'format': 'regex'},
                    'title': {'type': 'string', 'format': 'regex'}
                },
                'additionalProperties': {'type': 'string', 'format': 'regex'},
                'required': ['url', 'title']
            },
            'format': {
                'type': 'object',
                'additionalProperties': {'type': 'string'}
            }
        },
        'required': ['entry', 'url'],
        'additonalProperties': False
    }

    def format_entry(self, entry, d):
        for k, v in d.iteritems():
            entry[k] = v % entry

    @cached('text')
    @plugin.internet(log)
    def on_task_input(self, task, config):
        url = config['url']
        if '://' in url:
            lines = task.requests.get(url).iter_lines()
        else:
            lines = open(url, 'rb').readlines()

        entry_config = config.get('entry')
        format_config = config.get('format', {})

        entries = []
        # keep track what fields have been found
        used = {}
        entry = Entry()

        # now parse text
        for line in lines:
            for field, regexp in entry_config.iteritems():
                # log.debug('search field: %s regexp: %s' % (field, regexp))
                match = re.search(regexp, line)
                if match:
                    # check if used field detected, in such case start with new entry<commit_msg>[feature] Add 'encoding' option to text plugin. fix #469<commit_after>"""Plugin for text file or URL feeds via regex."""
from __future__ import unicode_literals, division, absolute_import
import re
import logging

import path

from flexget import plugin
from flexget.entry import Entry
from flexget.event import event
from flexget.utils.cached_input import cached

log = logging.getLogger('text')


class Text(object):

    """
    Parse any text for entries using regular expression.

    Example::

      url: <url>
      entry:
        <field>: <regexp to match value>
      format:
        <field>: <python string formatting>

    Note: each entry must have atleast two fields, title and url

    Example::

      text:
        url: http://www.nbc.com/Heroes/js/novels.js
        entry:
          title: novelTitle = "(.*)"
          url: novelPrint = "(.*)"
        format:
          url: http://www.nbc.com%(url)s
    """
    schema = {
        'type': 'object',
        'properties': {
            'url': {
                'oneOf': [
                    {'type': 'string', 'format': 'url'},
                    {'type': 'string', 'format': 'file'}
                ]
            },
            'encoding': {'type': 'string'},
            'entry': {
                'type': 'object',
                'properties': {
                    'url': {'type': 'string', 'format': 'regex'},
                    'title': {'type': 'string', 'format': 'regex'}
                },
                'additionalProperties': {'type': 'string', 'format': 'regex'},
                'required': ['url', 'title']
            },
            'format': {
                'type': 'object',
                'additionalProperties': {'type': 'string'}
            }
        },
        'required': ['entry', 'url'],
        'additonalProperties': False
    }

    def format_entry(self, entry, d):
        for k, v in d.iteritems():
            entry[k] = v % entry

    @cached('text')
    @plugin.internet(log)
    def on_task_input(self, task, config):
        url = config['url']
        if '://' in url:
            lines = task.requests.get(url).iter_lines(decode_unicode=True)
        else:
            lines = path.Path(url).lines(encoding=config.get('encoding', 'utf-8'))

        entry_config = config.get('entry')
        format_config = config.get('format', {})

        entries = []
        # keep track what fields have been found
        used = {}
        entry = Entry()

        # now parse text
        for line in lines:
            for field, regexp in entry_config.iteritems():
                # log.debug('search field: %s regexp: %s' % (field, regexp))
                match = re.search(regexp, line)
                if match:
                    # check if used field detected, in such case start with new entry
{'fraction_non_alphanumeric': 0.1393562545720556, 'fraction_numerical': 0.000731528895391368, 'mean_word_length': 1.7168405365126678, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2619575', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1471, 'n_words': 434}