<commit_before>from __future__ import unicode_literals, division, absolute_import
import logging
import urllib

import feedparser

from flexget import plugin
from flexget.entry import Entry
from flexget.event import event
from flexget.utils.search import normalize_unicode

log = logging.getLogger('nyaa')

# TODO: Other categories
CATEGORIES = {'all': '0_0',
              'anime': '1_0'}
FILTERS = ['all', 'filter remakes', 'trusted only', 'a+ only']


class UrlRewriteNyaa(object):
    """Nyaa urlrewriter and search plugin."""

    def validator(self):
        from flexget import validator

        root = validator.factory()
        root.accept('choice').accept_choices(CATEGORIES)
        advanced = root.accept('dict')
        advanced.accept('choice', key='category').accept_choices(CATEGORIES)
        advanced.accept('choice', key='filter').accept_choices(FILTERS)
        return root

    def search(self, entry, config):
        if not isinstance(config, dict):
            config = {'category': config}
        config.setdefault('category', 'anime')
        config.setdefault('filter', 'all')
        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            name = normalize_unicode(search_string)
            url = 'http://www.nyaa.eu/?page=rss&cats=%s&filter=%s&term=%s' % (
                  CATEGORIES[config['category']], FILTERS.index(config['filter']), urllib.quote(name.encode('utf-8')))

            log.debug('requesting: %s' % url)
            rss = feedparser.parse(url)

            status = rss.get('status', False)
            if status != 200:
                raise plugin.PluginWarning('Search result not 200 (OK), received %s' % status)

            ex = rss.get('bozo_exception', False)
            if ex:
                raise plugin.PluginWarning('Got bozo_exception (bad feed)')

            for item in rss.entries:

                entry = Entry()
                entry['title'] = item.title
                entry['url'] = item.link
                # TODO: parse some shit
                #entry['torrent_seeds'] = int(item.seeds)
                #entry['torrent_leeches'] = int(item.leechs)
                #entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
                #entry['content_size'] = int(item.size) / 1024 / 1024

                entries.add(entry)

        return entries

    def url_rewritable(self, task, entry):
        return entry['url'].startswith('http://www.nyaa.eu/?page=torrentinfo&tid=')

    def url_rewrite(self, task, entry):
        entry['url'] = entry['url'].replace('torrentinfo', 'download')


@event('plugin.register')
def register_plugin():
    plugin.register(UrlRewriteNyaa, 'nyaa', groups=['search', 'urlrewriter'], api_ver=2)
<commit_msg>Fix some issues with nyaa search<commit_after>from __future__ import unicode_literals, division, absolute_import
import logging
import urllib

import feedparser

from flexget import plugin
from flexget.entry import Entry
from flexget.event import event
from flexget.utils.search import normalize_unicode

log = logging.getLogger('nyaa')

# TODO: Other categories
CATEGORIES = {'all': '0_0',
              'anime': '1_0'}
FILTERS = ['all', 'filter remakes', 'trusted only', 'a+ only']


class UrlRewriteNyaa(object):
    """Nyaa urlrewriter and search plugin."""

    def validator(self):
        from flexget import validator

        root = validator.factory()
        root.accept('choice').accept_choices(CATEGORIES)
        advanced = root.accept('dict')
        advanced.accept('choice', key='category').accept_choices(CATEGORIES)
        advanced.accept('choice', key='filter').accept_choices(FILTERS)
        return root

    def search(self, entry, config):
        if not isinstance(config, dict):
            config = {'category': config}
        config.setdefault('category', 'anime')
        config.setdefault('filter', 'all')
        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            name = normalize_unicode(search_string)
            url = 'http://www.nyaa.se/?page=rss&cats=%s&filter=%s&term=%s' % (
                  CATEGORIES[config['category']], FILTERS.index(config['filter']), urllib.quote(name.encode('utf-8')))

            log.debug('requesting: %s' % url)
            rss = feedparser.parse(url)

            status = rss.get('status', False)
            if status != 200:
                log.debug('Search result not 200 (OK), received %s' % status)
            if status >= 400:
                continue

            ex = rss.get('bozo_exception', False)
            if ex:
                log.error('Got bozo_exception (bad feed) on %s' % url)
                continue

            for item in rss.entries:

                entry = Entry()
                entry['title'] = item.title
                entry['url'] = item.link
                # TODO: parse some shit
                #entry['torrent_seeds'] = int(item.seeds)
                #entry['torrent_leeches'] = int(item.leechs)
                #entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
                #entry['content_size'] = int(item.size) / 1024 / 1024

                entries.add(entry)

        return entries

    def url_rewritable(self, task, entry):
        return entry['url'].startswith('http://www.nyaa.eu/?page=torrentinfo&tid=')

    def url_rewrite(self, task, entry):
        entry['url'] = entry['url'].replace('torrentinfo', 'download')


@event('plugin.register')
def register_plugin():
    plugin.register(UrlRewriteNyaa, 'nyaa', groups=['search', 'urlrewriter'], api_ver=2)

{'fraction_non_alphanumeric': 0.13603651044409337, 'fraction_numerical': 0.007547832192381956, 'mean_word_length': 2.6619537275064267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4250389', 'n_tokens_mistral': 1757, 'n_tokens_neox': 1635, 'n_words': 367}