<commit_before>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # noqa pylint: disable=unused-import, redefined-builtin

import logging
import os
import xmlrpc.client
from socket import error as socket_error

from flexget import plugin
from flexget.event import event
from flexget.utils.template import RenderError

log = logging.getLogger('aria2')


class OutputAria2(object):
    """
    Simple Aria2 output

    Example::

        aria2:
          path: ~/downloads/

    """

    schema = {
        'type': 'object',
        'properties': {
            'server': {'type': 'string', 'default': 'localhost'},
            'port': {'type': 'integer', 'default': 6800},
            'secret': {'type': 'string', 'default': ''},
            'username': {'type': 'string', 'default': ''}, # NOTE: To be deprecated by aria2
            'password': {'type': 'string', 'default': ''},
            'path': {'type': 'string', 'format': 'path'},
            'options': {
                'type': 'object',
                'additionalProperties': {'oneOf': [{'type': 'string'}, {'type': 'integer'}]}
            }

        },
        'required': ['path'],
        'additionalProperties': False
    }

    def aria2_connection(self, server, port, username=None, password=None):
        if username and password:
            userpass = '%s:%s@' % (username, password)
        else:
            userpass = ''
        url = 'http://%s%s:%s/rpc' % (userpass, server, port)
        log.debug('aria2 url: %s' % url)
        log.info('Connecting to daemon at %s', url)
        try:
            return xmlrpc.client.ServerProxy(url).aria2
        except xmlrpc.client.ProtocolError as err:
            raise plugin.PluginError('Could not connect to aria2 at %s. Protocol error %s: %s'
                              % (url, err.errcode, err.errmsg), log)
        except xmlrpc.client.Fault as err:
            raise plugin.PluginError('XML-RPC fault: Unable to connect to aria2 daemon at %s: %s'
                              % (url, err.faultString), log)
        except socket_error as e:
            raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s' % (url, e), log)
        except:
            log.debug('Unexpected error during aria2 connection', exc_info=True)
            raise plugin.PluginError('Unidentified error during connection to aria2 daemon', log)

    def prepare_config(self, config):
        config.setdefault('server', 'localhost')
        config.setdefault('port', 6800)
        config.setdefault('username', '')
        config.setdefault('password', '')
        config.setdefault('secret', '')
        config.setdefault('options', {})
        return config

    def on_task_output(self, task, config):
        # don't add when learning
        if task.options.learn:
            return
        config = self.prepare_config(config)
        aria2 = self.aria2_connection(config['server'], config['port'],
                                      config['username'], config['password'])
        for entry in task.accepted:
            if task.options.test:
                log.verbose('Would add `%s` to aria2.', entry['title'])
                continue
            try:
                self.add_entry(aria2, entry, config)
            except socket_error as se:
                entry.fail('Unable to reach Aria2: %s', se)
            except xmlrpc.client.Fault as err:
                log.critical('Fault code %s message %s', err.faultCode, err.faultString)
                entry.fail('Aria2 communication Fault')
            except Exception as e:
                log.debug('Exception type %s', type(e), exc_info=True)
                raise

    def add_entry(self, aria2, entry, config):
        """
        Add entry to Aria2
        """
        options = config['options']
        try:
            options['dir'] = os.path.expanduser(entry.render(config['path']).rstrip('/'))
        except RenderError as e:
            entry.fail('failed to render \'path\': %s' % e)
            return
        secret = None
        if config['secret']:
            secret = 'token:%s' % config['secret']
        # handle torrent files
        if 'torrent' in entry:
            if secret:
                return aria2.addTorrent(secret, xmlrpc.client.Binary(open(entry['file'], mode='rb').read()))
            return aria2.addTorrent(xmlrpc.client.Binary(open(entry['file'], mode='rb').read()))
        # handle everything else (except metalink -- which is unsupported)
        # so magnets, https, http, ftp .. etc
        if secret:
            return aria2.addUri(secret, [entry['url']], options)
        return aria2.addUri([entry['url']], options)


@event('plugin.register')
def register_plugin():
    plugin.register(OutputAria2, 'aria2', api_ver=2)
<commit_msg>[change] aria2: no longer requiring that 'path' exists, closes #1493<commit_after>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # noqa pylint: disable=unused-import, redefined-builtin

import logging
import os
import xmlrpc.client
from socket import error as socket_error

from flexget import plugin
from flexget.event import event
from flexget.utils.template import RenderError

log = logging.getLogger('aria2')


class OutputAria2(object):
    """
    Simple Aria2 output

    Example::

        aria2:
          path: ~/downloads/

    """

    schema = {
        'type': 'object',
        'properties': {
            'server': {'type': 'string', 'default': 'localhost'},
            'port': {'type': 'integer', 'default': 6800},
            'secret': {'type': 'string', 'default': ''},
            'username': {'type': 'string', 'default': ''}, # NOTE: To be deprecated by aria2
            'password': {'type': 'string', 'default': ''},
            'path': {'type': 'string'},
            'options': {
                'type': 'object',
                'additionalProperties': {'oneOf': [{'type': 'string'}, {'type': 'integer'}]}
            }

        },
        'required': ['path'],
        'additionalProperties': False
    }

    def aria2_connection(self, server, port, username=None, password=None):
        if username and password:
            userpass = '%<PASSWORD>@' % (username, password)
        else:
            userpass = ''
        url = 'http://%s%s:%s/rpc' % (userpass, server, port)
        log.debug('aria2 url: %s' % url)
        log.info('Connecting to daemon at %s', url)
        try:
            return xmlrpc.client.ServerProxy(url).aria2
        except xmlrpc.client.ProtocolError as err:
            raise plugin.PluginError('Could not connect to aria2 at %s. Protocol error %s: %s'
                              % (url, err.errcode, err.errmsg), log)
        except xmlrpc.client.Fault as err:
            raise plugin.PluginError('XML-RPC fault: Unable to connect to aria2 daemon at %s: %s'
                              % (url, err.faultString), log)
        except socket_error as e:
            raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s' % (url, e), log)
        except:
            log.debug('Unexpected error during aria2 connection', exc_info=True)
            raise plugin.PluginError('Unidentified error during connection to aria2 daemon', log)

    def prepare_config(self, config):
        config.setdefault('server', 'localhost')
        config.setdefault('port', 6800)
        config.setdefault('username', '')
        config.setdefault('password', '')
        config.setdefault('secret', '')
        config.setdefault('options', {})
        return config

    def on_task_output(self, task, config):
        # don't add when learning
        if task.options.learn:
            return
        config = self.prepare_config(config)
        aria2 = self.aria2_connection(config['server'], config['port'],
                                      config['username'], config['password'])
        for entry in task.accepted:
            if task.options.test:
                log.verbose('Would add `%s` to aria2.', entry['title'])
                continue
            try:
                self.add_entry(aria2, entry, config)
            except socket_error as se:
                entry.fail('Unable to reach Aria2: %s', se)
            except xmlrpc.client.Fault as err:
                log.critical('Fault code %s message %s', err.faultCode, err.faultString)
                entry.fail('Aria2 communication Fault')
            except Exception as e:
                log.debug('Exception type %s', type(e), exc_info=True)
                raise

    def add_entry(self, aria2, entry, config):
        """
        Add entry to Aria2
        """
        options = config['options']
        try:
            options['dir'] = os.path.expanduser(entry.render(config['path']).rstrip('/'))
        except RenderError as e:
            entry.fail('failed to render \'path\': %s' % e)
            return
        secret = None
        if config['secret']:
            secret = 'token:%s' % config['secret']
        # handle torrent files
        if 'torrent' in entry:
            if secret:
                return aria2.addTorrent(secret, xmlrpc.client.Binary(open(entry['file'], mode='rb').read()))
            return aria2.addTorrent(xmlrpc.client.Binary(open(entry['file'], mode='rb').read()))
        # handle everything else (except metalink -- which is unsupported)
        # so magnets, https, http, ftp .. etc
        if secret:
            return aria2.addUri(secret, [entry['url']], options)
        return aria2.addUri([entry['url']], options)


@event('plugin.register')
def register_plugin():
    plugin.register(OutputAria2, 'aria2', api_ver=2)

{'fraction_non_alphanumeric': 0.13398659102630223, 'fraction_numerical': 0.007942238267148015, 'mean_word_length': 2.187376725838264, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4418121', 'n_tokens_mistral': 2718, 'n_tokens_neox': 2638, 'n_words': 733}