<commit_before>from collections import namedtuple
from itertools import groupby
from pathlib import Path
from typing import List, Optional
from urllib.parse import unquote, urlparse

from loguru import logger

from flexget import plugin
from flexget.components.ftp.sftp_client import SftpClient, SftpError
from flexget.config_schema import one_or_more
from flexget.entry import Entry
from flexget.event import event
from flexget.task import Task
from flexget.utils.template import RenderError, render_from_entry

logger = logger.bind(name='sftp')

# Constants
DEFAULT_SFTP_PORT: int = 22
DEFAULT_CONNECT_TRIES: int = 3
DEFAULT_SOCKET_TIMEOUT_SEC: int = 15


SftpConfig = namedtuple(
    'SftpConfig', ['host', 'port', 'username', 'password', 'private_key', 'private_key_pass']
)


class SftpList:
    """
    Generate entries from SFTP. This plugin requires the pysftp Python module and its dependencies.

    Configuration:

    host:                 Host to connect to.
    port:                 Port the remote SSH server is listening on (default 22).
    username:             Username to log in as.
    password:             The password to use. Optional if a private key is provided.
    private_key:          Path to the private key (if any) to log into the SSH server.
    private_key_pass:     Password for the private key (if needed).
    recursive:            Indicates whether the listing should be recursive.
    get_size:             Indicates whetern to calculate the size of the remote file/directory.
                          WARNING: This can be very slow when computing the size of directories!
    files_only:           Indicates wheter to omit diredtories from the results.
    dirs:                 List of directories to download.
    socket_timeout_sec:   Socket timeout in seconds (default 15 seconds).
    connection_tries:     Number of times to attempt to connect before failing (default 3).

    Example:

      sftp_list:
          host: example.com
          username: Username
          private_key: /Users/username/.ssh/id_rsa
          recursive: False
          get_size: True
          files_only: False
          dirs:
              - '/path/to/list/'
              - '/another/path/'
    """

    schema = {
        'type': 'object',
        'properties': {
            'host': {'type': 'string'},
            'username': {'type': 'string'},
            'password': {'type': 'string'},
            'port': {'type': 'integer', 'default': DEFAULT_SFTP_PORT},
            'files_only': {'type': 'boolean', 'default': True},
            'recursive': {'type': 'boolean', 'default': False},
            'get_size': {'type': 'boolean', 'default': True},
            'private_key': {'type': 'string'},
            'private_key_pass': {'type': 'string'},
            'dirs': one_or_more({'type': 'string'}),
            'socket_timeout_sec': {'type': 'integer', 'default': DEFAULT_SOCKET_TIMEOUT_SEC},
            'connection_tries': {'type': 'integer', 'default': DEFAULT_CONNECT_TRIES},
        },
        'additionProperties': False,
        'required': ['host', 'username'],
    }

    @staticmethod
    def prepare_config(config: dict) -> dict:
        """
        Sets defaults for the provided configuration
        """
        config.setdefault('password', None)
        config.setdefault('private_key', None)
        config.setdefault('private_key_pass', None)
        config.setdefault('dirs', ['.'])

        return config

    @classmethod
    def on_task_input(cls, task: Task, config: dict) -> List[Entry]:
        """
        Input task handler
        """

        config = cls.prepare_config(config)

        files_only: bool = config['files_only']
        recursive: bool = config['recursive']
        get_size: bool = config['get_size']
        socket_timeout_sec: int = config['socket_timeout_sec']
        connection_tries: int = config['connection_tries']
        directories: List[str] = []

        if isinstance(config['dirs'], list):
            directories.extend(config['dirs'])
        else:
            directories.append(config['dirs'])

        sftp_config: SftpConfig = task_config_to_sftp_config(config)
        sftp: SftpClient = sftp_connect(sftp_config, socket_timeout_sec, connection_tries)

        entries: List[Entry] = sftp.list_directories(directories, recursive, get_size, files_only)
        sftp.close()

        return entries


class SftpDownload:
    """
    Download files from a SFTP server. This plugin requires the pysftp Python module and its
    dependencies.

    Configuration:

    to:                  Destination path; supports Jinja2 templating on the input entry. Fields such
                         as series_name must be populated prior to input into this plugin using
                         metainfo_series or similar.
    recursive:           Indicates whether to download directory contents recursively.
    delete_origin:       Indicates whether to delete the remote files(s) once they've been downloaded.
    socket_timeout_sec:  Socket timeout in seconds
    connection_tries:    Number of times to attempt to connect before failing (default 3).

    Example:

      sftp_download:
          to: '/Volumes/External/Drobo/downloads'
          delete_origin: False
    """

    schema = {
        'type': 'object',
        'properties': {
            'to': {'type': 'string', 'format': 'path'},
            'recursive': {'type': 'boolean', 'default': True},
            'delete_origin': {'type': 'boolean', 'default': False},
            'socket_timeout_sec': {'type': 'integer', 'default': DEFAULT_SOCKET_TIMEOUT_SEC},
            'connection_tries': {'type': 'integer', 'default': DEFAULT_CONNECT_TRIES},
        },
        'required': ['to'],
        'additionalProperties': False,
    }

    @classmethod
    def download_entry(cls, entry: Entry, config: dict, sftp: SftpClient) -> None:
        """
        Downloads the file(s) described in entry
        """
        path: str = unquote(urlparse(entry['url']).path) or '.'
        delete_origin: bool = config['delete_origin']
        recursive: bool = config['recursive']
        to: str = config['to']

        try:
            sftp.download(path, to, recursive, delete_origin)
        except SftpError as e:
            entry.fail(e)  # type: ignore

    @classmethod
    def on_task_output(cls, task: Task, config: dict) -> None:
        """Register this as an output plugin"""
<commit_msg>Fix bug where sftp_download's `to` property wasn't rendering Jinja templates<commit_after>from collections import namedtuple
from itertools import groupby
from pathlib import Path
from typing import List, Optional
from urllib.parse import unquote, urlparse

from loguru import logger

from flexget import plugin
from flexget.components.ftp.sftp_client import SftpClient, SftpError
from flexget.config_schema import one_or_more
from flexget.entry import Entry
from flexget.event import event
from flexget.task import Task
from flexget.utils.template import RenderError, render_from_entry

logger = logger.bind(name='sftp')

# Constants
DEFAULT_SFTP_PORT: int = 22
DEFAULT_CONNECT_TRIES: int = 3
DEFAULT_SOCKET_TIMEOUT_SEC: int = 15


SftpConfig = namedtuple(
    'SftpConfig', ['host', 'port', 'username', 'password', 'private_key', 'private_key_pass']
)


class SftpList:
    """
    Generate entries from SFTP. This plugin requires the pysftp Python module and its dependencies.

    Configuration:

    host:                 Host to connect to.
    port:                 Port the remote SSH server is listening on (default 22).
    username:             Username to log in as.
    password:             The password to use. Optional if a private key is provided.
    private_key:          Path to the private key (if any) to log into the SSH server.
    private_key_pass:     Password for the private key (if needed).
    recursive:            Indicates whether the listing should be recursive.
    get_size:             Indicates whetern to calculate the size of the remote file/directory.
                          WARNING: This can be very slow when computing the size of directories!
    files_only:           Indicates wheter to omit diredtories from the results.
    dirs:                 List of directories to download.
    socket_timeout_sec:   Socket timeout in seconds (default 15 seconds).
    connection_tries:     Number of times to attempt to connect before failing (default 3).

    Example:

      sftp_list:
          host: example.com
          username: Username
          private_key: /Users/username/.ssh/id_rsa
          recursive: False
          get_size: True
          files_only: False
          dirs:
              - '/path/to/list/'
              - '/another/path/'
    """

    schema = {
        'type': 'object',
        'properties': {
            'host': {'type': 'string'},
            'username': {'type': 'string'},
            'password': {'type': 'string'},
            'port': {'type': 'integer', 'default': DEFAULT_SFTP_PORT},
            'files_only': {'type': 'boolean', 'default': True},
            'recursive': {'type': 'boolean', 'default': False},
            'get_size': {'type': 'boolean', 'default': True},
            'private_key': {'type': 'string'},
            'private_key_pass': {'type': 'string'},
            'dirs': one_or_more({'type': 'string'}),
            'socket_timeout_sec': {'type': 'integer', 'default': DEFAULT_SOCKET_TIMEOUT_SEC},
            'connection_tries': {'type': 'integer', 'default': DEFAULT_CONNECT_TRIES},
        },
        'additionProperties': False,
        'required': ['host', 'username'],
    }

    @staticmethod
    def prepare_config(config: dict) -> dict:
        """
        Sets defaults for the provided configuration
        """
        config.setdefault('password', None)
        config.setdefault('private_key', None)
        config.setdefault('private_key_pass', None)
        config.setdefault('dirs', ['.'])

        return config

    @classmethod
    def on_task_input(cls, task: Task, config: dict) -> List[Entry]:
        """
        Input task handler
        """

        config = cls.prepare_config(config)

        files_only: bool = config['files_only']
        recursive: bool = config['recursive']
        get_size: bool = config['get_size']
        socket_timeout_sec: int = config['socket_timeout_sec']
        connection_tries: int = config['connection_tries']
        directories: List[str] = []

        if isinstance(config['dirs'], list):
            directories.extend(config['dirs'])
        else:
            directories.append(config['dirs'])

        sftp_config: SftpConfig = task_config_to_sftp_config(config)
        sftp: SftpClient = sftp_connect(sftp_config, socket_timeout_sec, connection_tries)

        entries: List[Entry] = sftp.list_directories(directories, recursive, get_size, files_only)
        sftp.close()

        return entries


class SftpDownload:
    """
    Download files from a SFTP server. This plugin requires the pysftp Python module and its
    dependencies.

    Configuration:

    to:                  Destination path; supports Jinja2 templating on the input entry. Fields such
                         as series_name must be populated prior to input into this plugin using
                         metainfo_series or similar.
    recursive:           Indicates whether to download directory contents recursively.
    delete_origin:       Indicates whether to delete the remote files(s) once they've been downloaded.
    socket_timeout_sec:  Socket timeout in seconds
    connection_tries:    Number of times to attempt to connect before failing (default 3).

    Example:

      sftp_download:
          to: '/Volumes/External/Drobo/downloads'
          delete_origin: False
    """

    schema = {
        'type': 'object',
        'properties': {
            'to': {'type': 'string', 'format': 'path'},
            'recursive': {'type': 'boolean', 'default': True},
            'delete_origin': {'type': 'boolean', 'default': False},
            'socket_timeout_sec': {'type': 'integer', 'default': DEFAULT_SOCKET_TIMEOUT_SEC},
            'connection_tries': {'type': 'integer', 'default': DEFAULT_CONNECT_TRIES},
        },
        'required': ['to'],
        'additionalProperties': False,
    }

    @classmethod
    def download_entry(cls, entry: Entry, config: dict, sftp: SftpClient) -> None:
        """
        Downloads the file(s) described in entry
        """
        path: str = unquote(urlparse(entry['url']).path) or '.'
        delete_origin: bool = config['delete_origin']
        recursive: bool = config['recursive']
        to: str = config['to']

        try:
            to = render_from_entry(to, entry)
        except RenderError as e:
            logger.error('Could not render path: {}', to)
            entry.fail(str(e))  # type: ignore
            return

        try:
            sftp.download(path, to, recursive, delete_origin)
        except SftpError as e:
            entry.fail(e)  # type: ignore

    @classmethod
    def on_task_output(cls, task: Task, config: dict) -> None:
        """Register this as an output plugin"""

{'fraction_non_alphanumeric': 0.10620543164921863, 'fraction_numerical': 0.0018206645425580337, 'mean_word_length': 2.4931107578166403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7532790', 'n_tokens_mistral': 3697, 'n_tokens_neox': 3541, 'n_words': 1106}